diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e411947582333f378a851e465fa19616afcffb9..00baf4db0e93fd6b5894f5300d54bc56cfdd78a4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,8 @@ variables: OSDU_GCP_VENDOR: gcp OSDU_GCP_QUEUE_SA_EMAIL: pub-sub-indexer-queue@nice-etching-277309.iam.gserviceaccount.com OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_API/schema --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL" - OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s --set data.limits_cpu=1 --set data.limits_memory=1G" + OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: "--set data.key_ring=$OSDU_GCP_PARTITION_KEY_RING --set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_API/schema --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL" + OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s" OSDU_GCP_HELM_CONFIG_SERVICE: indexer-config OSDU_GCP_HELM_DEPLOYMENT_SERVICE: indexer-deploy OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap" @@ -46,7 +47,10 @@ include: file: "scanners/gitlab-ultimate.yml" - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/aws.yml" + file: "cloud-providers/aws-global.yml" + + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/aws-maven.yml" - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/azure.yml" @@ -57,10 +61,8 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "publishing/pages.yml" - - project: 'osdu/platform/ci-cd-pipelines' - ref: "master" - file: 'cloud-providers/osdu-gcp-gke.yml' - + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/osdu-gcp-gke.yml" aws-test-java: - tags: ['aws-internal-test'] + tags: ["aws-internal-test"] diff --git a/NOTICE b/NOTICE index cfbaa227ac26c58d0dd118846e95d03043fd3b11..efc25f4175db2c582c97a31de26de967d3a553e6 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,6 @@ # 3rd-Party Software License Notice Generated by fossa-cli (https://github.com/fossas/fossa-cli). +Formatted by fossa-with-cache (https://community.opengroup.org/divido/fossa-with-cache). This software includes the following software and licenses: ======================================================================== @@ -14,11 +15,11 @@ Apache-1.1 ======================================================================== The following software have components provided under the terms of this license: -- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/) -- Cobertura (from http://cobertura.sourceforge.net) +- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) -- Plexus Common Utilities (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) -- oro (from ) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) +- oro (from https://repo1.maven.org/maven2/oro/oro) ======================================================================== Apache-2.0 @@ -28,7 +29,6 @@ The following software have components provided under the terms of this license: - AMQP 1.0 JMS Spring Boot AutoConfiguration (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-autoconfigure) - AMQP 1.0 JMS Spring Boot Starter (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-starter) - ASM based accessors helper used by json-smart (from https://urielch.github.io/) -- ASM based accessors helper used by json-smart (from https://urielch.github.io/) - AWS Java SDK for AWS Amplify (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for AWS App Mesh (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for AWS AppSync (from https://aws.amazon.com/sdkforjava) @@ -272,7 +272,6 @@ The following software have components provided under the terms of this license: - AWS Java SDK for CodeArtifact (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for EC2 Image Builder (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for Elastic Load Balancing (from https://aws.amazon.com/sdkforjava) -- AWS Java SDK for Elastic Load Balancing (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for FinSpace Public API (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for FinSpace User Environment Management (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for Firewall Management (from https://aws.amazon.com/sdkforjava) @@ -288,60 +287,56 @@ The following software have components provided under the terms of this license: - AWS SDK for Java - Core (from https://aws.amazon.com/sdkforjava) - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava) - Adapter: RxJava (from https://repo1.maven.org/maven2/com/squareup/retrofit2/adapter-rxjava) -- Apache Ant + JUnit (from http://ant.apache.org/) -- Apache Ant Core (from ) +- Apache Ant + JUnit (from http://ant.apache.org/, https://ant.apache.org/) +- Apache Ant Core - Apache Ant Launcher (from http://ant.apache.org/) -- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/) -- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/) +- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli) - Apache Commons Codec (from https://commons.apache.org/proper/commons-codec/) - Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/) +- Apache Commons IO (from http://commons.apache.org/io/, https://commons.apache.org/proper/commons-io/, https://repo1.maven.org/maven2/commons-io/commons-io) - Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/) +- Apache Commons Logging (from http://commons.apache.org/logging/, http://commons.apache.org/proper/commons-logging/) +- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/, http://jakarta.apache.org/commons/${pom.artifactId.substring(8)}/, https://repo1.maven.org/maven2/commons-validator/commons-validator) - Apache Geronimo JMS Spec 2.0 (from http://geronimo.apache.org/maven/${siteId}/${version}) +- Apache Groovy (from http://groovy-lang.org, http://groovy.codehaus.org/, https://groovy-lang.org) - Apache HTTP transport v2 for the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-apache-v2) - Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient) -- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client) -- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga) +- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core-ga/, http://hc.apache.org/httpcomponents-core/) +- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core/) - Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api) - Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core) - Apache Log4j JUL Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul) - Apache Log4j SLF4J Binding (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl) - Apache Log4j to SLF4J Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-to-slf4j) -- Apache Lucene (module: grouping) (from https://lucene.apache.org/) -- Apache Lucene (module: highlighter) (from https://lucene.apache.org/) -- Apache Lucene (module: join) (from https://lucene.apache.org/) -- Apache Lucene (module: memory) (from https://lucene.apache.org/) -- Apache Lucene (module: spatial3d) (from https://lucene.apache.org/) +- Apache Maven Invoker (from https://repo1.maven.org/maven2/org/apache/maven/shared/maven-invoker) - Apache Maven Wagon :: API (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-provider-api) +- Apache Maven Wagon :: Providers :: File Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-file) - Apache Maven Wagon :: Providers :: HTTP Shared Library (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-http-shared) - Apache Maven Wagon :: Providers :: Lightweight HTTP Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-http-lightweight) - Apache Maven Wagon :: Providers :: SSH Common Library (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh-common) - Apache Maven Wagon :: Providers :: SSH External Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh-external) - Apache Maven Wagon :: Providers :: SSH Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh) -- Apache Velocity (from http://velocity.apache.org/engine/releases/velocity-1.6.1/) +- Apache Velocity (from http://velocity.apache.org/engine/releases/velocity-1.6.1/, http://velocity.apache.org/engine/releases/velocity-1.6.2/) - AssertJ fluent assertions (from https://repo1.maven.org/maven2/org/assertj/assertj-core) - Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client) - Asynchronous Http Client Netty Utils (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client-netty-utils) -- AutoValue Annotations (from https://github.com/google/auto/tree/master/value) -- AutoValue Annotations (from https://github.com/google/auto/tree/master/value) +- AutoValue Annotations (from https://github.com/google/auto/tree/master/value, https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations) - Azure Spring Boot Starter for Azure AD Spring Security Integration (from https://github.com/Azure/azure-sdk-for-java) -- BSON (from https://bsonspec.org) +- BSON (from http://bsonspec.org, https://bsonspec.org) - Bean Validation API (from http://beanvalidation.org) - Brave (from https://repo1.maven.org/maven2/io/zipkin/brave/brave) - Brave Instrumentation: Http Adapters (from https://repo1.maven.org/maven2/io/zipkin/brave/brave-instrumentation-http) - Brave instrumentation for Reactor Netty HTTP (from https://github.com/reactor/reactor-netty) -- Byte Buddy (without dependencies) (from ) +- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy) - Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent) - ClassMate (from http://github.com/cowtowncoder/java-classmate) -- Cloud Key Management Service (KMS) API v1-rev88-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms) -- Cloud Key Management Service (KMS) API v1-rev88-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms) -- Cloud Storage JSON API v1-rev20211201-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage) -- Cloud Storage JSON API v1-rev20211201-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage) +- Cloud Key Management Service (KMS) API v1-rev81-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms) +- Cloud Storage API v1-rev40-1.20.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage) - CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava) -- Cobertura (from http://cobertura.sourceforge.net) -- Commons Logging (from http://commons.apache.org/logging/) -- Converter: Jackson (from https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson) +- Cobertura code coverage (from http://cobertura.sourceforge.net) +- Collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections) +- Converter: Jackson (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson) - Core functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty) -- Data Mapper for Jackson (from http://jackson.codehaus.org) - Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) - Digester (from https://repo1.maven.org/maven2/commons-digester/commons-digester) - Doxia :: APT Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-apt) @@ -350,321 +345,260 @@ The following software have components provided under the terms of this license: - Doxia :: Sink API (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-sink-api) - Doxia :: XDoc Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-xdoc) - Doxia :: XHTML Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-xhtml) -- Doxia Sitetools :: Decoration Model (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-decoration-model) -- Doxia Sitetools :: Site Renderer (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-site-renderer) +- Doxia Sitetools :: Decoration Model (from http://maven.apache.org/doxia/doxia-sitetools/doxia-decoration-model/, https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-decoration-model) +- Doxia Sitetools :: Site Renderer (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/, https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-site-renderer) - Elastic JNA Distribution (from https://github.com/java-native-access/jna) -- Expression Language 3.0 (from http://el-spec.java.net) - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) - GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson) -- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson) - Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) -- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) -- Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine) - Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine) -- Google Cloud Core (from https://github.com/googleapis/java-core) -- Google Cloud Core (from https://github.com/googleapis/java-core) -- Google Cloud Core HTTP (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http) -- Google Cloud Core HTTP (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http) -- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc) -- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc) -- Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore) -- Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore) +- Google Cloud Core (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core, https://github.com/googleapis/java-core) +- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http, https://github.com/googleapis/java-core) +- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-grpc, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core) +- Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore, https://github.com/googleapis/java-datastore) - Google Cloud IAM Service Account Credentials (from https://github.com/googleapis/java-iamcredentials) -- Google Cloud IAM Service Account Credentials (from https://github.com/googleapis/java-iamcredentials) -- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging) -- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging) -- Google Cloud Pub/Sub (from https://github.com/googleapis/java-pubsub) -- Google Cloud Pub/Sub (from https://github.com/googleapis/java-pubsub) -- Google Cloud Storage (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-storage) -- Google Cloud Storage (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-storage) -- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client) -- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client) +- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging, https://github.com/googleapis/java-logging) +- Google Cloud Pub/Sub (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub, https://github.com/googleapis/java-pubsub) +- Google Cloud Storage (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-storage, https://github.com/googleapis/java-storage) - Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client) - Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client) -- Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client) -- Gson (from https://repo1.maven.org/maven2/com/google/code/gson/gson) -- Gson (from https://repo1.maven.org/maven2/com/google/code/gson/gson) +- Gson (from http://code.google.com/p/google-gson/, https://repo1.maven.org/maven2/com/google/code/gson/gson) - Guava InternalFutureFailureAccess and InternalFutures (from https://repo1.maven.org/maven2/com/google/guava/failureaccess) -- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) -- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) +- Guava: Google Core Libraries for Java (from http://code.google.com/p/guava-libraries, https://github.com/google/guava, https://repo1.maven.org/maven2/com/google/guava/guava) - HPPC Collections (from https://repo1.maven.org/maven2/com/carrotsearch/hppc) - HTTP functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty) -- Hibernate Validator Engine (from https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator) +- Hibernate Validator (from https://repo1.maven.org/maven2/org/hibernate/hibernate-validator, https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator) +- HikariCP (from https://github.com/brettwooldridge/HikariCP) +- Hop (from https://github.com/rabbitmq/hop, https://www.rabbitmq.com) - HttpClient (from http://hc.apache.org/httpcomponents-client) -- HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga) +- HttpClient Cache (from http://hc.apache.org/httpcomponents-client) - IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java) - IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java) - IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java) - IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java) -- Identity and Access Management (IAM) API v1-rev281-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam) -- J2ObjC Annotations (from https://github.com/google/j2objc/) +- Identity and Access Management (IAM) API v1-rev289-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam) - J2ObjC Annotations (from https://github.com/google/j2objc/) - JBoss Logging 3 (from http://www.jboss.org) - JCIP Annotations under Apache License (from http://stephenc.github.com/jcip-annotations) -- JDOM (from ) +- JDOM - JMES Path Query library (from https://aws.amazon.com/sdkforjava) -- JSON Small and Fast Parser (from https://urielch.github.io/) -- JSON Small and Fast Parser (from https://urielch.github.io/) +- JSON Small and Fast Parser (from https://repo1.maven.org/maven2/net/minidev/json-smart, https://urielch.github.io/) - JSON Web Token support for the JVM (from https://repo1.maven.org/maven2/io/jsonwebtoken/jjwt) - JSON library from Android SDK (from http://developer.android.com/sdk) - JSON.simple (from http://code.google.com/p/json-simple/) - JSONassert (from http://github.com/skyscreamer/yoga) - JSR107 API and SPI (from https://github.com/jsr107/jsr107spec) - Jackson (from http://jackson.codehaus.org) -- Jackson (from http://jackson.codehaus.org) -- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2) - Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2) - Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary) -- Jackson dataformat: Smile (from ) -- Jackson dataformat: Smile (from ) -- Jackson datatype: JSR310 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310) -- Jackson datatype: JSR310 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310) -- Jackson datatype: jdk8 (from ) -- Jackson datatype: jdk8 (from ) +- Jackson dataformat: Smile (from http://github.com/FasterXML/jackson-dataformat-smile, http://github.com/FasterXML/jackson-dataformats-binary) +- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310, https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310) +- Jackson datatype: Joda (from http://wiki.fasterxml.com/JacksonModuleJoda, https://github.com/FasterXML/jackson-datatype-joda) +- Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8) - Jackson extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson) -- Jackson module: Afterburner (from https://github.com/FasterXML/jackson-modules-base) -- Jackson-annotations (from http://github.com/FasterXML/jackson) -- Jackson-annotations (from http://github.com/FasterXML/jackson) -- Jackson-core (from https://github.com/FasterXML/jackson-core) -- Jackson-dataformat-XML (from https://github.com/FasterXML/jackson-dataformat-xml) -- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text) -- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text) -- Jackson-datatype-Joda (from http://wiki.fasterxml.com/JacksonModuleJoda) -- Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations) -- Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations) -- Jackson-module-parameter-names (from ) -- Jackson-module-parameter-names (from ) +- Jackson module: Afterburner (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-modules-base) +- Jackson module: JAXB Annotations (from http://github.com/FasterXML/jackson-module-jaxb-annotations, http://wiki.fasterxml.com/JacksonJAXBAnnotations, https://github.com/FasterXML/jackson-modules-base) +- Jackson-annotations (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome) +- Jackson-core (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-core) +- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding, https://github.com/FasterXML/jackson-dataformat-xml) +- Jackson-dataformat-YAML (from http://wiki.fasterxml.com/JacksonExtensionYAML, https://github.com/FasterXML/jackson, https://github.com/FasterXML/jackson-dataformats-text) +- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names) - Jakarta Bean Validation API (from https://beanvalidation.org) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- Jakarta Servlet (from http://servlet-spec.java.net, https://projects.eclipse.org/projects/ee4j.servlet) - Java Libraries for Amazon Simple WorkFlow (from https://aws.amazon.com/sdkforjava) -- Java Native Access (from https://github.com/twall/jna) +- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) -- Java Servlet API (from http://servlet-spec.java.net) - Java UUID Generator (from http://wiki.fasterxml.com/JugHome) - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) - JetBrains Java Annotations (from https://github.com/JetBrains/java-annotations) -- Jetty Server (from https://repo1.maven.org/maven2/org/mortbay/jetty/jetty) -- Jetty Utilities (from https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util) -- Joda-Time (from http://www.joda.org/joda-time/) +- Jetty :: Utilities (from http://jetty.mortbay.org, http://www.eclipse.org/jetty, https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util) +- Jetty Server (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty) +- Joda-Time (from http://joda-time.sourceforge.net, http://www.joda.org/joda-time/, https://www.joda.org/joda-time/) - KeePassJava2 :: All (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2) - KeePassJava2 :: DOM (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-dom) - KeePassJava2 :: JAXB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-jaxb) - KeePassJava2 :: KDB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdb) - KeePassJava2 :: KDBX (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdbx) - KeePassJava2 :: Simple (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-simple) +- Kotlin Stdlib (from https://kotlinlang.org/, https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib) +- Kotlin Stdlib Common (from https://kotlinlang.org/) - Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang) - Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic) - Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core) - Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Lucene Core (from ) -- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs) +- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) +- Lucene Grouping (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping) +- Lucene Highlighter (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter) +- Lucene Join (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-join) +- Lucene Memory (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs, https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory) - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc) -- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries) -- Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser) -- Lucene Sandbox (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox) +- Lucene Queries (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries) +- Lucene QueryParsers (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser) +- Lucene Sandbox (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox) +- Lucene Spatial 3D (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial3d) - Lucene Spatial Extras (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial-extras) - Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest) -- MapStruct Core (from http://mapstruct.org/mapstruct/) +- MapStruct Core (from http://mapstruct.org/mapstruct/, https://repo1.maven.org/maven2/org/mapstruct/mapstruct) - Maven Artifact (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact) - Maven Artifact Manager (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact-manager) - Maven Core (from https://repo1.maven.org/maven2/org/apache/maven/maven-core) - Maven Error Diagnostics (from https://repo1.maven.org/maven2/org/apache/maven/maven-error-diagnostics) -- Maven Local Settings Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-settings) - Maven Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-model) - Maven Monitor (from https://repo1.maven.org/maven2/org/apache/maven/maven-monitor) - Maven Plugin API (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-api) - Maven Plugin Descriptor Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-descriptor) - Maven Plugin Parameter Documenter API (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-parameter-documenter) - Maven Plugin Registry Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-registry) -- Maven Process Invoker (from https://repo1.maven.org/maven2/org/apache/maven/shared/maven-invoker) - Maven Profile Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-profile) - Maven Project Builder (from https://repo1.maven.org/maven2/org/apache/maven/maven-project) - Maven Reporting API (from https://repo1.maven.org/maven2/org/apache/maven/reporting/maven-reporting-api) - Maven Reporting Implementation (from https://repo1.maven.org/maven2/org/apache/maven/reporting/maven-reporting-impl) - Maven Repository Metadata Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-repository-metadata) -- Maven Wagon File Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-file) -- Metrics Core (from ) +- Maven Settings (from https://repo1.maven.org/maven2/org/apache/maven/maven-settings) +- Metrics Core (from https://repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) -- Mockito (from http://www.mockito.org) +- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito) - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/) - MongoDB Driver (from http://www.mongodb.org) -- MongoDB Java Driver (from http://www.mongodb.org) -- MongoDB Java Driver (from http://www.mongodb.org) -- MongoDB Java Driver Core (from http://www.mongodb.org) +- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org) - Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer) -- Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer) -- Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec) - Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec) - Netty/Codec/DNS (from https://repo1.maven.org/maven2/io/netty/netty-codec-dns) - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) -- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) - Netty/Codec/HTTP2 (from https://repo1.maven.org/maven2/io/netty/netty-codec-http2) - Netty/Codec/Socks (from https://repo1.maven.org/maven2/io/netty/netty-codec-socks) - Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) -- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) -- Netty/Handler (from https://repo1.maven.org/maven2/io/netty/netty-handler) - Netty/Handler (from https://repo1.maven.org/maven2/io/netty/netty-handler) - Netty/Handler/Proxy (from https://repo1.maven.org/maven2/io/netty/netty-handler-proxy) - Netty/Resolver (from https://repo1.maven.org/maven2/io/netty/netty-resolver) -- Netty/Resolver (from https://repo1.maven.org/maven2/io/netty/netty-resolver) - Netty/Resolver/DNS (from https://repo1.maven.org/maven2/io/netty/netty-resolver-dns) +- Netty/Resolver/DNS/Classes/MacOS (from https://repo1.maven.org/maven2/io/netty/netty-resolver-dns-classes-macos) - Netty/TomcatNative [BoringSSL - Static] (from https://repo1.maven.org/maven2/io/netty/netty-tcnative-boringssl-static) - Netty/Transport (from https://repo1.maven.org/maven2/io/netty/netty-transport) -- Netty/Transport (from https://repo1.maven.org/maven2/io/netty/netty-transport) -- Netty/Transport/Native/Unix/Common (from https://repo1.maven.org/maven2/io/netty/netty-transport-native-unix-common) +- Netty/Transport/Classes/Epoll (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-epoll) +- Netty/Transport/Classes/KQueue (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-kqueue) - Netty/Transport/Native/Unix/Common (from https://repo1.maven.org/maven2/io/netty/netty-transport-native-unix-common) - Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type) +- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt, https://bitbucket.org/nimbusds/nimbus-jose-jwt) - Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags) -- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags) -- NimbusDS JOSE JWT (from https://bitbucket.org/nimbusds/nimbus-jose-jwt) -- NimbusDS JOSE JWT (from https://bitbucket.org/nimbusds/nimbus-jose-jwt) -- NimbusDS JOSE JWT (from https://bitbucket.org/nimbusds/nimbus-jose-jwt) -- NimbusDS JOSE JWT (from https://bitbucket.org/nimbusds/nimbus-jose-jwt) -- NimbusDS JOSE JWT (from https://bitbucket.org/nimbusds/nimbus-jose-jwt) -- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor-core) -- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) -- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) -- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) +- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor, https://github.com/reactor/reactor-core) - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) - OAuth2 for Spring Security (from https://repo1.maven.org/maven2/org/springframework/security/oauth/spring-security-oauth2) -- OAuth2 for Spring Security (from https://repo1.maven.org/maven2/org/springframework/security/oauth/spring-security-oauth2) - Objenesis (from http://objenesis.org) -- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp) -- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp) -- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection) -- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection) -- Okio (from https://github.com/square/okio/) -- Okio (from https://github.com/square/okio/) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) +- OkHttp (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp, https://square.github.io/okhttp/) +- OkHttp Logging Interceptor (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor, https://square.github.io/okhttp/) +- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection, https://square.github.io/okhttp/) +- Okio (from https://github.com/square/okio/, https://repo1.maven.org/maven2/com/squareup/okio/okio) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) - PWDB :: Database (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/database) -- Plexus Common Utilities (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) - Plexus Velocity Component (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-velocity) +- PostgreSQL JDBC Driver +- PowerMock (from http://www.powermock.org, https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito) - Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf) -- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf) -- Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j) - Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j) - QpidJMS Client (from https://repo1.maven.org/maven2/org/apache/qpid/qpid-jms-client) -- RabbitMQ Java Client (from http://www.rabbitmq.com) -- Reactor Netty with all modules (from https://github.com/reactor/reactor-netty) -- Retrofit (from https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit) -- Servlet Specification 2.5 API (from https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) -- SnakeYAML (from https://bitbucket.org/snakeyaml/snakeyaml) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) +- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty) +- Retrofit (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit) +- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) +- Simple XML (safe) (from https://github.com/dweiss/simplexml) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) - Spatial4J (from https://projects.eclipse.org/projects/locationtech.spatial4j) -- Spring AMQP Core (from https://github.com/spring-projects/spring-amqp) -- Spring AOP (from https://github.com/spring-projects/spring-framework) -- Spring Beans (from https://github.com/spring-projects/spring-framework) -- Spring Boot (from http://projects.spring.io/spring-boot/) -- Spring Boot AMQP Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-amqp) -- Spring Boot AOP Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-aop) -- Spring Boot Actuator (from http://projects.spring.io/spring-boot/) -- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure) -- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/) -- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/) -- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/) -- Spring Boot Jersey Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-jersey) -- Spring Boot Log4j 2 Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-log4j2) -- Spring Boot Logging Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Reactor Netty Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-reactor-netty) -- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Test (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-test) -- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Tomcat Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat) -- Spring Boot Validation Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-validation) -- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot WebFlux Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-webflux) +- Spring AMQP Core (from http://www.springsource.org/spring-amqp, https://github.com/spring-projects/spring-amqp, https://projects.spring.io/spring-amqp) +- Spring AOP (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-aop) +- Spring Beans (from http://www.springframework.org, https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-beans) +- Spring Boot (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot, https://spring.io/projects/spring-boot) +- Spring Boot AMPQ Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-amqp, https://spring.io/projects/spring-boot) +- Spring Boot AOP Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Actuator (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator, https://spring.io/projects/spring-boot) +- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure, https://spring.io/projects/spring-boot) +- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-actuator, https://spring.io/projects/spring-boot) +- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-autoconfigure, https://spring.io/projects/spring-boot) +- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-tools/spring-boot-configuration-processor, https://spring.io/projects/spring-boot) +- Spring Boot Data MongoDB Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-data-mongodb, https://spring.io/projects/spring-boot) +- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot JDBC Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-jersey, https://spring.io/projects/spring-boot) +- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json, https://spring.io/projects/spring-boot) +- Spring Boot Log4j 2 Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-log4j2, https://spring.io/projects/spring-boot) +- Spring Boot Logging Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-logging, https://spring.io/projects/spring-boot) +- Spring Boot Reactor Netty Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-reactor-netty, https://spring.io/projects/spring-boot) +- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Test (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Test Auto-Configure (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-test-autoconfigure, https://spring.io/projects/spring-boot) +- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Tomcat Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat, https://spring.io/projects/spring-boot) +- Spring Boot Validation Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-validation, https://spring.io/projects/spring-boot) +- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot WebFlux Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-webflux, https://spring.io/projects/spring-boot) - Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework) -- Spring Context (from https://github.com/spring-projects/spring-framework) -- Spring Core (from https://github.com/spring-projects/spring-framework) -- Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons) +- Spring Context (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-context) +- Spring Core (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-core) - Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons) - Spring Data MongoDB - Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-mongodb) -- Spring Expression Language (SpEL) (from https://github.com/spring-projects/spring-framework) -- Spring JMS (from https://github.com/spring-projects/spring-framework) +- Spring Expression Language (SpEL) (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-expression) +- Spring JMS (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-jms) - Spring Messaging (from https://github.com/spring-projects/spring-framework) +- Spring Plugin - Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core) - Spring Plugin - Metadata Extension (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-metadata) -- Spring Plugin Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core) -- Spring RabbitMQ Support (from https://github.com/spring-projects/spring-amqp) +- Spring RabbitMQ Support (from http://www.springsource.org/spring-amqp, https://github.com/spring-projects/spring-amqp, https://projects.spring.io/spring-amqp) - Spring Retry (from http://www.springsource.org) +- Spring Security - Core (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-core, https://spring.io/projects/spring-security, https://spring.io/spring-security) +- Spring Security - Namespace Configuration Module (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-config, https://spring.io/spring-security) +- Spring Security - Web Application Security Module (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-web, https://spring.io/projects/spring-security, https://spring.io/spring-security) - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) -- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) -- Spring TestContext Framework (from https://github.com/spring-projects/spring-framework) -- Spring Transaction (from https://github.com/spring-projects/spring-framework) -- Spring Web (from https://github.com/spring-projects/spring-framework) -- Spring Web MVC (from https://github.com/spring-projects/spring-framework) +- Spring TestContext Framework (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework) +- Spring Transaction (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework) +- Spring Web (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-web) +- Spring Web MVC (from https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-webmvc) - Spring WebFlux (from https://github.com/spring-projects/spring-framework) - T-Digest (from https://github.com/tdunning/t-digest) +- Vavr (from http://vavr.io, https://www.vavr.io) - Vavr Match (from http://vavr.io) -- Woodstox (from ) -- Zipkin Core Library (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin) +- Woodstox (from https://github.com/FasterXML/woodstox) +- Xerces2-j (from https://repo1.maven.org/maven2/xerces/xercesImpl, https://xerces.apache.org/xerces2-j/) - Zipkin Reporter Brave (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter-brave) - Zipkin Reporter: Core (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter) -- aalto-xml (from https://repo1.maven.org/maven2/com/fasterxml/aalto-xml) +- Zipkin v2 (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin) +- aalto-xml (from https://github.com/FasterXML/aalto-xml, https://repo1.maven.org/maven2/com/fasterxml/aalto-xml) - aggs-matrix-stats (from https://github.com/elastic/elasticsearch) -- commons-collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections) -- commons-io (from https://repo1.maven.org/maven2/commons-io/commons-io) -- commons-validator (from https://repo1.maven.org/maven2/commons-validator/commons-validator) +- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis) +- commons-beanutils (from https://repo1.maven.org/maven2/commons-beanutils/commons-beanutils) - compiler (from http://github.com/spullara/mustache.java) -- core (from https://github.com/elastic/elasticsearch) -- datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client) - datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client) +- elasticsearch (from https://github.com/elastic/elasticsearch, https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch) - elasticsearch-cli (from https://github.com/elastic/elasticsearch) - elasticsearch-core (from https://github.com/elastic/elasticsearch) - elasticsearch-geo (from https://github.com/elastic/elasticsearch) - elasticsearch-secure-sm (from https://github.com/elastic/elasticsearch) - elasticsearch-x-content (from https://github.com/elastic/elasticsearch) - error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) -- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) -- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) -- grpc-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/grpc-google-cloud-pubsub-v1) -- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) - io.grpc:grpc-api (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-api (from https://github.com/grpc/grpc-java) - io.grpc:grpc-auth (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-context (from https://github.com/grpc/grpc-java) - io.grpc:grpc-context (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-context (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-context (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-core (from https://github.com/grpc/grpc-java) - io.grpc:grpc-core (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-core (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-grpclb (from https://github.com/grpc/grpc-java) - io.grpc:grpc-grpclb (from https://github.com/grpc/grpc-java) - io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java) - io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) - io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) - io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) -- ion-java (from https://github.com/amznlabs/ion-java/) -- ion-java (from https://github.com/amznlabs/ion-java/) -- jackson-databind (from http://wiki.fasterxml.com/JacksonHome) -- java-cloudant (from https://cloudant.com) +- ion-java (from https://github.com/amzn/ion-java/, https://github.com/amznlabs/ion-java/) +- jackson-databind (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome) +- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject) - java-cloudant (from https://cloudant.com) - javatuples (from http://www.javatuples.org) - javax.inject (from http://code.google.com/p/atinject/) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -676,158 +610,133 @@ The following software have components provided under the terms of this license: - jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) - jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - jose4j (from https://bitbucket.org/b_c/jose4j/) -- json-path (from http://code.google.com/p/json-path/) +- json-path (from http://code.google.com/p/json-path/, https://github.com/jayway/JsonPath) - lang-mustache (from https://github.com/elastic/elasticsearch) -- lettuce (from http://github.com/mp911de/lettuce/wiki) +- lettuce (from http://github.com/mp911de/lettuce/wiki, https://github.com/lettuce-io/lettuce-core/wiki) - mapper-extras (from https://github.com/elastic/elasticsearch) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer) -- mockito-core (from http://mockito.org) -- mockito-core (from http://mockito.org) -- mockito-core (from http://mockito.org) -- nio-multipart-parser (from ) -- nio-stream-storage (from https://github.com/synchronoss/nio-stream-storage) -- okhttp-logging-interceptor (from https://square.github.io/okhttp/) +- minio (from https://github.com/minio/minio-java) +- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian) - org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/) -- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/) +- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j) +- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/, https://www.xmlunit.org/) - parent-join (from https://github.com/elastic/elasticsearch) - perfmark:perfmark-api (from https://github.com/perfmark/perfmark) - powermock-api-mockito2 (from https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito2) - powermock-api-support (from https://repo1.maven.org/maven2/org/powermock/powermock-api-support) -- powermock-core (from http://www.powermock.org) -- powermock-module-junit4 (from http://www.powermock.org) -- powermock-module-junit4-common (from https://repo1.maven.org/maven2/org/powermock/powermock-module-junit4-common) - powermock-reflect (from https://repo1.maven.org/maven2/org/powermock/powermock-reflect) -- proto-google-cloud-datastore-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-datastore-v1) -- proto-google-cloud-datastore-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-datastore-v1) -- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1) -- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1) -- proto-google-cloud-logging-v2 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2) -- proto-google-cloud-logging-v2 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2) -- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1) -- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1) -- proto-google-common-protos (from https://github.com/googleapis/java-iam/proto-google-common-protos) -- proto-google-common-protos (from https://github.com/googleapis/java-iam/proto-google-common-protos) -- proto-google-common-protos (from https://github.com/googleapis/java-iam/proto-google-common-protos) -- proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1) -- proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1) +- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-datastore/proto-google-cloud-datastore-v1, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-datastore-v1) +- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-iamcredentials-v1) +- proto-google-cloud-logging-v2 (from https://github.com/googleapis/java-logging/proto-google-cloud-logging-v2, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2) +- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1) +- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos) +- proto-google-iam-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-iam-v1) - rank-eval (from https://github.com/elastic/elasticsearch) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) +- resilience4j (from https://github.com/resilience4j/resilience4j, https://resilience4j.readme.io, ttps://resilience4j.readme.io) - rest (from https://github.com/elastic/elasticsearch) - rest-high-level (from https://github.com/elastic/elasticsearch) - rxjava (from https://github.com/ReactiveX/RxJava) -- spring-boot-starter (from https://spring.io/projects/spring-boot) -- spring-boot-starter-data-mongodb (from https://spring.io/projects/spring-boot) -- spring-boot-starter-json (from https://spring.io/projects/spring-boot) -- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot) -- spring-security-config (from http://spring.io/spring-security) -- spring-security-core (from http://spring.io/spring-security) -- spring-security-oauth2-client (from https://spring.io/projects/spring-security) -- spring-security-oauth2-core (from https://spring.io/projects/spring-security) -- spring-security-oauth2-jose (from https://spring.io/projects/spring-security) -- spring-security-oauth2-jose (from https://spring.io/projects/spring-security) -- spring-security-test (from https://spring.io/spring-security) -- spring-security-web (from http://spring.io/spring-security) +- spring-jdbc (from https://repo1.maven.org/maven2/org/springframework/spring-jdbc) +- spring-security-oauth2-client (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security) +- spring-security-oauth2-core (from http://spring.io/spring-security, https://spring.io/spring-security) +- spring-security-oauth2-jose (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security) +- spring-security-test (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security) +- springfox-bean-validators (from https://github.com/springfox/springfox) +- springfox-boot-starter (from https://github.com/springfox/springfox) - springfox-core (from https://github.com/springfox/springfox) +- springfox-data-rest (from https://github.com/springfox/springfox) +- springfox-oas (from https://github.com/springfox/springfox) - springfox-schema (from https://github.com/springfox/springfox) - springfox-spi (from https://github.com/springfox/springfox) - springfox-spring-web (from https://github.com/springfox/springfox) +- springfox-spring-webflux (from https://github.com/springfox/springfox) +- springfox-spring-webmvc (from https://github.com/springfox/springfox) - springfox-swagger-common (from https://github.com/springfox/springfox) - springfox-swagger-ui (from https://github.com/springfox/springfox) - springfox-swagger2 (from https://github.com/springfox/springfox) -- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations) +- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/core/v3/swagger-annotations, https://repo1.maven.org/maven2/io/swagger/swagger-annotations) - swagger-jaxrs (from https://repo1.maven.org/maven2/io/swagger/swagger-jaxrs) -- swagger-models (from https://repo1.maven.org/maven2/io/swagger/swagger-models) -- tomcat-annotations-api (from http://tomcat.apache.org/) +- swagger-models (from https://repo1.maven.org/maven2/io/swagger/core/v3/swagger-models, https://repo1.maven.org/maven2/io/swagger/swagger-models) +- tomcat-annotations-api (from http://tomcat.apache.org/, https://tomcat.apache.org/) - tomcat-embed-core (from http://tomcat.apache.org/) -- tomcat-embed-el (from https://tomcat.apache.org/) -- tomcat-embed-websocket (from https://tomcat.apache.org/) -- vavr (from https://www.vavr.io) -- xercesImpl (from https://repo1.maven.org/maven2/xerces/xercesImpl) -- xml-apis (from ) +- tomcat-embed-el (from http://tomcat.apache.org/, https://tomcat.apache.org/) +- tomcat-embed-websocket (from http://tomcat.apache.org/, https://tomcat.apache.org/) +- xml-apis ======================================================================== BSD-2-Clause ======================================================================== The following software have components provided under the terms of this license: -- API Common (from https://github.com/googleapis/api-common-java) -- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) +- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java) +- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli) +- GAX (Google Api eXtensions) (from https://github.com/googleapis, https://github.com/googleapis/gax-java) +- Hamcrest (from http://hamcrest.org/JavaHamcrest/) +- Hamcrest Core (from https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Plexus Common Utilities (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) -- Reflections (from http://code.google.com/p/reflections/) -- Stax2 API (from ) -- ThreeTen backport (from https://github.com/ThreeTen/threetenbp) -- ThreeTen backport (from https://github.com/ThreeTen/threetenbp) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) +- PostgreSQL JDBC Driver +- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections) +- Stax2 API (from http://github.com/FasterXML/stax2-api) +- ThreeTen backport (from https://github.com/ThreeTen/threetenbp, https://www.threeten.org/threetenbp) +- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) - jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) ======================================================================== BSD-3-Clause ======================================================================== The following software have components provided under the terms of this license: -- API Common (from https://github.com/googleapis/api-common-java) -- API Common (from https://github.com/googleapis/api-common-java) -- ASM Analysis (from https://repo1.maven.org/maven2/org/ow2/asm/asm-analysis) +- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java) +- ASM Analysis (from http://asm.ow2.io/) +- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons) +- ASM Core (from http://asm.ow2.io/, http://asm.ow2.org/) +- ASM Tree (from http://asm.ow2.io/) +- ASM Util (from http://asm.ow2.io/) - ASM library repackaged as OSGi bundle (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) - Apache Commons Codec (from https://commons.apache.org/proper/commons-codec/) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) +- AspectJ Weaver (from http://www.aspectj.org, https://www.eclipse.org/aspectj/) +- GAX (Google Api eXtensions) (from https://github.com/googleapis, https://github.com/googleapis/gax-java) - Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) - Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials) -- Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials) -- Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http) - Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http) +- Hamcrest (from http://hamcrest.org/JavaHamcrest/) +- Hamcrest Core (from https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core) - JSch (from http://www.jcraft.com/jsch/) - Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) -- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api) +- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Lucene Core (from ) +- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest) +- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Mockito (from http://www.mockito.org) +- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito) - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) -- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) -- Plexus Common Utilities (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) -- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) -- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) -- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) -- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util) +- Netty/Transport/Classes/KQueue (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-kqueue) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) +- PostgreSQL JDBC Driver +- Protocol Buffer Java API (from http://code.google.com/p/protobuf, https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) - Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util) -- Reflections (from http://code.google.com/p/reflections/) -- SnakeYAML (from https://bitbucket.org/snakeyaml/snakeyaml) -- Spring Core (from https://github.com/spring-projects/spring-framework) -- ThreeTen backport (from https://github.com/ThreeTen/threetenbp) -- ThreeTen backport (from https://github.com/ThreeTen/threetenbp) -- asm (from ) -- asm (from ) -- asm-commons (from http://asm.ow2.org/) -- asm-tree (from http://asm.ow2.org/) -- asm-util (from http://asm.ow2.org/) -- classworlds (from http://classworlds.codehaus.org/) -- jaxen (from http://jaxen.codehaus.org/) +- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) +- Spring Core (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-core) +- ThreeTen backport (from https://github.com/ThreeTen/threetenbp, https://www.threeten.org/threetenbp) +- classworlds (from http://classworlds.codehaus.org/, https://repo1.maven.org/maven2/classworlds/classworlds) +- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -842,8 +751,7 @@ CC-BY-2.5 The following software have components provided under the terms of this license: - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) -- MongoDB Java Driver (from http://www.mongodb.org) -- MongoDB Java Driver Core (from http://www.mongodb.org) +- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org) ======================================================================== CC-BY-4.0 @@ -874,12 +782,12 @@ The following software have components provided under the terms of this license: - HK2 configuration module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-config) - HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) - HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) -- JavaBeans Activation Framework API jar (from ) +- JavaBeans Activation Framework API jar - Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) - ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) -- Servlet Specification 2.5 API (from https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) +- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) - aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -895,15 +803,14 @@ CDDL-1.1 ======================================================================== The following software have components provided under the terms of this license: -- Expression Language 3.0 (from http://el-spec.java.net) -- Java Servlet API (from http://servlet-spec.java.net) -- JavaBeans Activation Framework (from ) +- Jakarta Servlet (from http://servlet-spec.java.net, https://projects.eclipse.org/projects/ee4j.servlet) +- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) +- JavaBeans Activation Framework - JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp) -- JavaMail API (from ) +- JavaMail API (from https://repo1.maven.org/maven2/com/sun/mail/javax.mail) - OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) -- Servlet Specification 2.5 API (from https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) +- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) - javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250) -- jaxb-api (from https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) - jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) - tomcat-embed-core (from http://tomcat.apache.org/) @@ -913,26 +820,92 @@ CPL-1.0 ======================================================================== The following software have components provided under the terms of this license: -- JUnit (from ) -- System Rules (from http://stefanbirkner.github.io/system-rules/) +- JUnit (from http://junit.org) +- System Rules (from http://stefanbirkner.github.com/system-rules/) ======================================================================== EPL-1.0 ======================================================================== The following software have components provided under the terms of this license: -- AspectJ Weaver (from https://www.eclipse.org/aspectj/) -- Logback Classic Module (from http://logback.qos.ch) +- AspectJ Weaver (from http://www.aspectj.org, https://www.eclipse.org/aspectj/) +- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) +- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) +- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) +- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) +- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) +- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) +- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) +- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- Logback Classic Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-classic) - Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic) - Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core) - Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson) -- Logback Core Module (from http://logback.qos.ch) +- Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- SnakeYAML (from https://bitbucket.org/snakeyaml/snakeyaml) +- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) +- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) +- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) +- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) +- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common) +- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) +- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) +- jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core) -- jts-io-common (from ) +- jts-io-common (from https://repo1.maven.org/maven2/org/locationtech/jts/io/jts-io-common) + +======================================================================== +EPL-2.0 +======================================================================== +The following software have components provided under the terms of this license: + +- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) +- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) +- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) +- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) +- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) +- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) +- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) +- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) +- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) +- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) +- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) +- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common) +- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) +- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) +- jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) ======================================================================== GPL-2.0-only @@ -941,27 +914,26 @@ The following software have components provided under the terms of this license: - ASM library repackaged as OSGi bundle (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) - Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) -- Cobertura (from http://cobertura.sourceforge.net) - Cobertura Limited Runtime (from http://cobertura.sourceforge.net) -- Expression Language 3.0 (from http://el-spec.java.net) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) - HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) - HK2 config types (from https://repo1.maven.org/maven2/org/glassfish/hk2/config-types) - HK2 configuration module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-config) - HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) - HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) -- Java Servlet API (from http://servlet-spec.java.net) -- JavaBeans Activation Framework (from ) +- Jakarta Servlet (from http://servlet-spec.java.net, https://projects.eclipse.org/projects/ee4j.servlet) +- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) +- JavaBeans Activation Framework - Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang) - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/) - OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) - Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) - ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) - aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) - javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) -- jaxb-api (from https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -980,7 +952,7 @@ GPL-2.0-or-later ======================================================================== The following software have components provided under the terms of this license: -- SnakeYAML (from https://bitbucket.org/snakeyaml/snakeyaml) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) ======================================================================== GPL-2.0-with-classpath-exception @@ -988,18 +960,25 @@ GPL-2.0-with-classpath-exception The following software have components provided under the terms of this license: - Checker Qual (from https://checkerframework.org) -- Cobertura (from http://cobertura.sourceforge.net) -- Expression Language 3.0 (from http://el-spec.java.net) +- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) -- Java Servlet API (from http://servlet-spec.java.net) -- JavaBeans Activation Framework (from ) +- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) +- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) +- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) +- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- Jakarta Servlet (from http://servlet-spec.java.net, https://projects.eclipse.org/projects/ee4j.servlet) +- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) +- JavaBeans Activation Framework - OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) +- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) - ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) - aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) +- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject) - javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) -- jaxb-api (from https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -1011,6 +990,7 @@ The following software have components provided under the terms of this license: - jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) - jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - tomcat-embed-core (from http://tomcat.apache.org/) ======================================================================== @@ -1018,8 +998,11 @@ GPL-3.0-only ======================================================================== The following software have components provided under the terms of this license: -- Project Lombok (from https://projectlombok.org) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) +- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) +- Project Lombok (from http://projectlombok.org, https://projectlombok.org) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) ======================================================================== JSON @@ -1033,18 +1016,17 @@ LGPL-2.1-only ======================================================================== The following software have components provided under the terms of this license: -- Cobertura (from http://cobertura.sourceforge.net) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - Elastic JNA Distribution (from https://github.com/java-native-access/jna) -- Java Native Access (from https://github.com/twall/jna) +- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) - Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang) -- Logback Classic Module (from http://logback.qos.ch) +- Logback Classic Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-classic) - Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic) - Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core) - Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson) -- Logback Core Module (from http://logback.qos.ch) +- Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -1055,8 +1037,7 @@ LGPL-2.1-or-later The following software have components provided under the terms of this license: - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) -- SnakeYAML (from https://bitbucket.org/snakeyaml/snakeyaml) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) ======================================================================== LGPL-3.0-only @@ -1065,7 +1046,7 @@ The following software have components provided under the terms of this license: - Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api) - Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) ======================================================================== MIT @@ -1076,22 +1057,20 @@ The following software have components provided under the terms of this license: - AWS Java SDK for AWS License Manager (from https://aws.amazon.com/sdkforjava) - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava) - Animal Sniffer Annotations (from https://repo1.maven.org/maven2/org/codehaus/mojo/animal-sniffer-annotations) -- Animal Sniffer Annotations (from https://repo1.maven.org/maven2/org/codehaus/mojo/animal-sniffer-annotations) - Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Java Client Runtime for ARM (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Spring Boot AutoConfigure (from https://github.com/Azure/azure-sdk-for-java) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Checker Qual (from ) -- Checker Qual (from ) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html) - Checker Qual (from https://checkerframework.org) +- ClassGraph (from https://github.com/classgraph/classgraph) - Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) - Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions) -- JOpt Simple (from http://jopt-simple.github.io/jopt-simple) +- JOpt Simple (from http://jopt-simple.github.io/jopt-simple, http://pholser.github.io/jopt-simple) - JUL to SLF4J bridge (from http://www.slf4j.org) - Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) -- Lucene Core (from ) +- Java JWT (from http://www.jwt.io, https://github.com/auth0/java-jwt) +- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -1109,37 +1088,39 @@ The following software have components provided under the terms of this license: - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java) -- Mockito (from http://www.mockito.org) -- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) +- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito) - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) - Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) -- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) - Plexus Default Interactivity Handler (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-interactivity-api) -- Project Lombok (from https://projectlombok.org) +- Project Lombok (from http://projectlombok.org, https://projectlombok.org) - SLF4J API Module (from http://www.slf4j.org) - Spongy Castle (from http://rtyley.github.io/spongycastle/) - Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos) +- Spring Security - Core (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-core, https://spring.io/projects/spring-security, https://spring.io/spring-security) - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) -- azure-documentdb (from http://azure.microsoft.com/en-us/services/documentdb/) +- azure-documentdb (from http://azure.microsoft.com/en-us/services/documentdb/, https://azure.microsoft.com/en-us/services/cosmos-db/) - documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/) -- java jwt (from https://github.com/auth0/java-jwt) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) -- mockito-core (from http://mockito.org) -- mockito-core (from http://mockito.org) -- mockito-core (from http://mockito.org) +- mockito-junit-jupiter (from https://github.com/mockito/mockito) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java) -- spring-security-core (from http://spring.io/spring-security) ======================================================================== MPL-1.1 ======================================================================== The following software have components provided under the terms of this license: -- Cobertura (from http://cobertura.sourceforge.net) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) ======================================================================== MPL-2.0 @@ -1147,16 +1128,8 @@ MPL-2.0 The following software have components provided under the terms of this license: - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) -- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp) -- RabbitMQ Java Client (from http://www.rabbitmq.com) - -======================================================================== -MS-RL -======================================================================== -The following software have components provided under the terms of this license: - -- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) +- OkHttp (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp, https://square.github.io/okhttp/) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) ======================================================================== PHP-3.01 @@ -1164,14 +1137,14 @@ PHP-3.01 The following software have components provided under the terms of this license: - Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) -- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api) +- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec) ======================================================================== Public-Domain ======================================================================== The following software have components provided under the terms of this license: -- Plexus Common Utilities (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) - Spongy Castle (from http://rtyley.github.io/spongycastle/) - jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common) - jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) @@ -1182,39 +1155,47 @@ SAX-PD ======================================================================== The following software have components provided under the terms of this license: -- Apache Ant Core (from ) +- Apache Ant Core ======================================================================== SPL-1.0 ======================================================================== The following software have components provided under the terms of this license: -- Servlet Specification 2.5 API (from https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) +- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) ======================================================================== SunPro ======================================================================== The following software have components provided under the terms of this license: -- Lucene Core (from ) +- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) ======================================================================== W3C ======================================================================== The following software have components provided under the terms of this license: -- Apache Ant Core (from ) +- Apache Ant Core - JTidy (from http://jtidy.sourceforge.net) -- jaxen (from http://jaxen.codehaus.org/) -- xercesImpl (from https://repo1.maven.org/maven2/xerces/xercesImpl) -- xml-apis (from ) +- Xerces2-j (from https://repo1.maven.org/maven2/xerces/xercesImpl, https://xerces.apache.org/xerces2-j/) +- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) +- xml-apis ======================================================================== WTFPL ======================================================================== The following software have components provided under the terms of this license: -- Reflections (from http://code.google.com/p/reflections/) +- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections) ======================================================================== public-domain @@ -1222,30 +1203,30 @@ public-domain The following software have components provided under the terms of this license: - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava) +- Apache Groovy (from http://groovy-lang.org, http://groovy.codehaus.org/, https://groovy-lang.org) - Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client) - Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from https://www.bouncycastle.org/java.html) -- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from https://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) -- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html) +- Guava: Google Core Libraries for Java (from http://code.google.com/p/guava-libraries, https://github.com/google/guava, https://repo1.maven.org/maven2/com/google/guava/guava) - HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/) - JTidy (from http://jtidy.sourceforge.net) -- Joda-Time (from http://www.joda.org/joda-time/) +- Joda-Time (from http://joda-time.sourceforge.net, http://www.joda.org/joda-time/, https://www.joda.org/joda-time/) - LatencyUtils (from http://latencyutils.github.io/LatencyUtils/) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) -- Project Lombok (from https://projectlombok.org) -- RabbitMQ Java Client (from http://www.rabbitmq.com) -- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) +- PostgreSQL JDBC Driver +- Project Lombok (from http://projectlombok.org, https://projectlombok.org) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) -- Spring Web (from https://github.com/spring-projects/spring-framework) -- azure-documentdb (from http://azure.microsoft.com/en-us/services/documentdb/) +- Spring Web (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-web) +- azure-documentdb (from http://azure.microsoft.com/en-us/services/documentdb/, https://azure.microsoft.com/en-us/services/cosmos-db/) +- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common) +- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - reactive-streams (from http://www.reactive-streams.org/) -- xml-apis (from ) +- xml-apis ======================================================================== unknown @@ -1253,19 +1234,22 @@ unknown The following software have components provided under the terms of this license: - Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from https://www.bouncycastle.org/java.html) -- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from https://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Byte Buddy (without dependencies) (from ) -- Checker Qual (from ) -- Checker Qual (from ) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html) +- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy) +- Checker Qual (from https://checkerframework.org) - JSON in Java (from https://github.com/douglascrockford/JSON-java) - JTidy (from http://jtidy.sourceforge.net) +- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) +- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) - Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) -- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api) +- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec) - Spongy Castle (from http://rtyley.github.io/spongycastle/) -- System Rules (from http://stefanbirkner.github.io/system-rules/) -- jaxen (from http://jaxen.codehaus.org/) +- System Rules (from http://stefanbirkner.github.com/system-rules/) +- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -1274,8 +1258,7 @@ The following software have components provided under the terms of this license: - jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) - jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core) -- jts-io-common (from ) -- xml-apis (from ) - - +- jts-io-common (from https://repo1.maven.org/maven2/org/locationtech/jts/io/jts-io-common) +- xml-apis diff --git a/README.md b/README.md index 8bfcb90b5e426329e082c09d3ef9aeb0ad8b23dc..cf670738a775199c38be2b6ecc223b1880948a40 100644 --- a/README.md +++ b/README.md @@ -9,4 +9,8 @@ running the `os-indexer` Azure implementation ## GCP Implementation -All documentation for the GCP implementation of `os-indexer` lives [here](./provider/indexer-gcp/README.md) \ No newline at end of file +All documentation for the GCP implementation of `os-indexer` lives [here](./provider/indexer-gcp/README.md) + +## AWS Implementation + +All documentation for the AWS implementation of `os-indexer` lives [here](./provider/indexer-aws/README.md) \ No newline at end of file diff --git a/devops/azure/chart/templates/virtual-service.yaml b/devops/azure/chart/templates/virtual-service.yaml index 8d3a2cc1b23edee0a353b22fa8e6d8d3f09e53bb..93a8d8af9b645f3db447b8c15cab1a7e149e3f22 100644 --- a/devops/azure/chart/templates/virtual-service.yaml +++ b/devops/azure/chart/templates/virtual-service.yaml @@ -1,5 +1,6 @@ --- # Source: /devops/azure/chart/templates/virtual-service.yaml +{{- if .Values.istioDnsHost }} apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: @@ -34,4 +35,5 @@ spec: - PATCH - DELETE allowOrigins: - - prefix: "*" \ No newline at end of file + - prefix: "*" +{{- end }} \ No newline at end of file diff --git a/devops/gcp/configmap/templates/indexer-variables.yml b/devops/gcp/configmap/templates/indexer-variables.yml index 3db35220adcd84c70c91f585bce3cd4f33963833..d9b96cdcea4089f64637a1634e3fab3730348adc 100644 --- a/devops/gcp/configmap/templates/indexer-variables.yml +++ b/devops/gcp/configmap/templates/indexer-variables.yml @@ -6,23 +6,18 @@ metadata: name: "{{ .Values.conf.configmap }}" namespace: "{{ .Release.Namespace }}" data: - KEY_RING: "{{ .Values.data.key_ring }}" - KMS_KEY: "{{ .Values.data.kms_key }}" LOG_LEVEL: "{{ .Values.data.log_level }}" SCHEMA_HOST: "{{ .Values.data.schema_host }}" AUTHORIZE_API: "{{ .Values.data.authorize_api }}" - GOOGLE_CLOUD_PROJECT: "{{ .Values.data.google_cloud_project }}" REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}" REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}" SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}" STORAGE_QUERY_RECORD_HOST: "{{ .Values.data.storage_query_record_host }}" STORAGE_SCHEMA_HOST: "{{ .Values.data.storage_schema_host }}" STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "{{ .Values.data.storage_query_record_for_conversion_host }}" - STORAGE_HOSTNAME: "{{ .Values.data.storage_hostname }}" STORAGE_RECORDS_BATCH_SIZE: "{{ .Values.data.storage_records_batch_size }}" STORAGE_QUERY_KINDS_HOST: "{{ .Values.data.storage_query_kinds_host }}" INDEXER_QUEUE_HOST: "{{ .Values.data.indexer_queue_host }}" - LEGALTAG_API: "{{ .Values.data.legaltag_api }}" - CRS_API: "{{ .Values.data.crs_api }}" PARTITION_API: "{{ .Values.data.partition_api }}" GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}" + SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}" diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml index a423c75f0818b2e638467c912ba3f83a4c509d37..21247d9775534523bd1a1390374a9eb7d2cf1a0f 100644 --- a/devops/gcp/configmap/values.yaml +++ b/devops/gcp/configmap/values.yaml @@ -3,26 +3,21 @@ # Declare variables to be passed into your templates. data: - key_ring: "csqp" - kms_key: "searchService" log_level: "INFO" schema_host: "" authorize_api: "http://entitlements/api/entitlements/v2/" - google_cloud_project: "" redis_search_host: "" redis_group_host: "" security_https_certificate_trust: "true" storage_query_record_host: "http://storage/api/storage/v2/query/records" storage_schema_host: "http://storage/api/storage/v2/schemas" storage_query_record_for_conversion_host: "http://storage/api/storage/v2/query/records:batch" - storage_hostname: "storage" storage_records_batch_size: "20" storage_query_kinds_host: "http://storage/api/storage/v2/query/kinds" indexer_queue_host: "" - legaltag_api: "http://legal/api/legal/v1" - crs_api: "" partition_api: "http://partition/api/partition/v1/" google_audiences: "" + spring_profiles_active: "gcp" conf: diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml index 886588bcd32f711c53c1c27fbe4ff75bc303bd01..cb2dc074295ad64cccccaf0fa60382016736fb9b 100644 --- a/devops/gcp/deploy/values.yaml +++ b/devops/gcp/deploy/values.yaml @@ -3,8 +3,8 @@ # Declare variables to be passed into your templates. data: - requests_cpu: "0.25" - requests_memory: "256M" + requests_cpu: "0.1" + requests_memory: "640M" limits_cpu: "1" limits_memory: "1G" serviceAccountName: "" diff --git a/docs/tutorial/IndexerService.md b/docs/tutorial/IndexerService.md index 350af65429f6e8a8c82f1888e1e37c13e2c2e227..5208cc9b30936fdc9c2ebc75fab57e2f47f9fb82 100644 --- a/docs/tutorial/IndexerService.md +++ b/docs/tutorial/IndexerService.md @@ -1,130 +1,107 @@ ## Indexer service ### Table of contents <a name="TOC"></a> + - [Indexer service](#indexer-service) - [Introduction](#introduction) - [Indexer API access](#indexer-api-access) -- [Get indexing status <a name="get-indexing-status"></a>](#get-indexing-status) -- [Reindex <a name="reindex"></a>](#reindex) -- [Schema Service adoption <a name="schema-service-adoption"></a>](#schema-service-adoption) - - [R3 Schema Support <a name="r3-schema-support"></a>](#r3-schema-support) - [Version info endpoint](#version-info-endpoint) - +- [Reindex](#reindex) +- [Data Partition provision](#data-partition-provision) +- [Schema Service adoption](#schema-service-adoption) + * [R3 Schema Support](#r3-schema-support) +- [Troubleshoot Indexing Issues](#troubleshoot-indexing-issues) + * [Get indexing status](#get-indexing-status) ## Introduction <a name="introduction"></a> -The Indexer API provides a mechanism for indexing documents that contain structured or unstructured data. Documents and indices are saved in a separate persistent store optimized for search operations. The indexer API can index any number of documents. -The indexer is indexes attributes defined in the schema. Schema can be created at the time of record ingestion in Data Ecosystem via Storage Service. The Indexer service also adds number of Data Ecosystem meta attributes such as id, kind, parent, acl, namespace, type, version, legaltags, index to each record at the time of indexing. +The Indexer API provides a mechanism for indexing documents that contain structured or unstructured data. Documents and +indices are saved in a separate persistent store optimized for search operations. The indexer API can index any number +of documents. + +The indexer is indexes attributes defined in the schema. Schema can be created at the time of record ingestion in OSDU Data Platform +via Schema Service. The Indexer service also adds number of OSDU Data Platform meta attributes such as id, kind, +parent, acl, namespace, type, version, legaltags, index to each record at the time of indexing. ## Indexer API access <a name="indexer-api-access"></a> * Required roles - Indexer service requires that users (and service accounts) have dedicated roles in order to use it. The following roles should be created and assigned using the entitlements service: - - __users.OSDU.viewers__ + Indexer service requires that users (and service accounts) have dedicated roles in order to use it. Users must be a member of `users.datalake.viewers` or `users.datalake.editors` or `users.datalake.admins`, roles can be assigned using the [Entitlements Service](/solutions/osdu/tutorials/core-services/entitlementsservice). Please look at the API documentation for specific requirements. - __users.OSDU.editors__ - - __users.OSDU.admin__ - -* Required headers + In addition to service roles, users __must__ be a member of data groups to access the data. - The Data Ecosystem stores data in different data partitions depending on the different accounts in the OSDU system. +* Required headers - A user may belong to many partitions in OSDU e.g. a OSDU user may belong to both the OSDU partition and a customer's partition. When a user logs into the OSDU portal they choose which data partition they currently want to be active. + The OSDU Data Platform stores data in different partitions, depending on the different accounts in the OSDU system. - When using the Indexer APIs you need to specify which data partition they currently have active and send it in the OSDU-Data-Partition-Id header. e.g. + A user may belong to more than one account. As a user, after logging into the OSDU portal, you need to select the account you wish to be active. + Likewise, when using the Search APIs, you need to specify the active account in the header called `data-partition-id`. The correct `data-partition-id` can be obtained from the CFS services. The `data-partition-id` enables the search within the mapped partition. e.g. ``` - OSDU-Data-Partition-Id: OSDU + data-partition-id: opendes ``` - The correct values can be obtained from CFS services. - We use this value to work out which partition to use. There is also a special data partition known as common - ``` - OSDU-Data-Partition-Id: common - ``` - This has all public data in the Data Ecosystem. Users always have access to this as well as their current active data partition. +* Optional headers - You should also send a correlation id as a header so that a single request can be tracked throughout all the services it passes through. This can be a GUID on the header with a key + The `correlation-id` is a traceable ID to track the journey of a single request. The `correlation-id` can be a GUID on the header with a key. It is best practice to provide the correlation-id so the request can be tracked through all the services. ``` - OSDU-Correlation-Id: 1e0fef08-22fd-49b1-a5cc-dffa21bc0b70 + correlation-id: 1e0fef08-22fd-49b1-a5cc-dffa21bc0b70 ``` - If you are the service initiating the request you should generate the id, otherwise you should just forward it on in the request. - +If the service is initiating the request, an ID should be generated. If the `correlation-id` is not provided, then a new ID will be generated by the service so that the request would be traceable. + [Back to table of contents](#TOC) -## Get indexing status <a name="get-indexing-status"></a> +## Version info endpoint -Indexer service adds internal metadata to each record which registers the status of the indexing. The meta data includes the status and the last indexing date and time. This additional meta block helps to see the details of indexing. The format of the index meta block is as follows: +For deployment available public `/info` endpoint, which provides build and git related information. + +#### Example response: ```json { - "index": { - "trace": [ - String, - String - ], - "statusCode": Integer, - "lastUpdateTime": Datetime - } + "groupId": "org.opengroup.osdu", + "artifactId": "indexer-gcp", + "version": "0.10.0-SNAPSHOT", + "buildTime": "2021-07-09T14:29:51.584Z", + "branch": "feature/GONRG-2681_Build_info", + "commitId": "7777", + "commitMessage": "Added copyright to version info properties file", + "connectedOuterServices": [ + { + "name": "elasticSearch", + "version": "..." + }, + { + "name": "redis", + "version": "..." + } + ] } ``` -Example: -```json -{ - "results": [ - { - "index": { - "trace": [ - "datetime parsing error: unknown format for attribute: endDate | value: 9000-01-01T00:00:00.0000000", - "datetime parsing error: unknown format for attribute: startDate | value: 1990-01-01T00:00:00.0000000" - ], - "statusCode": 400, - "lastUpdateTime": "2018-11-16T01:44:08.687Z" - } - } - ], - "totalCount": 31895 -} -``` - -Details of the index block: -1) trace: This field collects all the issues related to the indexing and concatinates using '|'. This is a String field. -2) statusCode: This field determines the category of the error. This is integer field. It can have the following values: - * 200 - All OK - * 404 - Schema is missing in Storage - * 400 - Some fields were not properly mapped with the schema defined -3) lastUpdateTime: This field captures the last time the record was updated by by the indexer service. This is datetime field so you can do range queries on this field. -You can query the index status using the following example query: - -```bash -curl --request POST \ - --url /api/search/v2/query \ - --header 'Authorization: Token' \ - --header 'Content-Type: application/json' \ - --header 'OSDU-Data-Partition-Id: Data partition id' \ - --data '{"kind": "*:*:*:*","query": "index.statusCode:404","returnedFields": ["index"]}' - -NOTE: By default, the API response excludes the 'index' attribute block. The user must specify 'index' as the 'returnedFields" in order to see it in the response. -``` -The above query will return all records which had problems due to fields mismatch. +This endpoint takes information from files, generated by `spring-boot-maven-plugin`, `git-commit-id-plugin` plugins. Need to specify paths for generated files to matching properties: -Please refer to the [Search service](searchservice#query) documentation for examples on different kinds of search queries. +- `version.info.buildPropertiesPath` +- `version.info.gitPropertiesPath` [Back to table of contents](#TOC) - + ## Reindex <a name="reindex"></a> -Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as multiple factors contribute to latency, such as number of records in the kind, current load at the indexer service etc. +Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an +asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or +appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making +query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as +multiple factors contribute to latency, such as number of records in the kind, current load at the indexer service etc. -__Note__: If a kind has been previously indexed with particular schema and if you wish to apply the schema changes during re-indexing, previous kind index has to be deleted via Index Delete API. In absence of this clean-up, reindex API will use the same schema and overwrite records with the same ids. +__Note__: If a kind has been previously indexed with particular schema and if you wish to apply the schema changes +during re-indexing, previous kind index has to be deleted via Index Delete API. In absence of this clean-up, reindex API +will use the same schema and overwrite records with the same ids. ``` POST /api/indexer/v2/reindex { - "kind": "common:welldb:wellbore:1.0.0" + "kind": "opendes:welldb:wellbore:1.0.0" } ``` @@ -136,32 +113,61 @@ curl --request POST \ --header 'accept: application/json' \ --header 'authorization: Bearer <JWT>' \ --header 'content-type: application/json' \ - --header 'OSDU-Data-Partition-Id: common' \ + --header 'data-partition-id: opendes' \ --data '{ - "kind": "common:welldb:wellbore:1.0.0" + "kind": "opendes:welldb:wellbore:1.0.0" }' ``` + </details> [Back to table of contents](#TOC) -##Schema Service adoption <a name="schema-service-adoption"></a> +## Data Partition provision <a name="data-partition-provision"></a> + +Configures Search backend for a data partition. -Indexer service is in adaptation process to use schemas from the Schema service instead of Storage Service. -The Indexer Service retrieves a schema from the Schema Service if the schema is not found on the Storage Service. -Change affects only Azure implementation so far. Later call to the Storage Service will be deprecated and then removed (after the end of the deprecation period). +``` +PUT /api/indexer/v2/partitions/provision +``` + +<details><summary>**Curl**</summary> + +```bash +curl --request PUT \ + --url '/api/indexer/v2/partitions/provision' \ + --header 'accept: application/json' \ + --header 'authorization: Bearer <JWT>' \ + --header 'content-type: application/json' \ + --header 'data-partition-id: opendes'' +``` +</details> + +> __NOTE__: API should be run at-least once at the data partition provisioning to configure required resources/settings. + +[Back to table of contents](#TOC) + +## Schema Service adoption <a name="schema-service-adoption"></a> + +Indexer service is in adaptation process to use schemas from the Schema service instead of Storage Service. The Indexer +Service retrieves a schema from the Schema Service if the schema is not found on the Storage Service. Change affects +only Azure implementation so far. Later call to the Storage Service will be deprecated and then removed (after the end +of the deprecation period). [Back to table of contents](#TOC) -###R3 Schema Support <a name="r3-schema-support"></a> +### R3 Schema Support <a name="r3-schema-support"></a> -Indexer service support r3 schema. These schemas are created via Schema service. +Indexer service support r3 schema. These schemas are created via Schema service. Here is an example following end-to-end workflow can be exercised (please update the schema based on your environment): -* Ingest r3 schema for `opendes:wks:master-data--Wellbore:1.0.0`. Schema service payload can be found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.schema.json). +* Ingest r3 schema for `opendes:wks:master-data--Wellbore:1.0.0`. Schema service payload can be + found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.schema.json) + . -* Ingest r3 master-data Wellbore record. Storage service payload can be found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.json) +* Ingest r3 master-data Wellbore record. Storage service payload can be + found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.json) * Records can be searched via Search service. Here is sample payload: @@ -186,40 +192,76 @@ data-partition-id: opendes } } ``` + [Back to table of contents](#TOC) -## Version info endpoint -For deployment available public `/info` endpoint, which provides build and git related information. -#### Example response: +# Troubleshoot Indexing Issues <a name="troubleshoot-indexing-issues"></a> + +## Get indexing status <a name="get-indexing-status"></a> + +Indexer service adds internal metadata to each record which registers the status of the indexing. The meta data includes +the status and the last indexing date and time. This additional meta block helps to see the details of indexing. The +format of the index meta block is as follows: + ```json { - "groupId": "org.opengroup.osdu", - "artifactId": "storage-gcp", - "version": "0.10.0-SNAPSHOT", - "buildTime": "2021-07-09T14:29:51.584Z", - "branch": "feature/GONRG-2681_Build_info", - "commitId": "7777", - "commitMessage": "Added copyright to version info properties file", - "connectedOuterServices": [ - { - "name": "elasticSearch", - "version":"..." - }, - { - "name": "postgresSql", - "version":"..." - }, - { - "name": "redis", - "version":"..." - } - ] + "index": { + "trace": [ + String, + String + ], + "statusCode": Integer, + "lastUpdateTime": Datetime + } } ``` -This endpoint takes information from files, generated by `spring-boot-maven-plugin`, -`git-commit-id-plugin` plugins. Need to specify paths for generated files to matching -properties: -- `version.info.buildPropertiesPath` -- `version.info.gitPropertiesPath` -[Back to table of contents](#TOC) +Example: + +```json +{ + "results": [ + { + "index": { + "trace": [ + "datetime parsing error: unknown format for attribute: endDate | value: 9000-01-01T00:00:00.0000000", + "datetime parsing error: unknown format for attribute: startDate | value: 1990-01-01T00:00:00.0000000" + ], + "statusCode": 400, + "lastUpdateTime": "2018-11-16T01:44:08.687Z" + } + } + ], + "totalCount": 31895 +} +``` + +Details of the index block: + +1) trace: This field collects all the issues related to the indexing and concatenates using '|'. This is a String field. +2) statusCode: This field determines the category of the error. This is integer field. It can have the following values: + * 200 - All OK + * 404 - Schema is missing in Storage + * 400 - Some fields were not properly mapped with the schema defined +3) lastUpdateTime: This field captures the last time the record was updated by the indexer service. This is datetime + field so you can do range queries on this field. + +You can query the index status using the following example query: + +```bash +curl --request POST \ + --url /api/search/v2/query \ + --header 'Authorization: Token' \ + --header 'Content-Type: application/json' \ + --header 'data-partition-id: Data partition id' \ + --data '{"kind": "*:*:*:*","query": "index.statusCode:404","returnedFields": ["index"]}' + +NOTE: By default, the API response excludes the 'index' attribute block. The user must specify 'index' as the 'returnedFields" in order to see it in the response. +``` + +The above query will return all records which had problems due to fields mismatch. + +Please refer to the [Search service](https://community.opengroup.org/osdu/platform/system/search-service/-/blob/master/docs/api/search_openapi.yaml#L28) documentation for examples on different kinds of search +queries. + +[Back to table of contents](#TOC) \ No newline at end of file diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml index 72a8765968e63a730c463a5f130f9d989123faae..84e8a9e69da845786e9df5d6f2ac181c6d5975b6 100644 --- a/indexer-core/pom.xml +++ b/indexer-core/pom.xml @@ -4,21 +4,36 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <name>indexer-core</name> <description>Indexer Service Core</description> <packaging>jar</packaging> <properties> <commons-beanutils.version>1.9.4</commons-beanutils.version> - <osdu.oscorecommon.version>0.13.0-rc6</osdu.oscorecommon.version> + <osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version> + <spring-security-web.version>5.4.9</spring-security-web.version> + <gson.version>2.8.9</gson.version> + <netty.version>4.1.70.Final</netty.version> </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-bom</artifactId> + <version>${netty.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> <dependency> @@ -61,6 +76,17 @@ <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-security</artifactId> + <exclusions> + <exclusion> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-web</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-web</artifactId> + <version>${spring-security-web.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> @@ -111,7 +137,7 @@ <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> - <version>2.8.5</version> + <version>${gson.version}</version> </dependency> <dependency> <groupId>com.google.http-client</groupId> diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java new file mode 100644 index 0000000000000000000000000000000000000000..26435bfa27a8f90ff6d66d4721088b916a216f4c --- /dev/null +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java @@ -0,0 +1,52 @@ +// Copyright © Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.api; + +import org.opengroup.osdu.indexer.logging.AuditLogger; +import org.opengroup.osdu.indexer.service.IClusterConfigurationService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.annotation.RequestScope; + +import java.io.IOException; + +import static java.util.Collections.singletonList; +import static org.opengroup.osdu.core.common.model.http.DpsHeaders.DATA_PARTITION_ID; + +@RestController +@RequestMapping("/partitions") +@RequestScope +public class PartitionSetupApi { + + private static final String OPS = "users.datalake.ops"; + + @Autowired + private IClusterConfigurationService clusterConfigurationService; + @Autowired + private AuditLogger auditLogger; + + @PreAuthorize("@authorizationFilter.hasPermission('" + OPS + "')") + @PutMapping(path = "/provision", consumes = "application/json") + public ResponseEntity<?> provisionPartition(@RequestHeader(DATA_PARTITION_ID) String dataPartitionId) throws IOException { + this.clusterConfigurationService.updateClusterConfiguration(); + this.auditLogger.getConfigurePartition(singletonList(dataPartitionId)); + return new ResponseEntity<>(org.springframework.http.HttpStatus.OK); + } +} diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCache.java new file mode 100644 index 0000000000000000000000000000000000000000..0942b5ecbc5decfc6d9902f96080505b7659f662 --- /dev/null +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCache.java @@ -0,0 +1,51 @@ +// Copyright © Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +public class PartitionSafeIndexCache { + + @Autowired + private IIndexCache indexCache; + @Autowired + private IRequestInfo requestInfo; + + public void put(String s, Boolean o) { + this.indexCache.put(cacheKey(s), o); + } + + public Boolean get(String s) { + return (Boolean) this.indexCache.get(cacheKey(s)); + } + + public void delete(String s) { + this.indexCache.delete(cacheKey(s)); + } + + public void clearAll() { + this.indexCache.clearAll(); + } + + private String cacheKey(String s) { + return this.requestInfo.getPartitionId() + "-" + s; + } +} \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java index 1123e5ce21712f4ceac5a47cc78b322f4ecca5a9..cff663d6ca875c6c3bddbbd0c7ba68653ac77ac1 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java @@ -47,6 +47,7 @@ public class IndexerConfigurationProperties { private String storageQueryRecordForConversionHost; private String storageQueryRecordHost; private Integer storageRecordsBatchSize; + private Integer storageRecordsByKindBatchSize; private String storageSchemaHost; private String schemaHost; private String entitlementsHost; @@ -122,4 +123,12 @@ public class IndexerConfigurationProperties { public final Boolean isSmartSearchCcsDisabled() { return Boolean.TRUE.toString().equalsIgnoreCase(smartSearchCcsDisabled); } + + public Integer getStorageRecordsByKindBatchSize() { + if (this.storageRecordsByKindBatchSize != null) { + return this.storageRecordsByKindBatchSize; + } + // if property is not set, fall back to storageRecordsBatchSize property which is used by all CSPs to set batch size. + return this.storageRecordsBatchSize; + } } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java index 1f0970f1db6b0a629b2dbfa7a34d1b3d54f11a04..703d6747e2510d71ad0b27d1fe9843c697ea5ea8 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java @@ -57,6 +57,9 @@ public class AuditEvents { private static final String INDEX_MAPPING_UPDATE_SUCCESS = "Successfully updated index mapping"; private static final String INDEX_MAPPING_UPDATE_FAILURE = "Failed updating index mapping"; + private static final String CONFIGURE_PARTITION_ACTION_ID = "IN0012"; + private static final String CONFIGURE_PARTITION_OPERATION = "Data partition cluster configuration update"; + private final String user; public AuditEvents(String user) { @@ -209,6 +212,17 @@ public class AuditEvents { .build(); } + public AuditPayload getConfigurePartitionEvent(List<String> resources) { + return AuditPayload.builder() + .action(AuditAction.UPDATE) + .status(AuditStatus.SUCCESS) + .actionId(CONFIGURE_PARTITION_ACTION_ID) + .message(CONFIGURE_PARTITION_OPERATION) + .resources(resources) + .user(this.user) + .build(); + } + public AuditPayload getIndexMappingUpdateEvent(List<String> resources, boolean isSuccess) { if (isSuccess) { return AuditPayload.builder() diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java index 431e99afc29e4f4714ce3121057ecae2a81891ed..1a37d76881a1bbc49bb2b4c97cc037f8a8af2a32 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java @@ -100,6 +100,10 @@ public class AuditLogger { this.writeLog(this.getAuditEvents().getIndexMappingUpdateEvent(resources,false)); } + public void getConfigurePartition(List<String> resources) { + this.writeLog(this.getAuditEvents().getConfigurePartitionEvent(resources)); + } + private void writeLog(AuditPayload log) { this.logger.audit(log); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..63543864722f29546604a2dcd8df76faa78a5a4f --- /dev/null +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceImpl.java @@ -0,0 +1,50 @@ +// Copyright © Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.service; + +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.opengroup.osdu.indexer.util.ElasticClientHandler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.IOException; + +@Service +public class ClusterConfigurationServiceImpl implements IClusterConfigurationService { + + @Autowired + private ElasticClientHandler elasticClientHandler; + + @Override + public boolean updateClusterConfiguration() throws IOException { + ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); + + Settings persistentSettings = + Settings.builder() + .put("action.auto_create_index", "false") + .build(); + request.persistentSettings(persistentSettings); + request.timeout(TimeValue.timeValueMinutes(1)); + try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) { + ClusterUpdateSettingsResponse response = client.cluster().putSettings(request, RequestOptions.DEFAULT); + return response.isAcknowledged(); + } + } +} diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IClusterConfigurationService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IClusterConfigurationService.java new file mode 100644 index 0000000000000000000000000000000000000000..25255be14b13acb645e3849599fa15cb744608c3 --- /dev/null +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IClusterConfigurationService.java @@ -0,0 +1,22 @@ +// Copyright © Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.service; + +import java.io.IOException; + +public interface IClusterConfigurationService { + + boolean updateClusterConfiguration() throws IOException; +} diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java index 329ebd3245eef95724ac38a84c3cd15de49a89bc..6acd078e913b9733f3d95e335461f397a6966525 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java @@ -35,8 +35,8 @@ import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; -import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; import org.opengroup.osdu.core.common.search.Preconditions; +import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; import org.springframework.beans.factory.annotation.Autowired; @@ -55,7 +55,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa @Inject private ElasticClientHandler elasticClientHandler; @Autowired - private IIndexCache indexCache; + private PartitionSafeIndexCache indexCache; @Autowired private IMappingService mappingService; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java index 7b484eba71d74fede9c038f58d6d51f686a6c7c7..9ff875985bc789175565ffd3f882747e9ad47501 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java @@ -65,7 +65,7 @@ public class IndexerServiceImpl implements IndexerService { private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1); - private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE)); + private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE, RestStatus.NOT_FOUND)); private final Gson gson = new GsonBuilder().serializeNulls().create(); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java index 6fc15067ad54f86121a880c5e9721ffbf8caab0c..2027796610526b85510a9a9404fdccd45ca39ba8 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java @@ -40,9 +40,9 @@ import org.elasticsearch.rest.RestStatus; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.search.IndexInfo; -import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.opengroup.osdu.core.common.search.Preconditions; +import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -63,7 +63,7 @@ public class IndicesServiceImpl implements IndicesService { @Autowired private ElasticIndexNameResolver elasticIndexNameResolver; @Autowired - private IIndexCache indexCache; + private PartitionSafeIndexCache indexCache; @Autowired private JaxRsDpsLog log; @@ -261,8 +261,7 @@ public class IndicesServiceImpl implements IndicesService { Request request = new Request("GET", requestUrl); Response response = client.getLowLevelClient().performRequest(request); String str = EntityUtils.toString(response.getEntity()); - final Type typeOf = new TypeToken<List<IndexInfo>>() { - }.getType(); + final Type typeOf = new TypeToken<List<IndexInfo>>() {}.getType(); return new Gson().fromJson(str, typeOf); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java index 70fa630e6ca33621db37c7b4f1a15344925fb3e1..af15d671f8f1cbc4ad1907daf99be0cf0e9a0297 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java @@ -88,7 +88,7 @@ public class ReindexServiceImpl implements ReindexService { // don't call reindex-worker endpoint if it's the last batch // previous storage query result size will be less then requested (limit param) - if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()) { + if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsByKindBatchSize()) { String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build()); this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers); return newPayLoad; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java index 950774d23c9ebeb55bc6f04bc0779bdcff04b712..95d88bc77e1b7f4fc85607fa47e6c28bcc1f4ba3 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.client.http.HttpMethods; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -43,7 +42,6 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; import java.io.UnsupportedEncodingException; -import java.lang.reflect.Type; import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -189,7 +187,7 @@ public class StorageServiceImpl implements StorageService { public RecordQueryResponse getRecordsByKind(RecordReindexRequest reindexRequest) throws URISyntaxException { Map<String, String> queryParams = new HashMap<>(); queryParams.put(RecordMetaAttribute.KIND.getValue(), reindexRequest.getKind()); - queryParams.put("limit", configurationProperties.getStorageRecordsBatchSize().toString()); + queryParams.put("limit", configurationProperties.getStorageRecordsByKindBatchSize().toString()); if (!Strings.isNullOrEmpty(reindexRequest.getCursor())) { queryParams.put("cursor", reindexRequest.getCursor()); } diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java new file mode 100644 index 0000000000000000000000000000000000000000..03c6d718f261f8e4d7b9f7fa7120d5a11fff86be --- /dev/null +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java @@ -0,0 +1,57 @@ +// Copyright © Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.api; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.indexer.logging.AuditLogger; +import org.opengroup.osdu.indexer.service.IClusterConfigurationService; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.junit4.SpringRunner; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@RunWith(SpringRunner.class) +public class PartitionSetupApiTest { + + @Mock + private AuditLogger auditLogger; + @Mock + private IClusterConfigurationService clusterConfigurationService; + @InjectMocks + private PartitionSetupApi sut; + + @Test + public void should_return200_when_valid_kind_provided() throws IOException { + ResponseEntity<?> response = this.sut.provisionPartition("opendes"); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + } + + @Test(expected = AppException.class) + public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() throws IOException { + when(this.clusterConfigurationService.updateClusterConfiguration()).thenThrow(new AppException(500, "", "")); + + this.sut.provisionPartition("opendes"); + } +} diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCacheTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCacheTest.java new file mode 100644 index 0000000000000000000000000000000000000000..106aad0b910d85fc818b307f264a7218fb6cae63 --- /dev/null +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCacheTest.java @@ -0,0 +1,70 @@ +// Copyright © Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.cache; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.mockito.Mockito.*; + +@RunWith(SpringRunner.class) +public class PartitionSafeIndexCacheTest { + + @Mock + private IIndexCache cache; + @Mock + private IRequestInfo requestInfo; + @InjectMocks + private PartitionSafeIndexCache sut; + + @Test + public void should_addopendesNamToKey_when_addingToCache() { + when(this.requestInfo.getPartitionId()).thenReturn("opendes"); + + this.sut.put("key", true); + + verify(this.cache, times(1)).put("opendes-key", true); + } + + @Test + public void should_addopendesNamToKey_when_deletingFromCache() { + when(this.requestInfo.getPartitionId()).thenReturn("opendes"); + + this.sut.delete("key"); + + verify(this.cache, times(1)).delete("opendes-key"); + } + + @Test + public void should_addopendesNamToKey_when_retrievingfromCache() { + when(this.requestInfo.getPartitionId()).thenReturn("opendes"); + + this.sut.get("key"); + + verify(this.cache, times(1)).get("opendes-key"); + } + + @Test + public void should_callWrappedClearCache() { + this.sut.clearAll(); + + verify(this.cache, times(1)).clearAll(); + } +} diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java index 9f9b8d7123f5cd0243d9190f73df5a9b393d335b..685482e2833c2bfb02509b27c8c29b07b73691a8 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java @@ -227,4 +227,18 @@ public class AuditEventsTest { assertEquals("IN0011", payload.get("actionId")); assertEquals("testUser", payload.get("user")); } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void should_getConfigurePartitionEvent() { + AuditEvents auditEvent = new AuditEvents("testUser"); + Map<String, String> payload = (Map) auditEvent.getConfigurePartitionEvent(Lists.newArrayList("anything")) + .get("auditLog"); + assertEquals(Lists.newArrayList("anything"), payload.get("resources")); + assertEquals(AuditStatus.SUCCESS, payload.get("status")); + assertEquals("Data partition cluster configuration update", payload.get("message")); + assertEquals(AuditAction.UPDATE, payload.get("action")); + assertEquals("IN0012", payload.get("actionId")); + assertEquals("testUser", payload.get("user")); + } } diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java index d6e0cf1a06705e47e08f82918a59b2f8ba2fb6b5..63906cfdc636439514809aded529e17524b14a23 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java @@ -196,6 +196,7 @@ public class AuditLoggerTest { assertEquals("IN0011", ((Map) payload.get("auditLog")).get("actionId")); assertEquals("testUser", ((Map) payload.get("auditLog")).get("user")); } + @Test @SuppressWarnings("rawtypes") public void should_createAuditLogEvent_when_indexMappingUpdateSuccess() { @@ -206,4 +207,15 @@ public class AuditLoggerTest { assertEquals("IN0011", ((Map) payload.get("auditLog")).get("actionId")); assertEquals("testUser", ((Map) payload.get("auditLog")).get("user")); } + + @Test + @SuppressWarnings("rawtypes") + public void should_createAuditLogEvent_when_configurePartitionSuccess() { + this.sut.getConfigurePartition(Lists.newArrayList("anything")); + ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class); + verify(this.logger).audit(payloadCaptor.capture()); + AuditPayload payload = payloadCaptor.getValue(); + assertEquals("IN0012", ((Map) payload.get("auditLog")).get("actionId")); + assertEquals("testUser", ((Map) payload.get("auditLog")).get("user")); + } } \ No newline at end of file diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..08bf634fa5875d5b4ec26010ffcf296b414e0f65 --- /dev/null +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceTest.java @@ -0,0 +1,72 @@ +// Copyright © Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.service; + +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.client.ClusterClient; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.indexer.util.ElasticClientHandler; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.io.IOException; + +import static junit.framework.TestCase.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +@RunWith(SpringRunner.class) +@PrepareForTest({RestHighLevelClient.class, ClusterClient.class}) +public class ClusterConfigurationServiceTest { + + @Mock + private ElasticClientHandler elasticClientHandler; + @InjectMocks + private ClusterConfigurationServiceImpl sut; + + private RestHighLevelClient restHighLevelClient; + private ClusterClient clusterClient; + + @Before + public void setup() { + initMocks(this); + clusterClient = PowerMockito.mock(ClusterClient.class); + restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class); + } + + @Test + public void should_updateClusterConfiguration() throws IOException { + ClusterUpdateSettingsResponse clusterUpdateSettingsResponse = mock(ClusterUpdateSettingsResponse.class); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + when(clusterUpdateSettingsResponse.isAcknowledged()).thenReturn(true); + doReturn(clusterClient).when(restHighLevelClient).cluster(); + doReturn(clusterUpdateSettingsResponse).when(clusterClient).putSettings(any(ClusterUpdateSettingsRequest.class), any(RequestOptions.class)); + + boolean result = this.sut.updateClusterConfiguration(); + + assertTrue(result); + } +} diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java index b52abb7c8598bb3a7e8ed3a1c064cf8c445ee1b1..c3ff4bf968ffbf21d41dd516f340bd92cb3e6a41 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java @@ -27,7 +27,7 @@ import org.mockito.Mock; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; -import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; +import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; import org.powermock.api.mockito.PowerMockito; @@ -70,7 +70,7 @@ public class IndexerMappingServiceTest { @Mock private ElasticClientHandler elasticClientHandler; @Mock - private IIndexCache indexCache; + private PartitionSafeIndexCache indexCache; @Mock private IMappingService mappingService; @InjectMocks diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java index 3f52c4c6c77349ea4996f1e3ef7ceed8aa93f07f..17bce5783d76f793e848268d9f6a68de7f105a6a 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java @@ -16,7 +16,6 @@ package org.opengroup.osdu.indexer.service; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; - import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.entity.ContentType; @@ -34,11 +33,11 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; +import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.search.IndexInfo; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; +import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -62,7 +61,7 @@ public class IndicesServiceTest { @Mock private ElasticIndexNameResolver elasticIndexNameResolver; @Mock - private IIndexCache indicesExistCache; + private PartitionSafeIndexCache indicesExistCache; @Mock @Lazy private JaxRsDpsLog log; diff --git a/pom.xml b/pom.xml index 000613cc8352b686b89226d3a756f2033d51ef13..157543a4672741449bc5b59340a52a8423a863ed 100644 --- a/pom.xml +++ b/pom.xml @@ -2,23 +2,18 @@ <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-parent</artifactId> - <version>2.1.18.RELEASE</version> - <relativePath /> - </parent> - <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> <packaging>pom</packaging> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <description>Indexer Service</description> <properties> <java.version>1.8</java.version> + <maven.compiler.target>1.8</maven.compiler.target> + <maven.compiler.source>1.8</maven.compiler.source> <spring-cloud.version>Greenwich.SR2</spring-cloud.version> - <os-core-common.version>0.13.0-rc6</os-core-common.version> + <os-core-common.version>0.13.0</os-core-common.version> <snakeyaml.version>1.26</snakeyaml.version> <hibernate-validator.version>6.1.5.Final</hibernate-validator.version> <jackson.version>2.11.4</jackson.version> @@ -28,9 +23,9 @@ <netty.version>4.1.51.Final</netty.version> <reactor-netty.version>0.8.20.RELEASE</reactor-netty.version> <woodstox-core.version>6.2.3</woodstox-core.version> - <log4j2.version>2.17.0</log4j2.version> - <!-- <maven.compiler.target>1.8</maven.compiler.target>--> -<!-- <maven.compiler.source>1.8</maven.compiler.source>--> + <log4j2.version>2.17.1</log4j2.version> + <spring-boot.version>2.1.18.RELEASE</spring-boot.version> + <json-smart.version>2.4.7</json-smart.version> <!-- <maven.war.plugin>2.6</maven.war.plugin>--> <!-- <appengine.maven.plugin>1.0.0</appengine.maven.plugin>--> <!-- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>--> @@ -50,6 +45,13 @@ <dependencyManagement> <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>${spring-boot.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-to-slf4j</artifactId> @@ -127,6 +129,11 @@ <artifactId>woodstox-core</artifactId> <version>${woodstox-core.version}</version> </dependency> + <dependency> + <groupId>net.minidev</groupId> + <artifactId>json-smart</artifactId> + <version>${json-smart.version}</version> + </dependency> </dependencies> </dependencyManagement> diff --git a/provider/indexer-aws/README.md b/provider/indexer-aws/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3c2ad5d3ac00b155f30c9cf2fcd2bc7b77b2d9b9 --- /dev/null +++ b/provider/indexer-aws/README.md @@ -0,0 +1,202 @@ +# Indexer Service +indexer-aws is a [Spring Boot](https://spring.io/projects/spring-boot) service that provides a set of APIs to index storage records against Elasticsearch. It's not user-facing, all APIs are used internally by the platform. + +## Running Locally + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. + +### Prerequisites +Pre-requisites + +* JDK 8 (https://docs.aws.amazon.com/corretto/latest/corretto-8-ug/downloads-list.html) +* Maven 3.8.3 or later +* Lombok 1.16 or later +* OSDU Instance deployed on AWS + +### Service Configuration +In order to run the service locally or remotely, you will need to have the following environment variables defined. + +| name | example value | required | description | sensitive? | +| --- | --- | --- | --- | --- | +| `LOCAL_MODE` | `true` | yes | Set to 'true' to use env vars in place of the k8s variable resolver | no | +| `APPLICATION_PORT` | `8080` | yes | The port the service will be hosted on. | no | +| `AWS_REGION` | `us-east-1` | yes | The region where resources needed by the service are deployed | no | +| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | yes | The AWS Access Key for a user with access to Backend Resources required by the service | yes | +| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | yes | The AWS Secret Key for a user with access to Backend Resources required by the service | yes | +| `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxxx` | no | AWS Session token needed if using an SSO user session to authenticate | yes | +| `ENVIRONMENT` | `osdu-prefix` | yes | The Resource Prefix defined during deployment | no | +| `LOG_LEVEL` | `DEBUG` | yes | The Log Level severity to use (https://www.tutorialspoint.com/log4j/log4j_logging_levels.htm) | no | +| `SSM_ENABLED` | `true` | yes | Set to 'true' to use SSM to resolve config properties, otherwise use env vars | no | +| `SSL_ENABLED` | `false` | no | Set to 'false' to disable SSL for local development | no | +| `ENTITLEMENTS_BASE_URL` | `http://localhost:8081` or `https://some-hosted-url` | yes | Specify the base url for an entitlements service instance. Can be run locally or remote | no | +| `PARTITION_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no | +| `STORAGE_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no | +| `SCHEMA_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no | +| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - | +| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - | + + +### Run Locally +Check that maven is installed: + +example: +```bash +$ mvn --version +Apache Maven 3.8.3 (ff8e977a158738155dc465c6a97ffaf31982d739) +Maven home: /usr/local/Cellar/maven/3.8.3/libexec +Java version: 1.8.0_312, vendor: Amazon.com Inc., runtime: /Library/Java/JavaVirtualMachines/amazon-corretto-8.jdk/Contents/Home/jre +... +``` + +You may need to configure access to the remote maven repository that holds the OSDU dependencies. Copy one of the below files' content to your .m2 folder +* For development against the OSDU GitLab environment, leverage: `<REPO_ROOT>~/.mvn/community-maven.settings.xml` +* For development in an AWS Environment, leverage: `<REPO_ROOT>/provider/indexer-aws/maven/settings.xml` + +* Navigate to the service's root folder and run: + +```bash +mvn clean package -pl indexer-core,provider/indexer-aws +``` + +* If you wish to build the project without running tests + +```bash +mvn clean package -pl indexer-core,provider/indexer-aws -DskipTests +``` + +After configuring your environment as specified above, you can follow these steps to run the application. These steps should be invoked from the *repository root.* +<br/> +<br/> +NOTE: If not on osx/linux: Replace `*` with version numbers as defined in the provider/indexer-aws/pom.xml file + +```bash +java -jar provider/indexer-aws/target/indexer-aws-*.*.*-SNAPSHOT-spring-boot.jar +``` + +## Running Elasticsearch locally +For indexer to index anything, it needs to have access to an Elasticsearch cluster. The easiest way to do this is to spin one up locally. +You can spin one up locally using Docker or Kubernetes Helm. What's detailed below is simply downloading executable and running directly. + +Instructions copied from here for longevity: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/getting-started-install.html + +1. Download a distribution from here: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.20.tar.gz + +2. Extract the archive +Linux and macOS: `tar -xvf elasticsearch-6.8.20.tar.gz` +Windows PowerShell: `Expand-Archive elasticsearch-6.8.20-windows-x86_64.zip` + +3. Start Elasticsearch from the bin directory: +Linux and macOS: + ``` + cd elasticsearch-6.8.20/bin + ./elasticsearch + ``` + + Windows: + + ``` + cd %PROGRAMFILES%\Elastic\Elasticsearch\bin + .\elasticsearch.exe + ``` + +You should see in the logs that pop up what url and port it runs on. By default you should see http://localhost with port 9300 + +## Testing + + ### Running Integration Tests + This section describes how to run OSDU Integration tests (testing/indexer-test-aws). + + IMPORTANT: You need to setup indexer queue locally first using README in indexer-queue-aws. + + You will need to have the following environment variables defined. + export AWS_COGNITO_AUTH_FLOW=USER_PASSWORD_AUTH + export AWS_COGNITO_AUTH_PARAMS_PASSWORD=$ADMIN_PASSWORD + export AWS_COGNITO_AUTH_PARAMS_USER=$ADMIN_USER + export DEFAULT_DATA_PARTITION_ID_TENANT1=opendes + export DEFAULT_DATA_PARTITION_ID_TENANT2=common + export ENTITLEMENTS_DOMAIN=example.com + export OTHER_RELEVANT_DATA_COUNTRIES=US + export STORAGE_HOST=$STORAGE_URL + export HOST=$SCHEMA_URL + export ELASTIC_HOST=$ELASTIC_HOST + export ELASTIC_PORT=$ELASTIC_PORT + export ELASTIC_PASSWORD=$ELASTIC_PASSWORD + export ELASTIC_USER_NAME=$ELASTIC_USERNAME + + | name | example value | description | sensitive? + | --- | --- | --- | --- | + | `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes | + | `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes | + | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes | + | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no | + | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no | + | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no | + | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no | + | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition| no | + | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no | + | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no | + | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes | + | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no | + | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no | + | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no | + | `HOST` | `http://localhost:8080` | Base url for deployment | no | + | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no | + | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no | + | `ELASTIC_PASSWORD` | `xxxxxxxxxxxxxxx` | Password for user to access elasticsearch | yes | + | `ELASTIC_USER_NAME` | `xxxxxxxxxxxxxxxx` | Username for user to access elasticsearch | yes | + + + **Creating a new user to use for integration tests** + ``` + aws cognito-idp admin-create-user --user-pool-id ${AWS_COGNITO_USER_POOL_ID} --username ${AWS_COGNITO_AUTH_PARAMS_USER} --user-attributes Name=email,Value=${AWS_COGNITO_AUTH_PARAMS_USER} Name=email_verified,Value=True --message-action SUPPRESS + + aws cognito-idp initiate-auth --auth-flow ${AWS_COGNITO_AUTH_FLOW} --client-id ${AWS_COGNITO_CLIENT_ID} --auth-parameters USERNAME=${AWS_COGNITO_AUTH_PARAMS_USER},PASSWORD=${AWS_COGNITO_AUTH_PARAMS_PASSWORD} + ``` + + **Entitlements group configuration for integration accounts** + <br/> + In order to add user entitlements, run entitlements bootstrap scripts in the entitlements project + + | AWS_COGNITO_AUTH_PARAMS_USER | + | --- | + | service.indexer.admin | + | service.legal.admin | + | service.storage.admin | + + Execute following command to build code and run all the integration tests: + +### Run Tests simulating Pipeline + +* Prior to running tests, scripts must be executed locally to generate pipeline env vars + +```bash +testing/indexer-test-aws/build-aws/prepare-dist.sh + +#Set Neccessary ENV Vars here as defined in run-tests.sh + +dist/testing/integration/build-aws/run-tests.sh +``` + +### Run Tests using mvn +Set required env vars and execute the following: +``` +mvn clean package -f testing/pom.xml -pl indexer-test-core,indexer-test-aws -DskipTests +mvn test -f testing/indexer-test-aws/pom.xml +``` + + + +## License +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml index 983cb5530daf93e03e969306cf1c6e295979419d..1a8ba1dba23253591437777f227794591c9b5fb0 100644 --- a/provider/indexer-aws/pom.xml +++ b/provider/indexer-aws/pom.xml @@ -18,7 +18,7 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> @@ -26,13 +26,34 @@ <artifactId>indexer-aws</artifactId> <description>Indexer service on AWS</description> <packaging>jar</packaging> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <properties> <aws.version>1.11.1018</aws.version> <deployment.environment>dev</deployment.environment> + <netty.version>4.1.51.Final</netty.version> + <jackson.version>2.11.4</jackson.version> </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-bom</artifactId> + <version>${netty.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson</groupId> + <artifactId>jackson-bom</artifactId> + <version>${jackson.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> <!-- Internal packages --> <dependency> @@ -42,12 +63,12 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <dependency> <groupId>org.opengroup.osdu.core.aws</groupId> <artifactId>os-core-lib-aws</artifactId> - <version>0.13.0-rc3</version> + <version>0.13.0</version> </dependency> <!-- AWS managed packages --> diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md index bc6a63ec3e33284229283fc5ba59caab611971a6..9f120e4504136755a842219c45d2b80138e6d007 100644 --- a/provider/indexer-azure/README.md +++ b/provider/indexer-azure/README.md @@ -47,6 +47,8 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA | `STORAGE_SCHEMA_HOST` | `${storage_service_url}/schemas` | Endpoint of schema API | no | - | | `STORAGE_QUERY_RECORD_HOST` | `${storage_service_url}/query/records` | Endpoint of records API | no | - | | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | `${storage_service_url}/query/records:batch` | Endpoint of records batch API | no | - | +| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - | +| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | 100 | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - | | `KEYVAULT_URI` | ex `https://foo-kv.vault.azure.net/` | . | . | . | | `appinsights_key` | `********` | App Insights key | yes | output of infrastructure deployments | | `aad_client_id` | `********` | AAD client application ID | yes | output of infrastructure deployment | diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml index 663ddfb242aeb181be71350e647fa8b60a6ab505..109f4eea0d2af31a8a849da4876421483fb9bfaa 100644 --- a/provider/indexer-azure/pom.xml +++ b/provider/indexer-azure/pom.xml @@ -21,12 +21,12 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>indexer-azure</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <name>indexer-azure</name> <description>Indexer Service Azure</description> <packaging>jar</packaging> @@ -37,16 +37,16 @@ <azure.appservice.plan /> <azure.appservice.appname /> <azure.appservice.subscription /> - <log4j.version>2.17.0</log4j.version> + <log4j.version>2.17.1</log4j.version> <nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version> - <indexer-core.version>0.13.0-SNAPSHOT</indexer-core.version> + <indexer-core.version>0.14.0-SNAPSHOT</indexer-core.version> <spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version> - <osdu.corelibazure.version>0.13.0-rc6</osdu.corelibazure.version> - <osdu.oscorecommon.version>0.13.0-rc6</osdu.oscorecommon.version> + <osdu.corelibazure.version>0.14.0-rc2</osdu.corelibazure.version> + <osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version> <reactor-netty.version>0.9.12.RELEASE</reactor-netty.version> <java-jwt.version>3.8.1</java-jwt.version> <powermock.version>2.0.2</powermock.version> - <mockito.version>3.0.0</mockito.version> + <mockito.version>3.11.2</mockito.version> <cobertura-maven-plugin.version>2.7</cobertura-maven-plugin.version> <spring-security-oauth2.version>2.3.6.RELEASE</spring-security-oauth2.version> <spring-boot.version>2.1.18.RELEASE</spring-boot.version> @@ -59,18 +59,6 @@ <dependencyManagement> <dependencies> <!-- Override version defined in parent's POM --> - <dependency> - <groupId>io.projectreactor.netty</groupId> - <artifactId>reactor-netty</artifactId> - <version>${reactor-netty.version}</version> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-dependencies</artifactId> - <version>${spring-boot.version}</version> - <type>pom</type> - <scope>import</scope> - </dependency> <!-- Inherit managed dependencies from core-lib-azure --> <dependency> <groupId>org.opengroup.osdu</groupId> @@ -79,11 +67,6 @@ <type>pom</type> <scope>import</scope> </dependency> - <dependency> - <groupId>org.opengroup.osdu</groupId> - <artifactId>os-core-common</artifactId> - <version>${osdu.oscorecommon.version}</version> - </dependency> </dependencies> </dependencyManagement> @@ -249,6 +232,20 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.junit</groupId> + <artifactId>junit</artifactId> + </exclusion> + <exclusion> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.springframework.security</groupId> diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java index f36df01e3d05572696c0590442eb03bd57fba33e..6ee0289c6e1526316a71f3b94f372dcb6ed0c1bc 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java @@ -128,7 +128,7 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder { String recordChangedMessagePayload = gson.toJson(recordChangedMessages); createTask(recordChangedMessagePayload, headers); } - } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()); + } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsByKindBatchSize()); } catch (AppException e) { throw e; diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties index 6f65e89ecc3c8f14aa59007ff610e2bd5c7aa0b0..8f479afcd82f6acbe812168eae39a0980b076310 100644 --- a/provider/indexer-azure/src/main/resources/application.properties +++ b/provider/indexer-azure/src/main/resources/application.properties @@ -46,6 +46,8 @@ STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records STORAGE_QUERY_KINDS_HOST=${storage_service_url}/query/kinds STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 +STORAGE_RECORDS_BY_KIND_BATCH_SIZE=100 + INDEXER_QUEUE_HOST=http://127.0.0.1:9000 diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md index 842139cfd3e3ec77b6ecf15559f63beb6f7c290c..c4ea917bc3ffe5d2887485c75dd0bd0b3542bed3 100644 --- a/provider/indexer-gcp/README.md +++ b/provider/indexer-gcp/README.md @@ -1,42 +1,49 @@ # Indexer Service os-indexer-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU R2 domain searches against Elasticsearch. +## Table of Contents <a name="TOC"></a> +* [Getting started](#Getting-started) +* [Mappers](#Mappers) +* [Settings and Configuration](#Settings-and-Configuration) +* [Run service](#Run-service) +* [Testing](#Testing) +* [Deployment](#Deployment) +* [Entitlements groups](#Entitlements-groups) +* [Licence](#License) + ## Getting Started These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. -### Prerequisites -Pre-requisites +## Mappers -* GCloud SDK with java (latest version) -* JDK 8 -* Lombok 1.16 or later -* Maven +This is a universal solution created using EPAM OQM mappers technology. It allows you to work with various +implementations of message brokers. -### Installation -In order to run the service locally or remotely, you will need to have the following environment variables defined. +For more information about mappers: +- [OQM Readme](https://community.opengroup.org/osdu/platform/system/lib/cloud/gcp/oqm/-/blob/master/README.md) -| name | value | description | sensitive? | source | -| --- | --- | --- | --- | --- | -| `LOG_PREFIX` | `service` | Logging prefix | no | - | -| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - | -| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | -| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment | -| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment | -| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances | -| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment | -| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | -| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | -| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | -| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | -| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | -| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | -| `GOOGLE_CLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment | -| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | -| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | -| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | -| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - | -| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment | -| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment | +### Limitations of the current version + +In the current version, the mappers are equipped with several drivers to the stores and the message broker: + +- OQM (mapper to message brokers): Google PubSub; RabbitMQ + +## Settings and Configuration + +### Prerequisites +1. Mandatory + - JDK 8 + - Lombok 1.16 or later + - Maven +2. For Google Cloud only + - GCloud SDK with java (latest version) + +### Anthos Service Configuration: +[Anthos service configuration ](docs/anthos/README.md) +### GCP Service Configuration: +[Gcp service configuration ](docs/gcp/README.md) + +## Run service ### Run Locally Check that maven is installed: @@ -181,26 +188,6 @@ $ (cd testing/indexer-test-gcp/ && mvn clean test) or * Google Documentation: https://cloud.google.com/cloud-build/docs/deploying-builds/deploy-appengine -#### Cloud KMS Setup - -Enable cloud KMS on master project - -Create king ring and key in the ***master project*** - -```bash - gcloud services enable cloudkms.googleapis.com - export KEYRING_NAME="csqp" - export CRYPTOKEY_NAME="searchService" - gcloud kms keyrings create $KEYRING_NAME --location global - gcloud kms keys create $CRYPTOKEY_NAME --location global \ - --keyring $KEYRING_NAME \ - --purpose encryption -``` - -Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab - -Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab - #### Memory Store (Redis Instance) Setup Create a new Standard tier Redis instance on the ***service project*** @@ -211,6 +198,14 @@ The Redis instance must be created under the same region with the App Engine app gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD ``` +## Entitlements groups +Storage service account should have entitlements groups listed below: +- service.entitlements.user +- users +- service.storage.viewer +- service.schema-service.viewers +- data.default.viewers + ## Licence Copyright © Google LLC Copyright © EPAM Systems diff --git a/provider/indexer-gcp/docs/anthos/README.md b/provider/indexer-gcp/docs/anthos/README.md new file mode 100644 index 0000000000000000000000000000000000000000..90cb1a8b9ed4e39f7e9f80be0ad42524b77d7ace --- /dev/null +++ b/provider/indexer-gcp/docs/anthos/README.md @@ -0,0 +1,188 @@ +## Service Configuration for Anthos + +## Environment variables: + +Define the following environment variables. + +Must have: + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for GCP environment | false | - | +| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` |yes| - | +| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens| yes | - | +| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - | + +Defined in default application property file but possible to override: + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `LOG_PREFIX` | `service` | Logging prefix | no | - | +| `LOG_LEVEL` | `****` | Logging level | no | - | +| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | +| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment | +| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment | +| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment | +| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment | +| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment | + +These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested. +Usage of spring profiles is preferred. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - | +| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - | +| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - | + +## Elasticsearch configuration + +### Properties set in Partition service: + +**prefix:** `elasticsearch` + +It can be overridden by: + +- through the Spring Boot property `elastic-search-properties-prefix` +- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX` + +**Propertyset:** + +| Property | Description | +| --- | --- | +| elasticsearch.host | server URL | +| elasticsearch.port | server port | +| elasticsearch.configuration | username and password | + +<details><summary>Example of a definition for a single tenant</summary></details> + +``` + +curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{ + "properties": { + "elasticsearch.host": { + "sensitive": false, + "value": "elastic.us-central1.gcp.cloud.es.io" + }, + "elasticsearch.port": { + "sensitive": false, + "value": "9243" + }, + "elasticsearch.configuration": { + "sensitive": true, + "value": "elasticuser:elasticpassword" + } + } +}' + +``` + +### For OQM RabbitMQ + +**prefix:** `oqm.rabbitmq` +It can be overridden by: + +- through the Spring Boot property `oqm.rabbitmq.partition-properties-prefix` +- environment variable `OQM_RABBITMQ_PARTITION_PROPERTIES_PREFIX`` + +**Propertyset** (for two types of connection: messaging and admin operations): + +| Property | Description | +| --- | --- | +| oqm.rabbitmq.amqp.host | messaging hostnameorIP | +| oqm.rabbitmq.amqp.port | - port | +| oqm.rabbitmq.amqp.path | - path | +| oqm.rabbitmq.amqp.username | - username | +| oqm.rabbitmq.amqp.password | - password | +| oqm.rabbitmq.admin.schema | admin host schema | +| oqm.rabbitmq.admin.host | - host name | +| oqm.rabbitmq.admin.port | - port | +| oqm.rabbitmq.admin.path | - path | +| oqm.rabbitmq.admin.username | - username | +| oqm.rabbitmq.admin.password | - password | + +<details><summary>Example of a single tenant definition</summary> + +``` + +curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{ + "properties": { + "oqm.rabbitmq.amqp.host": { + "sensitive": false, + "value": "localhost" + }, + "oqm.rabbitmq.amqp.port": { + "sensitive": false, + "value": "5672" + }, + "oqm.rabbitmq.amqp.path": { + "sensitive": false, + "value": "" + }, + "oqm.rabbitmq.amqp.username": { + "sensitive": false, + "value": "guest" + }, + "oqm.rabbitmq.amqp.password": { + "sensitive": true, + "value": "guest" + }, + + "oqm.rabbitmq.admin.schema": { + "sensitive": false, + "value": "http" + }, + "oqm.rabbitmq.admin.host": { + "sensitive": false, + "value": "localhost" + }, + "oqm.rabbitmq.admin.port": { + "sensitive": false, + "value": "9002" + }, + "oqm.rabbitmq.admin.path": { + "sensitive": false, + "value": "/api" + }, + "oqm.rabbitmq.admin.username": { + "sensitive": false, + "value": "guest" + }, + "oqm.rabbitmq.admin.password": { + "sensitive": true, + "value": "guest" + } + } +}' + +``` + +</details> + +#### Exchanges and queues configuration + +At RabbitMq should be created exchange with name: + +**name:** `indexing-progress` + + + +## Keycloak configuration + +[Keycloak service accounts setup](https://www.keycloak.org/docs/latest/server_admin/#_service_accounts) + +Configure Clients. One Client per OSDU service. Set them “confidentialâ€. + + + +Each Client has embedded Service Account (SA) option. Enable SAs for Clients, make “Authorization enabledâ€: + + + +Add `partition-and-entitlements` scope to `Default Client Scopes` and generate Keys. + +Give `client-id` and `client-secret` to services, which should be authorized within the platform. \ No newline at end of file diff --git a/provider/indexer-gcp/docs/anthos/pics/client.png b/provider/indexer-gcp/docs/anthos/pics/client.png new file mode 100644 index 0000000000000000000000000000000000000000..8a2014e672b65ddf644558e2c9e8cb8823f549dd Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/client.png differ diff --git a/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG b/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG new file mode 100644 index 0000000000000000000000000000000000000000..e1c67734b91a536659fdf0b940a93208d749e3c8 Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG differ diff --git a/provider/indexer-gcp/docs/anthos/pics/sa.png b/provider/indexer-gcp/docs/anthos/pics/sa.png new file mode 100644 index 0000000000000000000000000000000000000000..b9c8124047b5834a2cf79baecbf751dc106feeff Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/sa.png differ diff --git a/provider/indexer-gcp/docs/gcp/README.md b/provider/indexer-gcp/docs/gcp/README.md new file mode 100644 index 0000000000000000000000000000000000000000..494169b1634af2e093c39038688987e78de6d726 --- /dev/null +++ b/provider/indexer-gcp/docs/gcp/README.md @@ -0,0 +1,94 @@ +## Service Configuration for GCP + +## Environment variables: + +Define the following environment variables. + +Must have: + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | +| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - | + +Defined in default application property file but possible to override: + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `LOG_PREFIX` | `service` | Logging prefix | no | - | +| `LOG_LEVEL` | `****` | Logging level | no | - | +| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | +| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment | +| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment | +| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment | +| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment | +| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment | +| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | + +These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested. +Usage of spring profiles is preferred. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - | +| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - | +| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - | + +## Pubsub configuration: + +At Pubsub should be created topic with name: + +**name:** `indexing-progress` + +## Elasticsearch configuration + +### Properties set in Partition service: + +**prefix:** `elasticsearch` + +It can be overridden by: + +- through the Spring Boot property `elastic-search-properties-prefix` +- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX` + +**Propertyset:** + +| Property | Description | +| --- | --- | +| elasticsearch.host | server URL | +| elasticsearch.port | server port | +| elasticsearch.configuration | username and password | + +<details><summary>Example of a definition for a single tenant</summary></details> + +``` + +curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{ + "properties": { + "elasticsearch.host": { + "sensitive": false, + "value": "elastic.us-central1.gcp.cloud.es.io" + }, + "elasticsearch.port": { + "sensitive": false, + "value": "9243" + }, + "elasticsearch.configuration": { + "sensitive": true, + "value": "elasticuser:elasticpassword" + } + } +}' + +``` + +## Google cloud service account configuration : +TBD + +| Required roles | +| --- | +| - | diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml index 394e63367d99c3bdabbc00c85674308752b134f3..3f7bb4987478522f50a8d113ae6094c2997c4583 100644 --- a/provider/indexer-gcp/pom.xml +++ b/provider/indexer-gcp/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>indexer-gcp</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <name>indexer-gcp</name> <description>Indexer Service GCP App Engine</description> <packaging>jar</packaging> @@ -19,12 +19,12 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-gcp</artifactId> - <version>0.13.0-rc4</version> + <version>0.14.0-rc1</version> </dependency> <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <!-- excluded due to runtime conflict with latest core-lib-gcp transient dependencies --> <exclusions> <exclusion> @@ -179,6 +179,4 @@ </plugin> </plugins> </build> - - </project> diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java deleted file mode 100644 index 6a3b330aad0dcc5a7ba09687a82e0c5e1f53548c..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.cache; - -import com.google.auth.oauth2.AccessToken; -import org.opengroup.osdu.core.common.cache.RedisCache; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -public class DatastoreCredentialCache extends RedisCache<String, AccessToken> { - - @Autowired - public DatastoreCredentialCache(final IndexerConfigurationProperties configurationProperties) { - super(configurationProperties.getRedisSearchHost(), Integer.parseInt(configurationProperties.getRedisSearchPort()), 58 * 60, String.class, AccessToken.class); - } -} \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java index 206787c8d06dd825dabf1b165c9c2f52f742a735..83ee124a964a148dcaba28b7b0e88d7c7a111b2f 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java @@ -18,29 +18,23 @@ package org.opengroup.osdu.indexer.cache; import com.google.gson.Gson; -import java.io.IOException; import java.util.Objects; import javax.inject.Inject; -import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.cache.RedisCache; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; -import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.stereotype.Component; @Component public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable { - private IKmsClient kmsClient; private RedisCache<String, String> cache; @Inject - public ElasticCredentialsCache(final IndexerConfigurationProperties properties, final IKmsClient kmsClient) { + public ElasticCredentialsCache(final IndexerConfigurationProperties properties) { this.cache = new RedisCache<>(properties.getRedisSearchHost(), Integer.parseInt(properties.getRedisSearchPort()), properties.getElasticCacheExpiration() * 60, String.class, String.class); - this.kmsClient = kmsClient; } @Override @@ -50,27 +44,17 @@ public class ElasticCredentialsCache implements IElasticCredentialsCache<String, @Override public void put(String s, ClusterSettings o) { - try { - String jsonSettings = new Gson().toJson(o); - String encryptString = kmsClient.encryptString(jsonSettings); - this.cache.put(s, encryptString); - } catch (IOException e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal server error", "Unable to encrypt settings before being cached", e); - } + String jsonSettings = new Gson().toJson(o); + this.cache.put(s, jsonSettings); } @Override public ClusterSettings get(String s) { - try { - String encryptedSettings = this.cache.get(s); - if (Objects.isNull(encryptedSettings) || encryptedSettings.isEmpty()) { - return null; - } - String jsonSettings = this.kmsClient.decryptString(encryptedSettings); - return new Gson().fromJson(jsonSettings, ClusterSettings.class); - } catch (IOException e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal server error", "Unable to decrypt settings from cache", e); + String jsonSettings = this.cache.get(s); + if (Objects.isNull(jsonSettings) || jsonSettings.isEmpty()) { + return null; } + return new Gson().fromJson(jsonSettings, ClusterSettings.class); } @Override diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java deleted file mode 100644 index 6f4d9b7505504f259ceca3d3b8e1834dc53f91ba..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2020 Google LLC - Copyright 2020 EPAM Systems, Inc - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package org.opengroup.osdu.indexer.di; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.core.common.cache.VmCache; -import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential; -import org.springframework.beans.factory.config.AbstractFactoryBean; -import org.springframework.stereotype.Component; - -@Component -public class DatastoreCredentialsCacheFactory extends - AbstractFactoryBean<ICache<String, DatastoreCredential>> { - - @Override - public Class<?> getObjectType() { - return ICache.class; - } - - @Override - protected ICache<String, DatastoreCredential> createInstance() throws Exception { - return new VmCache<>(5 * 60, 20); - } -} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..72cb1f6a3a1cedfc3f408853d99d10a9c80cca74 --- /dev/null +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020-2022 Google LLC + * Copyright 2020-2022 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + +import org.opengroup.osdu.core.common.partition.IPartitionProvider; +import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; +import org.opengroup.osdu.core.destination.elastic.ElasticSearchDestinationResolver; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ElasticSearchConfig { + + @Bean + public IElasticRepository elasticRepository(ElasticSearchConfigurationProperties properties, IPartitionProvider partitionProvider) { + return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider); + } +} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..586ce1ab9e9127282066a2e0fddf651836d12f18 --- /dev/null +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020-2022 Google LLC + * Copyright 2020-2022 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class ElasticSearchConfigurationProperties { + + private String elasticsearchPropertiesPrefix = "elasticsearch"; +} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java deleted file mode 100644 index c0995677c674dc1192b4c9533903454af441cfeb..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2020 Google LLC - Copyright 2020 EPAM Systems, Inc - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package org.opengroup.osdu.indexer.di; - -import org.opengroup.osdu.core.gcp.PubSub.PubSubExtensions; -import org.springframework.beans.factory.config.AbstractFactoryBean; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -@Component -@RequestScope -public class PubSubExtensionsFactory extends AbstractFactoryBean<PubSubExtensions> { - - - @Override - public Class<?> getObjectType() { - return PubSubExtensions.class; - } - - @Override - protected PubSubExtensions createInstance() throws Exception { - return new PubSubExtensions(); - } -} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java deleted file mode 100644 index fa51ef212c0427bd21e4a522a99891d38163ccc4..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.di; - -import lombok.extern.java.Log; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory; -import org.springframework.beans.factory.config.AbstractFactoryBean; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -@Log -@Component -@RequestScope -public class TenantFactoryService extends AbstractFactoryBean<ITenantFactory> { - - @Override - protected ITenantFactory createInstance() throws Exception { - return new TenantFactory(); - } - - @Override - public Class<?> getObjectType() { - return ITenantFactory.class; - } -} \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java deleted file mode 100644 index 9a6dc9ef7e3920668aa2155c1c39dc8531ed3529..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2021 Google LLC - * Copyright 2021 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.persistence; - -import com.google.api.gax.retrying.RetrySettings; -import com.google.cloud.TransportOptions; -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.http.HttpTransportOptions; -import java.util.HashMap; -import java.util.Map; -import javax.inject.Inject; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential; -import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; -import org.springframework.stereotype.Component; -import org.threeten.bp.Duration; - -@Component("indexerDatastoreFactory") -public class DatastoreFactory { - - @Inject - private DatastoreCredentialCache cache; - - private static Map<String, Datastore> datastoreClients = new HashMap<>(); - - private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() - .setMaxAttempts(6) - .setInitialRetryDelay(Duration.ofSeconds(10)) - .setMaxRetryDelay(Duration.ofSeconds(32)) - .setRetryDelayMultiplier(2.0) - .setTotalTimeout(Duration.ofSeconds(50)) - .setInitialRpcTimeout(Duration.ofSeconds(50)) - .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofSeconds(50)) - .build(); - - private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder() - .setReadTimeout(30000) - .build(); - - public Datastore getDatastoreInstance(TenantInfo tenantInfo) { - if (datastoreClients.get(tenantInfo.getName()) == null) { - Datastore googleDatastore = DatastoreOptions.newBuilder() - .setCredentials(new DatastoreCredential(tenantInfo)) - .setRetrySettings(RETRY_SETTINGS) - .setTransportOptions(TRANSPORT_OPTIONS) - .setNamespace(tenantInfo.getName()) - .setProjectId(tenantInfo.getProjectId()) - .build().getService(); - datastoreClients.put(tenantInfo.getName(), googleDatastore); - } - return datastoreClients.get(tenantInfo.getName()); - } -} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java deleted file mode 100644 index a8e3373f44926527a271d4d4390b5f4d5e647177..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.persistence; - -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.Entity; -import com.google.cloud.datastore.Key; -import lombok.extern.java.Log; -import org.apache.http.HttpStatus; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.search.ClusterSettings; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; -import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; -import org.opengroup.osdu.core.common.search.Preconditions; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; - -@Log -@Component -public class ElasticRepositoryDatastore implements IElasticRepository { - - static final String HOST = "host"; - static final String PORT = "port"; - static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration"; - - @Inject - private IKmsClient kmsClient; - @Inject - private DatastoreFactory datastoreFactory; - - @Inject - private IndexerConfigurationProperties properties; - - - @Override - public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) { - - Datastore googleDatastore = this.datastoreFactory.getDatastoreInstance(tenantInfo); - Key key = googleDatastore.newKeyFactory().setKind(properties.getElasticDatastoreKind()).newKey(properties.getElasticDatastoreId()); - Entity datastoreEntity = googleDatastore.get(key); - - if (datastoreEntity == null) { - throw new AppException(HttpStatus.SC_NOT_FOUND, "Cluster setting not found", "The requested cluster setting was not found in datastore.", String.format("Cluster setting with key: '%s' does not exist in datastore.", key.getName())); - } - - String encryptedHost = null; - String encryptedPort = null; - String encryptedConfiguration = null; - - try { - encryptedHost = datastoreEntity.getString(HOST); - encryptedPort = datastoreEntity.getString(PORT); - encryptedConfiguration = datastoreEntity.getString(XPACK_RESTCLIENT_CONFIGURATION); - - String host = this.kmsClient.decryptString(encryptedHost); - String portString = this.kmsClient.decryptString(encryptedPort); - String usernameAndPassword = this.kmsClient.decryptString(encryptedConfiguration); - - Preconditions.checkNotNullOrEmpty(host, "host cannot be null"); - Preconditions.checkNotNullOrEmpty(portString, "port cannot be null"); - Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null"); - - int port = Integer.parseInt(portString); - - return new ClusterSettings(host, port, usernameAndPassword); - } catch (GoogleJsonResponseException e) { - String debuggingInfo = String.format("Host: %s | port: %s | configuration: %s", encryptedHost, encryptedPort, encryptedConfiguration); - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting decryption error", "An error has occurred decrypting cluster settings.", debuggingInfo, e); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", "An error has occurred fetching cluster settings from the datastore.", e); - } - } -} \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java index 31d242cd46515f9b37907e458b510dd7a37557bd..2a35a96f5684f9e0c1227d549c0419da067c8118 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java @@ -14,129 +14,58 @@ package org.opengroup.osdu.indexer.publish; -import com.google.api.gax.retrying.RetrySettings; -import com.google.cloud.pubsub.v1.Publisher; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; -import com.google.gson.reflect.TypeToken; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.PubsubMessage; +import java.util.HashMap; +import java.util.Map; +import lombok.RequiredArgsConstructor; import lombok.extern.java.Log; -import org.apache.http.HttpStatus; -import org.elasticsearch.common.Strings; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.indexer.JobStatus; -import org.opengroup.osdu.core.common.model.indexer.RecordStatus; -import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.core.gcp.PubSub.PubSubExtensions; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver; +import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination; +import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage; +import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic; import org.opengroup.osdu.indexer.provider.interfaces.IPublisher; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import org.threeten.bp.Duration; - -import javax.inject.Inject; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.List; -import java.util.Map; @Log @Component @RequestScope +@RequiredArgsConstructor public class PublisherImpl implements IPublisher { - private static final Map<String, Publisher> PUBSUB_CLIENTS = new HashMap<>(); - private static final String TOPIC_ID = "indexing-progress"; + private final OqmDriver driver; - @Inject - private ITenantFactory tenantStorageFactory; - - @Inject - private PubSubExtensions pubSubExtensions; - - @Inject - private IndexerConfigurationProperties properties; + private final OqmTopic oqmTopic = OqmTopic.builder().name(TOPIC_ID).build(); @Override - public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception { - - // Don't publish to pubsub when testing locally - if (properties.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) { - return; - } - - String tenant = headers.getPartitionId(); - if(Strings.isNullOrEmpty(tenant)) - tenant = headers.getAccountId(); - - Publisher publisher = this.getPublisher(tenant); - if (publisher == null) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", "A fatal internal error occurred creating publisher client."); - } - PubsubMessage pubsubMessage = getPubsubMessage(headers, indexerBatchStatus); - - pubSubExtensions.publishAndCreateTopicIfNotExist(publisher, pubsubMessage); - - } - - private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() - .setTotalTimeout(Duration.ofSeconds(30)) - .setInitialRetryDelay(Duration.ofSeconds(2)) - .setRetryDelayMultiplier(2) - .setMaxRetryDelay(Duration.ofSeconds(5)) - .setInitialRpcTimeout(Duration.ofSeconds(10)) - .setRpcTimeoutMultiplier(2) - .setMaxRpcTimeout(Duration.ofSeconds(10)) + public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) { + OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId()) .build(); + String json = generatePubSubMessage(indexerBatchStatus); - - private PubsubMessage getPubsubMessage(DpsHeaders headers, JobStatus indexerBatchStatus) { - - Gson gson = new GsonBuilder().create(); - Type listType = new TypeToken<List<RecordStatus>>() {}.getType(); - JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType); - ByteString statusChangedTagsData = ByteString.copyFromUtf8(statusChangedTagsJson.toString()); - - PubsubMessage.Builder builder = PubsubMessage.newBuilder(); - String tenant = headers.getPartitionId(); - //This code it to provide backward compatibility to slb-account-id - if(!Strings.isNullOrEmpty(tenant)) { - builder.putAttributes(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionId()); - } else { - builder.putAttributes(DpsHeaders.ACCOUNT_ID, headers.getAccountId()); - } - - builder.putAttributes(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); - // TODO: uncomment when deploying to production -// builder.putAttributes( AppEngineHeaders.CLOUD_TRACE_CONTEXT, headers.getHeaders().get(AppEngineHeaders.CLOUD_TRACE_CONTEXT)); - builder.setData(statusChangedTagsData); - - return builder.build(); + Map<String, String> attributes = getAttributes(headers); + OqmMessage oqmMessage = OqmMessage.builder().data(json).attributes(attributes).build(); + driver.publish(oqmMessage, oqmTopic, oqmDestination); } - private Publisher getPublisher(String tenantName) throws IOException { - TenantInfo info = this.tenantStorageFactory.getTenantInfo(tenantName); - if (info == null) { - return null; - } else { - if (PUBSUB_CLIENTS.containsKey(tenantName)) return PUBSUB_CLIENTS.get(tenantName); - - ProjectTopicName topicName = ProjectTopicName.newBuilder().setProject(info.getProjectId()).setTopic(TOPIC_ID).build(); - Publisher publisher = Publisher.newBuilder(topicName).setRetrySettings(RETRY_SETTINGS).build(); - - if (publisher == null) return null; + private Map<String, String> getAttributes(DpsHeaders headers) { + Map<String, String> attributes = new HashMap<>(); + attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); + attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + headers.addCorrelationIdIfMissing(); + attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + return attributes; + } - PUBSUB_CLIENTS.put(tenantName, publisher); - return publisher; - } + private String generatePubSubMessage(JobStatus jobStatus) { + Gson gson = new GsonBuilder().create(); + JsonElement statusChangedTagsJson = gson.toJsonTree(jobStatus, JobStatus.class); + return statusChangedTagsJson.toString(); } } \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java similarity index 95% rename from provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java rename to provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java index 0f70a40bc444ae9fd7e1553c319684df1286273b..47b42552ccdef36a19103027cdde783bd4d50359 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java @@ -27,7 +27,7 @@ import org.springframework.security.config.annotation.web.configuration.WebSecur @Configuration @EnableWebSecurity @EnableGlobalMethodSecurity(prePostEnabled = true) -public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter { +public class GcpSecurityConfig extends WebSecurityConfigurerAdapter { @Override protected void configure(HttpSecurity http) throws Exception { diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java index 9950bb31d3eb14da96c466aeb283eea7b2f2110b..360f576f68021a2fdb398db4d5c368284069bc2e 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java @@ -25,6 +25,7 @@ import java.util.logging.Level; import javax.inject.Inject; import lombok.extern.java.Log; import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.auth.TokenProvider; import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.model.entitlements.AuthorizationResponse; import org.opengroup.osdu.core.common.model.http.AppException; @@ -34,7 +35,6 @@ import org.opengroup.osdu.core.common.model.search.SearchServiceRole; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.stereotype.Component; @@ -50,7 +50,7 @@ public class RequestInfoImpl implements IRequestInfo { private DpsHeaders dpsHeaders; @Inject - private IServiceAccountJwtClient serviceAccountJwtClient; + private TokenProvider tokenProvider; @Inject private TenantInfo tenantInfo; @@ -99,12 +99,12 @@ public class RequestInfoImpl implements IRequestInfo { @Override public boolean isTaskQueueRequest() { - if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){ - log.log(Level.INFO,"Request confirmed as cloud task, token validation in progress"); + if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)) { + log.log(Level.INFO, "Request confirmed as cloud task, token validation in progress"); return isCloudTaskRequest(); } - if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){ - log.log(Level.INFO,"Request confirmed as AppEngine, headers validation in progress"); + if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) { + log.log(Level.INFO, "Request confirmed as AppEngine, headers validation in progress"); return isAppEngineTaskRequest(); } return false; @@ -116,7 +116,7 @@ public class RequestInfoImpl implements IRequestInfo { return true; } - private boolean isAppEngineTaskRequest(){ + private boolean isAppEngineTaskRequest() { if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) { return false; } @@ -136,7 +136,7 @@ public class RequestInfoImpl implements IRequestInfo { } return authHeader; } else { - return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName()); + return "Bearer " + this.tokenProvider.getIdToken(); } } } diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java deleted file mode 100644 index 6c57527219c328d75ab5acaffbee7d85f7c558e8..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.util; - -import com.auth0.jwt.JWT; -import com.google.cloud.iam.credentials.v1.GenerateIdTokenResponse; -import com.google.cloud.iam.credentials.v1.IamCredentialsClient; -import com.google.cloud.iam.credentials.v1.ServiceAccountName; -import java.util.Collections; -import javax.inject.Inject; -import org.apache.http.HttpStatus; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.model.search.IdToken; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; -import org.opengroup.osdu.core.gcp.multitenancy.credentials.IamCredentialsProvider; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -@Primary -@Component -@RequestScope -public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient { - - private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/-/serviceAccounts/%s"; - - private final IamCredentialsProvider iamCredentialsProvider = new IamCredentialsProvider(); - - @Value("${GOOGLE_AUDIENCES}") - private String audiences; - - @Inject - private ITenantFactory tenantInfoServiceProvider; - @Inject - private IJwtCache cacheService; - @Inject - private JaxRsDpsLog log; - @Inject - private DpsHeaders dpsHeaders; - @Inject - private IndexerConfigurationProperties properties; - - public String getIdToken(String tenantName) { - this.log.info("Tenant name received for auth token is: " + tenantName); - TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName); - if (tenant == null) { - this.log.error("Invalid tenant name receiving from pubsub"); - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub"); - } - try { - - IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); - // Add the user to DpsHeaders directly - this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); - - if (!IdToken.refreshToken(cachedToken)) { - return cachedToken.getTokenValue(); - } - - try (IamCredentialsClient iamCredentialsClient = iamCredentialsProvider.getIamCredentialsClient()) { - ServiceAccountName serviceAccountName = ServiceAccountName.parse(String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getServiceAccount())); - GenerateIdTokenResponse idTokenResponse = iamCredentialsClient.generateIdToken(serviceAccountName, Collections.emptyList(), audiences, true); - String token = idTokenResponse.getToken(); - IdToken idToken = IdToken.builder().tokenValue(token).expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build(); - this.cacheService.put(tenant.getServiceAccount(), idToken); - return token; - } - } catch (AppException e) { - throw e; - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", e); - } - } -} diff --git a/provider/indexer-gcp/src/main/resources/application-anthos.properties b/provider/indexer-gcp/src/main/resources/application-anthos.properties new file mode 100644 index 0000000000000000000000000000000000000000..9e553661404fae2ae946a221d506b5b56d16f8a0 --- /dev/null +++ b/provider/indexer-gcp/src/main/resources/application-anthos.properties @@ -0,0 +1,6 @@ +oqmDriver=rabbitmq +service.token.provider=OPENID +partition-auth-enabled=false +openid.provider-url= +openid.provider-client-id= +openid.provider-client-secret= \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-dev.properties b/provider/indexer-gcp/src/main/resources/application-dev.properties deleted file mode 100644 index 6c49ae23e07fddb7668a52d645d70268579ce9a9..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/resources/application-dev.properties +++ /dev/null @@ -1,21 +0,0 @@ -google-cloud-project=opendes - -indexer-host=indexer-jvmvia5dea-uc.a.run.app -STORAGE_HOSTNAME=storage-jvmvia5dea-uc.a.run.app - -storage-schema-host=http://127.0.0.1:8081/api/storage/v2/schemas -storage-query-record-host=http://127.0.0.1:8081/api/storage/v2/query/records -storage-query-record-for-conversion-host=https://storage-jvmvia5dea-uc.a.run.app/api/storage/v2/query/records:batch -storage-records-batch-size=20 - -indexer-queue-host=https://indexer-queue-jvmvia5dea-uc.a.run.app/_dps/task-handlers/enqueue - -AUTHORIZE_API=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1 -LEGALTAG_API=https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1 -CRS_API=https://crs-converter-jvmvia5dea-uc.a.run.app/api/crs/v1 - -## use below values for gcp: opendes -REDIS_GROUP_HOST=127.0.0.1 -redis-search-host=127.0.0.1 - -google-audiences=519000754840-09v7ssbpku7sevhvbtq3bdoi70es16p5.apps.googleusercontent.com \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-gcp.properties b/provider/indexer-gcp/src/main/resources/application-gcp.properties new file mode 100644 index 0000000000000000000000000000000000000000..fd738d7f484bf44fc3b2f2043191259dd0b3a1b8 --- /dev/null +++ b/provider/indexer-gcp/src/main/resources/application-gcp.properties @@ -0,0 +1,3 @@ +oqmDriver=pubsub +service.token.provider=GCP +partition-auth-enabled=true \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-kuber.properties b/provider/indexer-gcp/src/main/resources/application-kuber.properties deleted file mode 100644 index 22d3ba0253112794113a96f90fa094f5468902e1..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/resources/application-kuber.properties +++ /dev/null @@ -1,24 +0,0 @@ -google-cloud-project=${GOOGLE_CLOUD_PROJECT} - -indexer-host=os-indexer-service -STORAGE_HOSTNAME=os-storage-service - -storage-schema-host=http://os-storage-service/api/storage/v2/schemas -storage-query-record-host=http://os-storage-service/api/storage/v2/query/records -storage-query-record-for-conversion-host=http://os-storage-service/api/storage/v2/query/records:batch -storage-records-batch-size=20 - -indexer-queue-host=http://os-storage-service/_dps/task-handlers/enqueue - -AUTHORIZE_API=http://os-entitlement-service/entitlements/v1 -LEGALTAG_API=http://os-legal-service/api/legal/v1 -CRS_API=${CRS_API} - -## use below values for gcp: opendes -REDIS_GROUP_HOST=${REDIS_GROUP_HOST} -redis-search-host=${REDIS_SEARCH_HOST} - -google-audiences=${GOOGLE_AUDIENCES} - -deployment-environment=cloud -disable.appengine.log.factory=true \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-local.properties b/provider/indexer-gcp/src/main/resources/application-local.properties deleted file mode 100644 index a5511b15c0564250bca41b67c6c39646f9512e79..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/resources/application-local.properties +++ /dev/null @@ -1,21 +0,0 @@ -google-cloud-project=osdu - -indexer-host=os-indexer-dot-opendes.appspot.com -STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com - -storage-schema-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas -storage-query-record-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records -storage-query-record-for-conversion-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch -storage-records-batch-size=20 - -indexer-queue-host=https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue - -AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1 -LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1 -CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1 - -## use below values for gcp: opendes -REDIS_GROUP_HOST=127.0.0.1 -redis-search-host=127.0.0.1 - -google-audiences=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com diff --git a/provider/indexer-gcp/src/main/resources/application-testing.properties b/provider/indexer-gcp/src/main/resources/application-testing.properties deleted file mode 100644 index 67941ce47b5fd27c99e487fa59c935d0bcacd827..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/resources/application-testing.properties +++ /dev/null @@ -1,21 +0,0 @@ -google-cloud-project=opendes-evt - -indexer-host=os-indexer-dot-opendes-evt.appspot.com -STORAGE_HOSTNAME=os-storage-dot-opendes-evt.appspot.com - -storage-schema-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/schemas -storage-query-record-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records -storage-query-record-for-conversion-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records:batch -storage-records-batch-size=20 - -indexer-queue-host=https://os-indexer-queue-dot-opendes-evt.appspot.com/_dps/task-handlers/enqueue - -AUTHORIZE_API=https://entitlements-dot-opendes-evt.appspot.com/entitlements/v1 -LEGALTAG_API=https://os-legal-dot-opendes-evt.appspot.com/api/legal/v1 -CRS_API=https://crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v1 - -## use below values for gcp: opendes -REDIS_GROUP_HOST=10.253.209.196 -redis-search-host=10.118.2.140 - -google-audiences=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties index e18440679858d1208f1f6f683220e78a7e372978..f8ffcf5aea0314dd8077f87179ceac92a173160d 100644 --- a/provider/indexer-gcp/src/main/resources/application.properties +++ b/provider/indexer-gcp/src/main/resources/application.properties @@ -1,18 +1,15 @@ -LOG_PREFIX=indexer - +#server config +server.port=8080 server.servlet.contextPath=/api/indexer/v2 + +#log config +LOG_PREFIX=indexer logging.level.org.springframework.web=${LOG_LEVEL:DEBUG} -server.port=8080 + +#jvm config JAVA_OPTS=-Xms3072m -Xmx3072m JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 -deployment-environment=CLOUD - -REDIS_GROUP_PORT=6379 -DEFAULT_DATA_COUNTRY=US - -redis-search-port=6379 - #Default Cache Settings schema-cache-expiration=60 index-cache-expiration=60 @@ -22,26 +19,41 @@ cursor-cache-expiration=60 kinds-cache-expiration=2880 # Attributes Cache expiration 2*24*60 attributes-cache-expiration=2880 - kinds-redis-database=1 cron-index-cleanup-threshold-days=3 cron-empty-index-cleanup-threshold-days=7 +#indexer service config +DEFAULT_DATA_COUNTRY=US gae-service=indexer -key-ring=csqp -kms-key=searchService +security.https.certificate.trust=false +storage-records-by-kind-batch-size=20 +storage-records-batch-size=20 -KEY_RING=${key-ring} -KMS_KEY=${kms-key} -GOOGLE_CLOUD_PROJECT=${google-cloud-project} +REDIS_GROUP_PORT=6379 +REDIS_GROUP_HOST=redis-cache-groups -elastic-datastore-kind=SearchSettings -elastic-datastore-id=indexer-service +REDIS_SEARCH_PORT=6379 +REDIS_SEARCH_HOST=redis-cache-search -security.https.certificate.trust=false -indexer.que.service.mail=default@iam.gserviceaccount.com -SCHEMA_HOST=${HOST}/api/schema-service/v1/schema -storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds +PARTITION_HOST=http://partition +PARTITION_PATH=/api/partition/v1 +PARTITION_API=${PARTITION_HOST}${PARTITION_PATH} + +ENTITLEMENTS_HOST=http://entitlements +ENTITLEMENTS_PATH=/api/entitlements/v2 +AUTHORIZE_API=${ENTITLEMENTS_HOST}${ENTITLEMENTS_PATH} + +STORAGE_HOST=http://storage +STORAGE_PATH=/api/storage/v2 +STORAGE_API=${STORAGE_HOST}${STORAGE_PATH} +STORAGE_QUERY_KINDS_HOST=${STORAGE_API}/query/kinds +STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_API}/query/records:batch +STORAGE_QUERY_RECORD_HOST=${STORAGE_API}/query/records +STORAGE_SCHEMA_HOST=${STORAGE_API}/schemas + +SCHEMA_BASE_HOST=http://schema +SCHEMA_PATH=/api/schema-service/v1/schema +SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH} -google.audiences=123.apps.googleusercontent.com -partition.api=http://localhost:8081/api/partition/v1 \ No newline at end of file +INDEXER_QUEUE_HOST=http://indexer-queue \ No newline at end of file diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java deleted file mode 100644 index c716cb1a637f5a5d843ebccd73656f2fa1c0752f..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java +++ /dev/null @@ -1,337 +0,0 @@ -package org.opengroup.osdu.indexer.service;//// Copyright 2017-2019, Schlumberger -//// -//// Licensed under the Apache License, Version 2.0 (the "License"); -//// you may not use this file except in compliance with the License. -//// You may obtain a copy of the License at -//// -//// http://www.apache.org/licenses/LICENSE-2.0 -//// -//// Unless required by applicable law or agreed to in writing, software -//// distributed under the License is distributed on an "AS IS" BASIS, -//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -//// See the License for the specific language governing permissions and -//// limitations under the License. -// -//package org.opendes.indexer.service; -// -//import com.google.gson.Gson; -//import com.google.gson.reflect.TypeToken; -//import org.elasticsearch.action.bulk.BulkItemResponse; -//import org.elasticsearch.action.bulk.BulkResponse; -//import org.elasticsearch.client.RequestOptions; -//import org.elasticsearch.client.RestHighLevelClient; -//import org.junit.Before; -//import org.junit.Ignore; -//import org.junit.Test; -//import org.junit.runner.RunWith; -//import org.mockito.InjectMocks; -//import org.mockito.Mock; -//import org.mockito.Spy; -//import org.opendes.client.api.DpsHeaders; -//import org.opendes.core.logging.JaxRsDpsLog; -//import org.opendes.core.model.DeploymentEnvironment; -//import org.opendes.core.model.RecordChangedMessages; -//import org.opendes.core.service.IndicesService; -//import org.opendes.core.util.Config; -//import org.opendes.core.util.ElasticClientHandler; -//import org.opendes.core.util.ElasticIndexNameResolver; -//import org.opendes.core.util.HeadersUtil; -//import org.opendes.indexer.logging.AuditLogger; -//import org.opendes.indexer.model.*; -//import org.opendes.indexer.publish.IPublisher; -//import org.opendes.indexer.util.IRequestInfo; -//import org.opendes.indexer.util.IndexerQueueTaskBuilder; -//import org.opendes.indexer.util.JobStatus; -//import org.opendes.indexer.util.RecordInfo; -//import org.powermock.core.classloader.annotations.PrepareForTest; -//import javax.inject.Inject; -//import org.springframework.context.annotation.Lazy; -//import org.springframework.test.context.junit4.SpringRunner; -// -//import java.io.IOException; -//import java.lang.reflect.Type; -//import java.util.*; -// -//import static java.util.Collections.singletonList; -//import static org.junit.Assert.*; -//import static org.mockito.Matchers.any; -//import static org.mockito.Mockito.verify; -//import static org.mockito.Mockito.when; -//import static org.powermock.api.mockito.PowerMockito.mock; -//import static org.powermock.api.mockito.PowerMockito.mockStatic; -// -//@Ignore -//@RunWith(SpringRunner.class) -//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class}) -//public class IndexerServiceTest { -// -// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," + -// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]"; -// private final String kind1 = "tenant1:testindexer1:well:1.0.0"; -// private final String kind2 = "tenant1:testindexer2:well:1.0.0"; -// private final String recordId1 = "tenant1:doc:test1"; -// private final String recordId2 = "tenant1:doc:test2"; -// private final String failureMassage = "test failure"; -// -// @Mock -// private IndexSchemaService indexSchemaService; -// @Mock -// private IndicesService indicesService; -// @Mock -// private IndexerMappingService indexerMappingService; -// @Mock -// private StorageService storageService; -// @Mock -// private IPublisher publisherImpl; -// @Mock -// private RestHighLevelClient restHighLevelClient; -// @Mock -// private ElasticClientHandler elasticClientHandler; -// @Mock -// private BulkResponse bulkResponse; -// @Mock -// private IRequestInfo requestInfo; -// @Mock -// private ElasticIndexNameResolver elasticIndexNameResolver; -// @Mock -// private AttributeParsingServiceImpl attributeParsingServiceImpl; -// @Mock -// private IndexerQueueTaskBuilder indexerQueueTaskBuilder; -// @Mock -// private JaxRsDpsLog log; -// @Mock -// private AuditLogger auditLogger; -// @InjectMocks -// private IndexerServiceImpl sut; -// @InjectMocks @Spy -// private JobStatus jobStatus = new JobStatus(); -// -// @Inject -// @Lazy -// private DpsHeaders dpsHeaders; -// private RecordChangedMessages recordChangedMessages; -// private List<RecordInfo> recordInfos; -// -// @Before -// public void setup() throws IOException { -// -// mockStatic(StorageAcl.class); -// mockStatic(Config.class); -// -// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); -// when(Config.getElasticClusterName()).thenReturn("CLUSTER"); -// when(Config.getElasticServerAddress()).thenReturn("testsite"); -// -// dpsHeaders = new DpsHeaders(); -// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1"); -// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); -// when(requestInfo.getHeaders()).thenReturn(dpsHeaders); -// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders()); -// -// Type listType = new TypeToken<List<RecordInfo>>() {}.getType(); -// recordInfos = (new Gson()).fromJson(pubsubMsg, listType); -// -// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); -// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse); -// -// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()}; -// when(bulkResponse.getItems()).thenReturn(responses); -// Map<String, String> attr = new HashMap<>(); -// attr.put(DpsHeaders.ACCOUNT_ID, "slb"); -// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build(); -// when(StorageAcl.flattenAcl(any())).thenReturn(null); -// } -// -// @Test -// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception { -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>()); -// -// assertNull(jobStatus); -// } -// -// @Test -// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() { -// try { -// indexSchemaServiceMock(kind1, null); -// indexSchemaServiceMock(kind2, null); -// List<ConversionStatus> conversionStatus = new LinkedList<>(); -// List<Records.Entity> validRecords = new ArrayList<>(); -// Map<String, Object> storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() { -// try { -// indexSchemaServiceMock(kind1, null); -// indexSchemaServiceMock(kind2, null); -// List<ConversionStatus> conversionStatuses = new LinkedList<>(); -// List<String> status=new ArrayList<>(); -// status.add("crs bla bla"); -// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build(); -// conversionStatuses.add(conversionStatus); -// List<Records.Entity> validRecords = new ArrayList<>(); -// Map<String, Object> storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla")); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() { -// try { -// List<Records.Entity> validRecords = new ArrayList<>(); -// List<ConversionStatus> conversionStatus = new LinkedList<>(); -// Map<String, Object> storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// -// Map<String, String> schema = createSchema(); -// indexSchemaServiceMock(kind1, schema); -// indexSchemaServiceMock(kind2, null); -// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0"); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop()); -// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() { -// try { -// Map<String, Object> storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// List<ConversionStatus> conversionStatus = new LinkedList<>(); -// List<Records.Entity> validRecords = new ArrayList<>(); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// Map<String, String> schema = createSchema(); -// indexSchemaServiceMock(kind2, schema); -// indexSchemaServiceMock(kind1, null); -// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() { -// try { -// Map<String, Object> storageData = new HashMap<>(); -// List<ConversionStatus> conversionStatus = new LinkedList<>(); -// -// storageData.put("schema1", "test-value"); -// List<Records.Entity> validRecords = new ArrayList<>(); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// Map<String, String> schema = createSchema(); -// indexSchemaServiceMock(kind2, schema); -// indexSchemaServiceMock(kind1, null); -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size()); -// -// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)")); -// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)")); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// private BulkItemResponse prepareResponseFail() { -// BulkItemResponse responseFail = mock(BulkItemResponse.class); -// when(responseFail.isFailed()).thenReturn(true); -// when(responseFail.getFailureMessage()).thenReturn(failureMassage); -// when(responseFail.getId()).thenReturn(recordId1); -// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure"))); -// return responseFail; -// } -// -// private BulkItemResponse prepareResponseSuccess() { -// BulkItemResponse responseSuccess = mock(BulkItemResponse.class); -// when(responseSuccess.getId()).thenReturn(recordId2); -// return responseSuccess; -// } -// -// private void indexSchemaServiceMock(String kind, Map<String, String> schema) { -// if (schema == null) { -// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build(); -// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema); -// } else { -// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build(); -// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema); -// } -// } -// -// private Map<String, String> createSchema() { -// Map<String, String> schema = new HashMap<>(); -// schema.put("schema1", "keyword"); -// schema.put("schema2", "boolean"); -// schema.put("schema3", "date"); -// schema.put("schema6", "object"); -// return schema; -// } -//} diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml index 6dc55aa9b59597be164515bf0a6379cd69802fdf..378bf7dace7c028b53b55110d81fced5a63d35e3 100644 --- a/provider/indexer-ibm/pom.xml +++ b/provider/indexer-ibm/pom.xml @@ -21,7 +21,7 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> @@ -31,7 +31,7 @@ <packaging>jar</packaging> <properties> - <os-core-lib-ibm.version>0.13.0-rc6</os-core-lib-ibm.version> + <os-core-lib-ibm.version>0.13.0</os-core-lib-ibm.version> </properties> <profiles> @@ -53,7 +53,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <exclusions> <exclusion> <groupId>io.netty</groupId> diff --git a/provider/indexer-reference/README.md b/provider/indexer-reference/README.md index 9fa7b2bb954f311e570265d8734c2fb0e989fe01..13529b214dccc308fa01f9439efeb5bc07274047 100644 --- a/provider/indexer-reference/README.md +++ b/provider/indexer-reference/README.md @@ -27,6 +27,8 @@ In order to run the service locally or remotely, you will need to have the follo | `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | | `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - | +| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - | | `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | | `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | | `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml index 0a0e1d0964271cc66be97fbf0c528956036e818a..3e1359fa94dae1c660471bfd797f0dc482d8b961 100644 --- a/provider/indexer-reference/pom.xml +++ b/provider/indexer-reference/pom.xml @@ -22,12 +22,12 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>indexer-reference</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <name>indexer-reference</name> <description>Indexer Service GCP Anthos</description> <packaging>jar</packaging> @@ -36,7 +36,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <dependency> diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml index 1fd89ff029cac32e6c9580dd231add4c010c9ec3..538651fb4407d563794d76bc9f5259b443484991 100644 --- a/testing/indexer-test-aws/pom.xml +++ b/testing/indexer-test-aws/pom.xml @@ -21,13 +21,13 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-aws</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <packaging>jar</packaging> <properties> @@ -42,7 +42,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <!-- AWS specific packages --> diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml index 2b157b63a7ad882852ac5e43e61e56e8ba39cbc7..4f4c6e50ac203625417c450c7fab210604ec13cb 100644 --- a/testing/indexer-test-azure/pom.xml +++ b/testing/indexer-test-azure/pom.xml @@ -21,13 +21,13 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-azure</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <packaging>jar</packaging> <properties> @@ -45,7 +45,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml index fcc2b5e81bc27d8f33b10d1ac5a46f1fcbd33e90..51bf9d386095dbcf92745592525ebaa1b360f105 100644 --- a/testing/indexer-test-core/pom.xml +++ b/testing/indexer-test-core/pom.xml @@ -5,13 +5,13 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <properties> <maven.compiler.target>1.8</maven.compiler.target> diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java index 51502c68198ab4c42c236e052c148c11287bda0e..b37e7daa324d2f73d3c85598dcb64a5317abaf5b 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java @@ -17,8 +17,10 @@ package org.opengroup.osdu.util; +import com.google.api.client.util.Strings; import com.google.gson.Gson; +import java.net.URI; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @@ -411,14 +413,22 @@ public class ElasticUtils { restHighLevelClient = new RestHighLevelClient(builder); } catch (Exception e) { - throw new AssertionError("Setup elastic error"); + throw new AssertionError("Setup elastic error: %s" + e.getMessage()); } return restHighLevelClient; } - public RestClientBuilder createClientBuilder(String host, String usernameAndPassword, int port) { + public RestClientBuilder createClientBuilder(String url, String usernameAndPassword, int port) throws Exception { String scheme = this.sslEnabled ? "https" : "http"; - RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, scheme)); + + url = url.trim().replaceAll("^(?i)(https?)://",""); + URI uri = new URI(scheme + "://" + url); + + RestClientBuilder builder = RestClient.builder(new HttpHost(uri.getHost(), port, uri.getScheme())); + if (!Strings.isNullOrEmpty(uri.getPath())) { + builder.setPathPrefix(uri.getPath()); + } + builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT) .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT)); diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml index e86a33cbd846f37586836b850bc340f176093c5b..c32245324cbf5bf9290df99dab3b0ff2886c3bce 100644 --- a/testing/indexer-test-gcp/pom.xml +++ b/testing/indexer-test-gcp/pom.xml @@ -6,13 +6,13 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-gcp</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <packaging>jar</packaging> <properties> @@ -37,7 +37,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <!-- Cucumber --> diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml index ce81d0fff255e43e061f97828393d31465a031c9..4531fdba9cfa2a310062e6161c6a32b359282d7f 100644 --- a/testing/indexer-test-ibm/pom.xml +++ b/testing/indexer-test-ibm/pom.xml @@ -6,20 +6,20 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-ibm</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <packaging>jar</packaging> <properties> <maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.source>1.8</maven.compiler.source> <cucumber.version>1.2.5</cucumber.version> - <os-core-lib-ibm.version>0.13.0-rc5</os-core-lib-ibm.version> + <os-core-lib-ibm.version>0.13.0</os-core-lib-ibm.version> </properties> <dependencies> @@ -38,7 +38,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <dependency> diff --git a/testing/pom.xml b/testing/pom.xml index 679e6dcc19bb2beca2e96243c9cbe5105db0dc47..08218f0dd1511159b46ec16efe599ca5d66ca7e2 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -18,7 +18,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <description>Indexer Service Integration Test Root Project</description> <properties> <spring.version>5.1.19.RELEASE</spring.version>