diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e411947582333f378a851e465fa19616afcffb9..15f4038e15cb89d98a9d7c1b63e235ea6183bb1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ variables: OSDU_GCP_VENDOR: gcp OSDU_GCP_QUEUE_SA_EMAIL: pub-sub-indexer-queue@nice-etching-277309.iam.gserviceaccount.com OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_API/schema --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL" + OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: "--set data.key_ring=$OSDU_GCP_PARTITION_KEY_RING --set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_API/schema --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL" OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s --set data.limits_cpu=1 --set data.limits_memory=1G" OSDU_GCP_HELM_CONFIG_SERVICE: indexer-config OSDU_GCP_HELM_DEPLOYMENT_SERVICE: indexer-deploy @@ -46,7 +47,10 @@ include: file: "scanners/gitlab-ultimate.yml" - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/aws.yml" + file: "cloud-providers/aws-global.yml" + + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/aws-maven.yml" - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/azure.yml" @@ -57,10 +61,8 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "publishing/pages.yml" - - project: 'osdu/platform/ci-cd-pipelines' - ref: "master" - file: 'cloud-providers/osdu-gcp-gke.yml' - + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/osdu-gcp-gke.yml" aws-test-java: - tags: ['aws-internal-test'] + tags: ["aws-internal-test"] diff --git a/NOTICE b/NOTICE index 6367281abcc8275e2f5e619733c59c3bfaf5140d..efc25f4175db2c582c97a31de26de967d3a553e6 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,6 @@ # 3rd-Party Software License Notice Generated by fossa-cli (https://github.com/fossas/fossa-cli). +Formatted by fossa-with-cache (https://community.opengroup.org/divido/fossa-with-cache). This software includes the following software and licenses: ======================================================================== @@ -14,11 +15,11 @@ Apache-1.1 ======================================================================== The following software have components provided under the terms of this license: -- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/) -- Cobertura (from http://cobertura.sourceforge.net) -- Plexus :: Default Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) -- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) -- oro (from ) +- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli) +- Cobertura code coverage (from http://cobertura.sourceforge.net) +- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) +- oro (from https://repo1.maven.org/maven2/oro/oro) ======================================================================== Apache-2.0 @@ -27,8 +28,7 @@ The following software have components provided under the terms of this license: - AMQP 1.0 JMS Spring Boot AutoConfiguration (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-autoconfigure) - AMQP 1.0 JMS Spring Boot Starter (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-starter) -- ASM based accessors helper used by json-smart (from ) -- ASM based accessors helper used by json-smart (from ) +- ASM based accessors helper used by json-smart (from https://urielch.github.io/) - AWS Java SDK for AWS Amplify (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for AWS App Mesh (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for AWS AppSync (from https://aws.amazon.com/sdkforjava) @@ -272,7 +272,6 @@ The following software have components provided under the terms of this license: - AWS Java SDK for CodeArtifact (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for EC2 Image Builder (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for Elastic Load Balancing (from https://aws.amazon.com/sdkforjava) -- AWS Java SDK for Elastic Load Balancing (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for FinSpace Public API (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for FinSpace User Environment Management (from https://aws.amazon.com/sdkforjava) - AWS Java SDK for Firewall Management (from https://aws.amazon.com/sdkforjava) @@ -288,40 +287,42 @@ The following software have components provided under the terms of this license: - AWS SDK for Java - Core (from https://aws.amazon.com/sdkforjava) - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava) - Adapter: RxJava (from https://repo1.maven.org/maven2/com/squareup/retrofit2/adapter-rxjava) -- Apache Ant + JUnit (from http://ant.apache.org/) -- Apache Ant Core (from https://ant.apache.org/) -- Apache Ant Launcher (from https://ant.apache.org/) -- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/) -- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/) +- Apache Ant + JUnit (from http://ant.apache.org/, https://ant.apache.org/) +- Apache Ant Core +- Apache Ant Launcher (from http://ant.apache.org/) +- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli) - Apache Commons Codec (from https://commons.apache.org/proper/commons-codec/) - Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/) +- Apache Commons IO (from http://commons.apache.org/io/, https://commons.apache.org/proper/commons-io/, https://repo1.maven.org/maven2/commons-io/commons-io) - Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/) -- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/) +- Apache Commons Logging (from http://commons.apache.org/logging/, http://commons.apache.org/proper/commons-logging/) +- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/, http://jakarta.apache.org/commons/${pom.artifactId.substring(8)}/, https://repo1.maven.org/maven2/commons-validator/commons-validator) - Apache Geronimo JMS Spec 2.0 (from http://geronimo.apache.org/maven/${siteId}/${version}) +- Apache Groovy (from http://groovy-lang.org, http://groovy.codehaus.org/, https://groovy-lang.org) - Apache HTTP transport v2 for the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-apache-v2) - Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient) -- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client) -- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga) -- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga) +- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core-ga/, http://hc.apache.org/httpcomponents-core/) +- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core/) - Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api) - Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core) - Apache Log4j JUL Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul) - Apache Log4j SLF4J Binding (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl) - Apache Log4j to SLF4J Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-to-slf4j) +- Apache Maven Invoker (from https://repo1.maven.org/maven2/org/apache/maven/shared/maven-invoker) - Apache Maven Wagon :: API (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-provider-api) - Apache Maven Wagon :: Providers :: File Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-file) - Apache Maven Wagon :: Providers :: HTTP Shared Library (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-http-shared) +- Apache Maven Wagon :: Providers :: Lightweight HTTP Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-http-lightweight) - Apache Maven Wagon :: Providers :: SSH Common Library (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh-common) - Apache Maven Wagon :: Providers :: SSH External Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh-external) - Apache Maven Wagon :: Providers :: SSH Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh) -- Apache Velocity (from http://velocity.apache.org/engine/devel/) +- Apache Velocity (from http://velocity.apache.org/engine/releases/velocity-1.6.1/, http://velocity.apache.org/engine/releases/velocity-1.6.2/) - AssertJ fluent assertions (from https://repo1.maven.org/maven2/org/assertj/assertj-core) - Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client) - Asynchronous Http Client Netty Utils (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client-netty-utils) -- AutoValue Annotations (from https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations) -- AutoValue Annotations (from https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations) +- AutoValue Annotations (from https://github.com/google/auto/tree/master/value, https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations) - Azure Spring Boot Starter for Azure AD Spring Security Integration (from https://github.com/Azure/azure-sdk-for-java) -- BSON (from https://bsonspec.org) +- BSON (from http://bsonspec.org, https://bsonspec.org) - Bean Validation API (from http://beanvalidation.org) - Brave (from https://repo1.maven.org/maven2/io/zipkin/brave/brave) - Brave Instrumentation: Http Adapters (from https://repo1.maven.org/maven2/io/zipkin/brave/brave-instrumentation-http) @@ -329,151 +330,121 @@ The following software have components provided under the terms of this license: - Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy) - Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent) - ClassMate (from http://github.com/cowtowncoder/java-classmate) -- Cloud Key Management Service (KMS) API v1-rev20210820-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms) -- Cloud Key Management Service (KMS) API v1-rev20210820-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms) -- Cloud Storage JSON API v1-rev20210918-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage) -- Cloud Storage JSON API v1-rev20210918-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage) +- Cloud Key Management Service (KMS) API v1-rev81-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms) +- Cloud Storage API v1-rev40-1.20.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage) - CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava) -- Cobertura (from http://cobertura.sourceforge.net) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - Collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections) -- Commons IO (from http://commons.apache.org/io/) -- Commons Lang (from http://commons.apache.org/lang/) -- Commons Validator (from http://commons.apache.org/validator/) -- Converter: Jackson (from https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson) +- Converter: Jackson (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson) - Core functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty) -- Data Mapper for Jackson (from http://jackson.codehaus.org) +- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) - Digester (from https://repo1.maven.org/maven2/commons-digester/commons-digester) - Doxia :: APT Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-apt) - Doxia :: Core (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-core) - Doxia :: FML Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-fml) -- Doxia :: Sink API (from http://maven.apache.org/doxia/doxia/doxia-sink-api/) +- Doxia :: Sink API (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-sink-api) - Doxia :: XDoc Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-xdoc) - Doxia :: XHTML Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-xhtml) -- Doxia Sitetools :: Decoration Model (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-decoration-model) -- Doxia Sitetools :: Site Renderer Component (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-site-renderer) +- Doxia Sitetools :: Decoration Model (from http://maven.apache.org/doxia/doxia-sitetools/doxia-decoration-model/, https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-decoration-model) +- Doxia Sitetools :: Site Renderer (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/, https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-site-renderer) - Elastic JNA Distribution (from https://github.com/java-native-access/jna) -- Elasticsearch: Core (from https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch) -- Expression Language 3.0 (from http://uel.java.net) - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) - GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson) -- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson) -- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) - Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) - Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine) -- Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine) -- Google Cloud Core (from https://github.com/googleapis/java-core) -- Google Cloud Core (from https://github.com/googleapis/java-core) -- Google Cloud Core HTTP (from https://github.com/googleapis/java-core) -- Google Cloud Core HTTP (from https://github.com/googleapis/java-core) -- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc) -- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc) -- Google Cloud Datastore (from https://github.com/googleapis/java-datastore) -- Google Cloud Datastore (from https://github.com/googleapis/java-datastore) -- Google Cloud IAM Service Account Credentials (from https://github.com/googleapis/java-iamcredentials) +- Google Cloud Core (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core, https://github.com/googleapis/java-core) +- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http, https://github.com/googleapis/java-core) +- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-grpc, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core) +- Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore, https://github.com/googleapis/java-datastore) - Google Cloud IAM Service Account Credentials (from https://github.com/googleapis/java-iamcredentials) -- Google Cloud Logging (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging) -- Google Cloud Logging (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging) -- Google Cloud Pub/Sub (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub) -- Google Cloud Pub/Sub (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub) -- Google Cloud Storage (from https://github.com/googleapis/java-storage) -- Google Cloud Storage (from https://github.com/googleapis/java-storage) +- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging, https://github.com/googleapis/java-logging) +- Google Cloud Pub/Sub (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub, https://github.com/googleapis/java-pubsub) +- Google Cloud Storage (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-storage, https://github.com/googleapis/java-storage) - Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client) -- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client) -- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client) -- Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client) - Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client) -- Gson (from http://code.google.com/p/google-gson/) -- Gson (from http://code.google.com/p/google-gson/) +- Gson (from http://code.google.com/p/google-gson/, https://repo1.maven.org/maven2/com/google/code/gson/gson) - Guava InternalFutureFailureAccess and InternalFutures (from https://repo1.maven.org/maven2/com/google/guava/failureaccess) -- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) -- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) +- Guava: Google Core Libraries for Java (from http://code.google.com/p/guava-libraries, https://github.com/google/guava, https://repo1.maven.org/maven2/com/google/guava/guava) - HPPC Collections (from https://repo1.maven.org/maven2/com/carrotsearch/hppc) - HTTP functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty) -- Hibernate Validator Engine (from https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator) +- Hibernate Validator (from https://repo1.maven.org/maven2/org/hibernate/hibernate-validator, https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator) +- HikariCP (from https://github.com/brettwooldridge/HikariCP) +- Hop (from https://github.com/rabbitmq/hop, https://www.rabbitmq.com) - HttpClient (from http://hc.apache.org/httpcomponents-client) +- HttpClient Cache (from http://hc.apache.org/httpcomponents-client) - IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java) - IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java) - IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java) - IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java) -- Identity and Access Management (IAM) API v1-rev20191010-1.30.3 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam) -- J2ObjC Annotations (from https://github.com/google/j2objc/) +- Identity and Access Management (IAM) API v1-rev289-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam) - J2ObjC Annotations (from https://github.com/google/j2objc/) - JBoss Logging 3 (from http://www.jboss.org) - JCIP Annotations under Apache License (from http://stephenc.github.com/jcip-annotations) -- JDOM (from http://www.jdom.org) +- JDOM - JMES Path Query library (from https://aws.amazon.com/sdkforjava) -- JSON Small and Fast Parser (from https://repo1.maven.org/maven2/net/minidev/json-smart) -- JSON Small and Fast Parser (from https://repo1.maven.org/maven2/net/minidev/json-smart) +- JSON Small and Fast Parser (from https://repo1.maven.org/maven2/net/minidev/json-smart, https://urielch.github.io/) - JSON Web Token support for the JVM (from https://repo1.maven.org/maven2/io/jsonwebtoken/jjwt) - JSON library from Android SDK (from http://developer.android.com/sdk) - JSON.simple (from http://code.google.com/p/json-simple/) - JSONassert (from http://github.com/skyscreamer/yoga) - JSR107 API and SPI (from https://github.com/jsr107/jsr107spec) - Jackson (from http://jackson.codehaus.org) -- Jackson (from http://jackson.codehaus.org) -- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2) - Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2) - Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary) -- Jackson datatype: JSR310 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310) -- Jackson datatype: JSR310 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310) -- Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8) +- Jackson dataformat: Smile (from http://github.com/FasterXML/jackson-dataformat-smile, http://github.com/FasterXML/jackson-dataformats-binary) +- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310, https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310) +- Jackson datatype: Joda (from http://wiki.fasterxml.com/JacksonModuleJoda, https://github.com/FasterXML/jackson-datatype-joda) - Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8) - Jackson extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson) -- Jackson module: Afterburner (from https://github.com/FasterXML/jackson-modules-base) -- Jackson-annotations (from http://github.com/FasterXML/jackson) -- Jackson-annotations (from http://github.com/FasterXML/jackson) -- Jackson-core (from https://github.com/FasterXML/jackson) -- Jackson-dataformat-Smile (from http://wiki.fasterxml.com/JacksonForSmile) -- Jackson-dataformat-Smile (from http://wiki.fasterxml.com/JacksonForSmile) -- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding) -- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text) -- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text) -- Jackson-datatype-Joda (from http://wiki.fasterxml.com/JacksonModuleJoda) -- Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations) -- Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations) -- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names) +- Jackson module: Afterburner (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-modules-base) +- Jackson module: JAXB Annotations (from http://github.com/FasterXML/jackson-module-jaxb-annotations, http://wiki.fasterxml.com/JacksonJAXBAnnotations, https://github.com/FasterXML/jackson-modules-base) +- Jackson-annotations (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome) +- Jackson-core (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-core) +- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding, https://github.com/FasterXML/jackson-dataformat-xml) +- Jackson-dataformat-YAML (from http://wiki.fasterxml.com/JacksonExtensionYAML, https://github.com/FasterXML/jackson, https://github.com/FasterXML/jackson-dataformats-text) - Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names) - Jakarta Bean Validation API (from https://beanvalidation.org) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- Jakarta Servlet (from http://servlet-spec.java.net, https://projects.eclipse.org/projects/ee4j.servlet) - Java Libraries for Amazon Simple WorkFlow (from https://aws.amazon.com/sdkforjava) -- Java Native Access (from https://github.com/java-native-access/jna) +- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) -- Java Servlet API (from http://servlet-spec.java.net) - Java UUID Generator (from http://wiki.fasterxml.com/JugHome) - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) - JetBrains Java Annotations (from https://github.com/JetBrains/java-annotations) -- Jetty Server (from http://jetty.mortbay.org) -- Jetty Utilities (from http://jetty.mortbay.org) -- Joda-Time (from https://www.joda.org/joda-time/) +- Jetty :: Utilities (from http://jetty.mortbay.org, http://www.eclipse.org/jetty, https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util) +- Jetty Server (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty) +- Joda-Time (from http://joda-time.sourceforge.net, http://www.joda.org/joda-time/, https://www.joda.org/joda-time/) - KeePassJava2 :: All (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2) - KeePassJava2 :: DOM (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-dom) - KeePassJava2 :: JAXB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-jaxb) - KeePassJava2 :: KDB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdb) - KeePassJava2 :: KDBX (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdbx) - KeePassJava2 :: Simple (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-simple) +- Kotlin Stdlib (from https://kotlinlang.org/, https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib) +- Kotlin Stdlib Common (from https://kotlinlang.org/) +- Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang) - Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic) - Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core) - Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) -- Lucene Grouping (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping) -- Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter) -- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join) -- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory) -- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs) +- Lucene Grouping (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping) +- Lucene Highlighter (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter) +- Lucene Join (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-join) +- Lucene Memory (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs, https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory) - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc) -- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries) -- Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser) -- Lucene Sandbox (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox) -- Lucene Spatial 3D (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial3d) +- Lucene Queries (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries) +- Lucene QueryParsers (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser) +- Lucene Sandbox (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox) +- Lucene Spatial 3D (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial3d) - Lucene Spatial Extras (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial-extras) - Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest) -- MapStruct Core (from https://repo1.maven.org/maven2/org/mapstruct/mapstruct) +- MapStruct Core (from http://mapstruct.org/mapstruct/, https://repo1.maven.org/maven2/org/mapstruct/mapstruct) - Maven Artifact (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact) - Maven Artifact Manager (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact-manager) - Maven Core (from https://repo1.maven.org/maven2/org/apache/maven/maven-core) - Maven Error Diagnostics (from https://repo1.maven.org/maven2/org/apache/maven/maven-error-diagnostics) -- Maven Invoker (from https://repo1.maven.org/maven2/org/apache/maven/shared/maven-invoker) - Maven Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-model) - Maven Monitor (from https://repo1.maven.org/maven2/org/apache/maven/maven-monitor) - Maven Plugin API (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-api) @@ -486,7 +457,6 @@ The following software have components provided under the terms of this license: - Maven Reporting Implementation (from https://repo1.maven.org/maven2/org/apache/maven/reporting/maven-reporting-impl) - Maven Repository Metadata Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-repository-metadata) - Maven Settings (from https://repo1.maven.org/maven2/org/apache/maven/maven-settings) -- Maven Wagon Lightweight HTTP Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-http-lightweight) - Metrics Core (from https://repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -494,174 +464,141 @@ The following software have components provided under the terms of this license: - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) -- Mockito (from http://www.mockito.org) +- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito) - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/) -- MongoDB Java Driver (from http://www.mongodb.org) -- MongoDB Java Driver Core (from http://www.mongodb.org) -- Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer) +- MongoDB Driver (from http://www.mongodb.org) +- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org) - Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer) - Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec) -- Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec) - Netty/Codec/DNS (from https://repo1.maven.org/maven2/io/netty/netty-codec-dns) - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) -- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) - Netty/Codec/HTTP2 (from https://repo1.maven.org/maven2/io/netty/netty-codec-http2) - Netty/Codec/Socks (from https://repo1.maven.org/maven2/io/netty/netty-codec-socks) - Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) -- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) -- Netty/Handler (from https://repo1.maven.org/maven2/io/netty/netty-handler) - Netty/Handler (from https://repo1.maven.org/maven2/io/netty/netty-handler) - Netty/Handler/Proxy (from https://repo1.maven.org/maven2/io/netty/netty-handler-proxy) - Netty/Resolver (from https://repo1.maven.org/maven2/io/netty/netty-resolver) -- Netty/Resolver (from https://repo1.maven.org/maven2/io/netty/netty-resolver) - Netty/Resolver/DNS (from https://repo1.maven.org/maven2/io/netty/netty-resolver-dns) +- Netty/Resolver/DNS/Classes/MacOS (from https://repo1.maven.org/maven2/io/netty/netty-resolver-dns-classes-macos) - Netty/TomcatNative [BoringSSL - Static] (from https://repo1.maven.org/maven2/io/netty/netty-tcnative-boringssl-static) - Netty/Transport (from https://repo1.maven.org/maven2/io/netty/netty-transport) -- Netty/Transport (from https://repo1.maven.org/maven2/io/netty/netty-transport) -- Netty/Transport/Native/Unix/Common (from https://repo1.maven.org/maven2/io/netty/netty-transport-native-unix-common) +- Netty/Transport/Classes/Epoll (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-epoll) +- Netty/Transport/Classes/KQueue (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-kqueue) - Netty/Transport/Native/Unix/Common (from https://repo1.maven.org/maven2/io/netty/netty-transport-native-unix-common) - Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type) -- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) -- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) -- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) -- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) -- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) -- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags) +- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt, https://bitbucket.org/nimbusds/nimbus-jose-jwt) - Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags) -- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor-core) -- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) -- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) +- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor, https://github.com/reactor/reactor-core) - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) -- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) -- OAuth2 for Spring Security (from https://repo1.maven.org/maven2/org/springframework/security/oauth/spring-security-oauth2) - OAuth2 for Spring Security (from https://repo1.maven.org/maven2/org/springframework/security/oauth/spring-security-oauth2) - Objenesis (from http://objenesis.org) -- OkHttp Logging Interceptor (from https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor) -- Okio (from https://github.com/square/okio/) -- Okio (from https://github.com/square/okio/) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) -- OpenCensus (from https://github.com/census-instrumentation/opencensus-java) +- OkHttp (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp, https://square.github.io/okhttp/) +- OkHttp Logging Interceptor (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor, https://square.github.io/okhttp/) +- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection, https://square.github.io/okhttp/) +- Okio (from https://github.com/square/okio/, https://repo1.maven.org/maven2/com/squareup/okio/okio) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) - PWDB :: Database (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/database) -- Plexus :: Default Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) -- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) - Plexus Velocity Component (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-velocity) -- PowerMock (from http://www.powermock.org) -- PowerMock (from http://www.powermock.org) -- PowerMock (from http://www.powermock.org) -- PowerMock (from http://www.powermock.org) -- PowerMock (from http://www.powermock.org) -- PowerMock (from http://www.powermock.org) -- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf) +- PostgreSQL JDBC Driver +- PowerMock (from http://www.powermock.org, https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito) - Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf) - Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j) - QpidJMS Client (from https://repo1.maven.org/maven2/org/apache/qpid/qpid-jms-client) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) - Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty) -- Retrofit (from https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit) -- Servlet Specification 2.5 API (from http://jetty.mortbay.org) -- SnakeYAML (from http://www.snakeyaml.org) -- Spatial4J (from http://www.locationtech.org/projects/locationtech.spatial4j) -- Spring AMQP Core (from https://projects.spring.io/spring-amqp) -- Spring AOP (from https://github.com/spring-projects/spring-framework) -- Spring Beans (from https://github.com/spring-projects/spring-framework) -- Spring Boot (from http://projects.spring.io/spring-boot/) -- Spring Boot AMQP Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot AOP Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Actuator (from http://projects.spring.io/spring-boot/) -- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/) -- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/) -- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Logging Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Test (from http://projects.spring.io/spring-boot/) -- Spring Boot Test Auto-Configure (from http://projects.spring.io/spring-boot/) -- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Tomcat Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Validation Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/) +- Retrofit (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit) +- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) +- Simple XML (safe) (from https://github.com/dweiss/simplexml) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) +- Spatial4J (from https://projects.eclipse.org/projects/locationtech.spatial4j) +- Spring AMQP Core (from http://www.springsource.org/spring-amqp, https://github.com/spring-projects/spring-amqp, https://projects.spring.io/spring-amqp) +- Spring AOP (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-aop) +- Spring Beans (from http://www.springframework.org, https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-beans) +- Spring Boot (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot, https://spring.io/projects/spring-boot) +- Spring Boot AMPQ Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-amqp, https://spring.io/projects/spring-boot) +- Spring Boot AOP Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Actuator (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator, https://spring.io/projects/spring-boot) +- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure, https://spring.io/projects/spring-boot) +- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-actuator, https://spring.io/projects/spring-boot) +- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-autoconfigure, https://spring.io/projects/spring-boot) +- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-tools/spring-boot-configuration-processor, https://spring.io/projects/spring-boot) +- Spring Boot Data MongoDB Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-data-mongodb, https://spring.io/projects/spring-boot) +- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot JDBC Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-jersey, https://spring.io/projects/spring-boot) +- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json, https://spring.io/projects/spring-boot) +- Spring Boot Log4j 2 Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-log4j2, https://spring.io/projects/spring-boot) +- Spring Boot Logging Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-logging, https://spring.io/projects/spring-boot) +- Spring Boot Reactor Netty Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-reactor-netty, https://spring.io/projects/spring-boot) +- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Test (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Test Auto-Configure (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-test-autoconfigure, https://spring.io/projects/spring-boot) +- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot Tomcat Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat, https://spring.io/projects/spring-boot) +- Spring Boot Validation Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-validation, https://spring.io/projects/spring-boot) +- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot) +- Spring Boot WebFlux Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-webflux, https://spring.io/projects/spring-boot) - Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework) -- Spring Context (from https://github.com/spring-projects/spring-framework) -- Spring Core (from https://github.com/spring-projects/spring-framework) -- Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons) +- Spring Context (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-context) +- Spring Core (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-core) - Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons) - Spring Data MongoDB - Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-mongodb) -- Spring Expression Language (SpEL) (from https://github.com/spring-projects/spring-framework) -- Spring JMS (from https://github.com/spring-projects/spring-framework) +- Spring Expression Language (SpEL) (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-expression) +- Spring JMS (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-jms) - Spring Messaging (from https://github.com/spring-projects/spring-framework) +- Spring Plugin - Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core) - Spring Plugin - Metadata Extension (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-metadata) -- Spring Plugin Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core) -- Spring RabbitMQ Support (from https://projects.spring.io/spring-amqp) +- Spring RabbitMQ Support (from http://www.springsource.org/spring-amqp, https://github.com/spring-projects/spring-amqp, https://projects.spring.io/spring-amqp) - Spring Retry (from http://www.springsource.org) +- Spring Security - Core (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-core, https://spring.io/projects/spring-security, https://spring.io/spring-security) +- Spring Security - Namespace Configuration Module (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-config, https://spring.io/spring-security) +- Spring Security - Web Application Security Module (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-web, https://spring.io/projects/spring-security, https://spring.io/spring-security) - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) -- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) -- Spring TestContext Framework (from https://github.com/spring-projects/spring-framework) -- Spring Transaction (from https://github.com/spring-projects/spring-framework) -- Spring Web (from https://github.com/spring-projects/spring-framework) -- Spring Web MVC (from https://github.com/spring-projects/spring-framework) +- Spring TestContext Framework (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework) +- Spring Transaction (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework) +- Spring Web (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-web) +- Spring Web MVC (from https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-webmvc) - Spring WebFlux (from https://github.com/spring-projects/spring-framework) - T-Digest (from https://github.com/tdunning/t-digest) -- Vavr (from http://vavr.io) +- Vavr (from http://vavr.io, https://www.vavr.io) - Vavr Match (from http://vavr.io) - Woodstox (from https://github.com/FasterXML/woodstox) -- Xerces2-j (from https://xerces.apache.org/xerces2-j/) -- Zipkin Core Library (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin) +- Xerces2-j (from https://repo1.maven.org/maven2/xerces/xercesImpl, https://xerces.apache.org/xerces2-j/) - Zipkin Reporter Brave (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter-brave) - Zipkin Reporter: Core (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter) -- aalto-xml (from https://repo1.maven.org/maven2/com/fasterxml/aalto-xml) +- Zipkin v2 (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin) +- aalto-xml (from https://github.com/FasterXML/aalto-xml, https://repo1.maven.org/maven2/com/fasterxml/aalto-xml) - aggs-matrix-stats (from https://github.com/elastic/elasticsearch) -- cli (from https://github.com/elastic/elasticsearch) +- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis) +- commons-beanutils (from https://repo1.maven.org/maven2/commons-beanutils/commons-beanutils) - compiler (from http://github.com/spullara/mustache.java) -- core (from https://github.com/elastic/elasticsearch) -- datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client) - datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client) +- elasticsearch (from https://github.com/elastic/elasticsearch, https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch) +- elasticsearch-cli (from https://github.com/elastic/elasticsearch) +- elasticsearch-core (from https://github.com/elastic/elasticsearch) - elasticsearch-geo (from https://github.com/elastic/elasticsearch) +- elasticsearch-secure-sm (from https://github.com/elastic/elasticsearch) +- elasticsearch-x-content (from https://github.com/elastic/elasticsearch) - error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) -- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) -- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) -- grpc-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/grpc-google-cloud-pubsub-v1) -- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) - io.grpc:grpc-api (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-api (from https://github.com/grpc/grpc-java) - io.grpc:grpc-auth (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-context (from https://github.com/grpc/grpc-java) - io.grpc:grpc-context (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-context (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-context (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-core (from https://github.com/grpc/grpc-java) - io.grpc:grpc-core (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-core (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-grpclb (from https://github.com/grpc/grpc-java) - io.grpc:grpc-grpclb (from https://github.com/grpc/grpc-java) - io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java) - io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) - io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) -- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) - io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) -- ion-java (from https://github.com/amznlabs/ion-java/) -- ion-java (from https://github.com/amznlabs/ion-java/) -- jackson-databind (from http://github.com/FasterXML/jackson) -- java-cloudant (from https://cloudant.com) +- ion-java (from https://github.com/amzn/ion-java/, https://github.com/amznlabs/ion-java/) +- jackson-databind (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome) +- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject) - java-cloudant (from https://cloudant.com) - javatuples (from http://www.javatuples.org) - javax.inject (from http://code.google.com/p/atinject/) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -673,158 +610,133 @@ The following software have components provided under the terms of this license: - jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) - jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - jose4j (from https://bitbucket.org/b_c/jose4j/) +- json-path (from http://code.google.com/p/json-path/, https://github.com/jayway/JsonPath) - lang-mustache (from https://github.com/elastic/elasticsearch) -- lettuce (from http://github.com/mp911de/lettuce/wiki) +- lettuce (from http://github.com/mp911de/lettuce/wiki, https://github.com/lettuce-io/lettuce-core/wiki) - mapper-extras (from https://github.com/elastic/elasticsearch) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer) -- mockito-core (from https://github.com/mockito/mockito) -- mockito-core (from https://github.com/mockito/mockito) -- mockito-core (from https://github.com/mockito/mockito) -- nio-multipart-parser (from ) -- nio-stream-storage (from https://github.com/synchronoss/nio-stream-storage) -- okhttp (from https://square.github.io/okhttp/) -- okhttp (from https://square.github.io/okhttp/) -- okhttp-urlconnection (from https://square.github.io/okhttp/) -- okhttp-urlconnection (from https://square.github.io/okhttp/) +- minio (from https://github.com/minio/minio-java) +- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian) - org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/) -- org.xmlunit:xmlunit-core (from https://www.xmlunit.org/) +- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j) +- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/, https://www.xmlunit.org/) - parent-join (from https://github.com/elastic/elasticsearch) - perfmark:perfmark-api (from https://github.com/perfmark/perfmark) -- project ':json-path' (from https://github.com/jayway/JsonPath) -- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/java-datastore/proto-google-cloud-datastore-v1) -- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/java-datastore/proto-google-cloud-datastore-v1) -- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1) -- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1) -- proto-google-cloud-logging-v2 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2) -- proto-google-cloud-logging-v2 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2) -- proto-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-pubsub-v1) -- proto-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-pubsub-v1) -- proto-google-common-protos (from https://github.com/googleapis/java-iam/proto-google-common-protos) -- proto-google-common-protos (from https://github.com/googleapis/java-iam/proto-google-common-protos) -- proto-google-common-protos (from https://github.com/googleapis/java-iam/proto-google-common-protos) -- proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1) -- proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1) +- powermock-api-mockito2 (from https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito2) +- powermock-api-support (from https://repo1.maven.org/maven2/org/powermock/powermock-api-support) +- powermock-reflect (from https://repo1.maven.org/maven2/org/powermock/powermock-reflect) +- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-datastore/proto-google-cloud-datastore-v1, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-datastore-v1) +- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-iamcredentials-v1) +- proto-google-cloud-logging-v2 (from https://github.com/googleapis/java-logging/proto-google-cloud-logging-v2, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2) +- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1) +- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos) +- proto-google-iam-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-iam-v1) - rank-eval (from https://github.com/elastic/elasticsearch) -- resilience4j (from https://resilience4j.readme.io) -- resilience4j (from https://resilience4j.readme.io) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://resilience4j.readme.io) -- resilience4j (from https://github.com/resilience4j/resilience4j) -- resilience4j (from https://github.com/resilience4j/resilience4j) +- resilience4j (from https://github.com/resilience4j/resilience4j, https://resilience4j.readme.io, ttps://resilience4j.readme.io) - rest (from https://github.com/elastic/elasticsearch) - rest-high-level (from https://github.com/elastic/elasticsearch) - rxjava (from https://github.com/ReactiveX/RxJava) -- secure-sm (from https://github.com/elastic/elasticsearch) -- spring-boot-actuator-autoconfigure (from https://spring.io/projects/spring-boot) -- spring-boot-dependencies (from https://spring.io/projects/spring-boot) -- spring-boot-starter-data-mongodb (from https://spring.io/projects/spring-boot) -- spring-boot-starter-json (from https://spring.io/projects/spring-boot) -- spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot) -- spring-boot-starter-reactor-netty (from https://spring.io/projects/spring-boot) -- spring-boot-starter-webflux (from https://spring.io/projects/spring-boot) -- spring-security-config (from http://spring.io/spring-security) -- spring-security-core (from http://spring.io/spring-security) -- spring-security-oauth2-client (from https://spring.io/spring-security) -- spring-security-oauth2-core (from https://spring.io/spring-security) -- spring-security-oauth2-jose (from https://spring.io/spring-security) -- spring-security-oauth2-jose (from https://spring.io/spring-security) -- spring-security-oauth2-resource-server (from https://spring.io/spring-security) -- spring-security-test (from http://spring.io/spring-security) -- spring-security-web (from http://spring.io/spring-security) +- spring-jdbc (from https://repo1.maven.org/maven2/org/springframework/spring-jdbc) +- spring-security-oauth2-client (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security) +- spring-security-oauth2-core (from http://spring.io/spring-security, https://spring.io/spring-security) +- spring-security-oauth2-jose (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security) +- spring-security-test (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security) +- springfox-bean-validators (from https://github.com/springfox/springfox) +- springfox-boot-starter (from https://github.com/springfox/springfox) - springfox-core (from https://github.com/springfox/springfox) +- springfox-data-rest (from https://github.com/springfox/springfox) +- springfox-oas (from https://github.com/springfox/springfox) - springfox-schema (from https://github.com/springfox/springfox) - springfox-spi (from https://github.com/springfox/springfox) - springfox-spring-web (from https://github.com/springfox/springfox) +- springfox-spring-webflux (from https://github.com/springfox/springfox) +- springfox-spring-webmvc (from https://github.com/springfox/springfox) - springfox-swagger-common (from https://github.com/springfox/springfox) - springfox-swagger-ui (from https://github.com/springfox/springfox) - springfox-swagger2 (from https://github.com/springfox/springfox) -- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations) +- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/core/v3/swagger-annotations, https://repo1.maven.org/maven2/io/swagger/swagger-annotations) - swagger-jaxrs (from https://repo1.maven.org/maven2/io/swagger/swagger-jaxrs) -- swagger-models (from https://repo1.maven.org/maven2/io/swagger/swagger-models) -- tomcat-annotations-api (from http://tomcat.apache.org/) +- swagger-models (from https://repo1.maven.org/maven2/io/swagger/core/v3/swagger-models, https://repo1.maven.org/maven2/io/swagger/swagger-models) +- tomcat-annotations-api (from http://tomcat.apache.org/, https://tomcat.apache.org/) - tomcat-embed-core (from http://tomcat.apache.org/) -- tomcat-embed-el (from http://tomcat.apache.org/) -- tomcat-embed-websocket (from http://tomcat.apache.org/) -- x-content (from https://github.com/elastic/elasticsearch) -- xml-apis (from ) +- tomcat-embed-el (from http://tomcat.apache.org/, https://tomcat.apache.org/) +- tomcat-embed-websocket (from http://tomcat.apache.org/, https://tomcat.apache.org/) +- xml-apis ======================================================================== BSD-2-Clause ======================================================================== The following software have components provided under the terms of this license: -- API Common (from https://github.com/googleapis/api-common-java) -- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) +- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java) +- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli) +- GAX (Google Api eXtensions) (from https://github.com/googleapis, https://github.com/googleapis/gax-java) +- Hamcrest (from http://hamcrest.org/JavaHamcrest/) +- Hamcrest Core (from https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) -- Reflections (from http://github.com/ronmamo/reflections) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) +- PostgreSQL JDBC Driver +- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections) - Stax2 API (from http://github.com/FasterXML/stax2-api) -- ThreeTen backport (from https://www.threeten.org/threetenbp) -- ThreeTen backport (from https://www.threeten.org/threetenbp) +- ThreeTen backport (from https://github.com/ThreeTen/threetenbp, https://www.threeten.org/threetenbp) +- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) - jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) ======================================================================== BSD-3-Clause ======================================================================== The following software have components provided under the terms of this license: -- API Common (from https://github.com/googleapis/api-common-java) -- API Common (from https://github.com/googleapis/api-common-java) +- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java) +- ASM Analysis (from http://asm.ow2.io/) +- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons) +- ASM Core (from http://asm.ow2.io/, http://asm.ow2.org/) +- ASM Tree (from http://asm.ow2.io/) +- ASM Util (from http://asm.ow2.io/) +- ASM library repackaged as OSGi bundle (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) - Apache Commons Codec (from https://commons.apache.org/proper/commons-codec/) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) -- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) +- AspectJ Weaver (from http://www.aspectj.org, https://www.eclipse.org/aspectj/) +- GAX (Google Api eXtensions) (from https://github.com/googleapis, https://github.com/googleapis/gax-java) - Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) - Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials) -- Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials) -- Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http) - Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http) +- Hamcrest (from http://hamcrest.org/JavaHamcrest/) +- Hamcrest Core (from https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core) +- JSch (from http://www.jcraft.com/jsch/) - Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) -- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api) +- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest) +- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Mockito (from http://www.mockito.org) +- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito) - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) -- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) -- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) -- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) -- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) -- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) -- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util) +- Netty/Transport/Classes/KQueue (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-kqueue) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) +- PostgreSQL JDBC Driver +- Protocol Buffer Java API (from http://code.google.com/p/protobuf, https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) - Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util) -- Reflections (from http://github.com/ronmamo/reflections) -- SnakeYAML (from http://www.snakeyaml.org) -- Spring Core (from https://github.com/spring-projects/spring-framework) -- ThreeTen backport (from https://www.threeten.org/threetenbp) -- ThreeTen backport (from https://www.threeten.org/threetenbp) -- asm (from http://asm.ow2.io/) -- asm (from http://asm.ow2.io/) -- asm-all-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) -- asm-analysis (from http://asm.ow2.io/) -- asm-commons (from http://asm.ow2.io/) -- asm-tree (from http://asm.ow2.io/) -- asm-util (from http://asm.ow2.io/) -- classworlds (from http://classworlds.codehaus.org/) -- jaxen (from http://jaxen.codehaus.org/) +- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) +- Spring Core (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-core) +- ThreeTen backport (from https://github.com/ThreeTen/threetenbp, https://www.threeten.org/threetenbp) +- classworlds (from http://classworlds.codehaus.org/, https://repo1.maven.org/maven2/classworlds/classworlds) +- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -832,7 +744,6 @@ The following software have components provided under the terms of this license: - jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) - jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) - jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) -- jsch (from https://repo1.maven.org/maven2/com/jcraft/jsch) ======================================================================== CC-BY-2.5 @@ -840,7 +751,7 @@ CC-BY-2.5 The following software have components provided under the terms of this license: - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) -- MongoDB Java Driver Core (from http://www.mongodb.org) +- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org) ======================================================================== CC-BY-4.0 @@ -863,6 +774,7 @@ CDDL-1.0 ======================================================================== The following software have components provided under the terms of this license: +- ASM library repackaged as OSGi bundle (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) - Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) - HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) - HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) @@ -870,13 +782,12 @@ The following software have components provided under the terms of this license: - HK2 configuration module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-config) - HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) - HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) -- JavaBeans Activation Framework API jar (from ) +- JavaBeans Activation Framework API jar - Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) - ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) -- Servlet Specification 2.5 API (from http://jetty.mortbay.org) +- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) - aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) -- asm-all-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -892,14 +803,13 @@ CDDL-1.1 ======================================================================== The following software have components provided under the terms of this license: -- Expression Language 3.0 (from http://uel.java.net) -- Java Architecture for XML Binding (from http://jaxb.java.net/) -- Java Servlet API (from http://servlet-spec.java.net) -- JavaBeans Activation Framework (from ) +- Jakarta Servlet (from http://servlet-spec.java.net, https://projects.eclipse.org/projects/ee4j.servlet) +- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) +- JavaBeans Activation Framework - JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp) - JavaMail API (from https://repo1.maven.org/maven2/com/sun/mail/javax.mail) - OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) -- Servlet Specification 2.5 API (from http://jetty.mortbay.org) +- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) - javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250) - jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) @@ -911,54 +821,119 @@ CPL-1.0 The following software have components provided under the terms of this license: - JUnit (from http://junit.org) -- System Rules (from http://stefanbirkner.github.io/system-rules/) +- System Rules (from http://stefanbirkner.github.com/system-rules/) ======================================================================== EPL-1.0 ======================================================================== The following software have components provided under the terms of this license: -- AspectJ weaver (from http://www.aspectj.org) -- Logback Classic Module (from http://logback.qos.ch) +- AspectJ Weaver (from http://www.aspectj.org, https://www.eclipse.org/aspectj/) +- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) +- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) +- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) +- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) +- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) +- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) +- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) +- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- Logback Classic Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-classic) - Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic) - Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core) - Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson) -- Logback Core Module (from http://logback.qos.ch) +- Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- SnakeYAML (from http://www.snakeyaml.org) +- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) +- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) +- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) +- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) +- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common) +- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) +- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) +- jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core) -- jts-io-common (from ) +- jts-io-common (from https://repo1.maven.org/maven2/org/locationtech/jts/io/jts-io-common) + +======================================================================== +EPL-2.0 +======================================================================== +The following software have components provided under the terms of this license: + +- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) +- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) +- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) +- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) +- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) +- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) +- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) +- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) +- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) +- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) +- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) +- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common) +- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) +- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) +- jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) ======================================================================== GPL-2.0-only ======================================================================== The following software have components provided under the terms of this license: +- ASM library repackaged as OSGi bundle (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) - Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) -- Cobertura (from http://cobertura.sourceforge.net) - Cobertura Limited Runtime (from http://cobertura.sourceforge.net) -- Commons Lang (from http://commons.apache.org/lang/) -- Expression Language 3.0 (from http://uel.java.net) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) - HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) - HK2 config types (from https://repo1.maven.org/maven2/org/glassfish/hk2/config-types) - HK2 configuration module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-config) - HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) - HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) -- Java Architecture for XML Binding (from http://jaxb.java.net/) -- Java Servlet API (from http://servlet-spec.java.net) -- JavaBeans Activation Framework (from ) +- Jakarta Servlet (from http://servlet-spec.java.net, https://projects.eclipse.org/projects/ee4j.servlet) +- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) +- JavaBeans Activation Framework +- Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang) - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/) - OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) - Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) - ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) - aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) -- asm-all-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) - javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -977,7 +952,7 @@ GPL-2.0-or-later ======================================================================== The following software have components provided under the terms of this license: -- SnakeYAML (from http://www.snakeyaml.org) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) ======================================================================== GPL-2.0-with-classpath-exception @@ -985,18 +960,25 @@ GPL-2.0-with-classpath-exception The following software have components provided under the terms of this license: - Checker Qual (from https://checkerframework.org) -- Cobertura (from http://cobertura.sourceforge.net) -- Expression Language 3.0 (from http://uel.java.net) +- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) -- Java Architecture for XML Binding (from http://jaxb.java.net/) -- Java Servlet API (from http://servlet-spec.java.net) -- JavaBeans Activation Framework (from ) +- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge) +- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core) +- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2) +- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- Jakarta Servlet (from http://servlet-spec.java.net, https://projects.eclipse.org/projects/ee4j.servlet) +- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api) +- JavaBeans Activation Framework - OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) +- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel) - ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) - aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged) +- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject) - javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -1008,6 +990,7 @@ The following software have components provided under the terms of this license: - jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) - jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - tomcat-embed-core (from http://tomcat.apache.org/) ======================================================================== @@ -1015,8 +998,11 @@ GPL-3.0-only ======================================================================== The following software have components provided under the terms of this license: -- Project Lombok (from http://projectlombok.org) -- javax.ws.rs-api (from http://jax-rs-spec.java.net) +- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) +- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator) +- Project Lombok (from http://projectlombok.org, https://projectlombok.org) +- javax.ws.rs-api (from http://jax-rs-spec.java.net, https://github.com/eclipse-ee4j/jaxrs-api) ======================================================================== JSON @@ -1030,18 +1016,17 @@ LGPL-2.1-only ======================================================================== The following software have components provided under the terms of this license: -- Cobertura (from http://cobertura.sourceforge.net) -- Commons Lang (from http://commons.apache.org/lang/) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - Elastic JNA Distribution (from https://github.com/java-native-access/jna) -- Java Native Access (from https://github.com/java-native-access/jna) +- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) -- Logback Classic Module (from http://logback.qos.ch) +- Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang) +- Logback Classic Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-classic) - Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic) - Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core) - Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson) -- Logback Core Module (from http://logback.qos.ch) +- Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -1052,8 +1037,7 @@ LGPL-2.1-or-later The following software have components provided under the terms of this license: - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) -- SnakeYAML (from http://www.snakeyaml.org) +- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) ======================================================================== LGPL-3.0-only @@ -1062,7 +1046,7 @@ The following software have components provided under the terms of this license: - Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api) - Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) ======================================================================== MIT @@ -1073,20 +1057,19 @@ The following software have components provided under the terms of this license: - AWS Java SDK for AWS License Manager (from https://aws.amazon.com/sdkforjava) - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava) - Animal Sniffer Annotations (from https://repo1.maven.org/maven2/org/codehaus/mojo/animal-sniffer-annotations) -- Animal Sniffer Annotations (from https://repo1.maven.org/maven2/org/codehaus/mojo/animal-sniffer-annotations) - Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Java Client Runtime for ARM (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Spring Boot AutoConfigure (from https://github.com/Azure/azure-sdk-for-java) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Checker Qual (from https://checkerframework.org) -- Checker Qual (from https://checkerframework.org) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html) - Checker Qual (from https://checkerframework.org) +- ClassGraph (from https://github.com/classgraph/classgraph) +- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) - Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions) -- JOpt Simple (from http://pholser.github.com/jopt-simple) +- JOpt Simple (from http://jopt-simple.github.io/jopt-simple, http://pholser.github.io/jopt-simple) - JUL to SLF4J bridge (from http://www.slf4j.org) - Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) +- Java JWT (from http://www.jwt.io, https://github.com/auth0/java-jwt) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -1100,43 +1083,44 @@ The following software have components provided under the terms of this license: - Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for eventgrid (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) +- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for KeyVault Keys (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java) -- Mockito (from http://www.mockito.org) +- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito) - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) -- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) -- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) - Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) -- Plexus :: Default Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) - Plexus Default Interactivity Handler (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-interactivity-api) -- Project Lombok (from http://projectlombok.org) +- Project Lombok (from http://projectlombok.org, https://projectlombok.org) - SLF4J API Module (from http://www.slf4j.org) - Spongy Castle (from http://rtyley.github.io/spongycastle/) - Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos) +- Spring Security - Core (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-core, https://spring.io/projects/spring-security, https://spring.io/spring-security) - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) -- azure-documentdb (from http://azure.microsoft.com/en-us/services/documentdb/) -- azure-identity (from https://repo1.maven.org/maven2/com/azure/azure-identity) +- azure-documentdb (from http://azure.microsoft.com/en-us/services/documentdb/, https://azure.microsoft.com/en-us/services/cosmos-db/) - documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/) -- java jwt (from http://www.jwt.io) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) -- mockito-core (from https://github.com/mockito/mockito) -- mockito-core (from https://github.com/mockito/mockito) -- mockito-core (from https://github.com/mockito/mockito) +- mockito-junit-jupiter (from https://github.com/mockito/mockito) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java) -- spring-security-core (from http://spring.io/spring-security) ======================================================================== MPL-1.1 ======================================================================== The following software have components provided under the terms of this license: -- Cobertura (from http://cobertura.sourceforge.net) +- Cobertura code coverage (from http://cobertura.sourceforge.net) - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) ======================================================================== MPL-2.0 @@ -1144,16 +1128,8 @@ MPL-2.0 The following software have components provided under the terms of this license: - Javassist (from http://www.javassist.org/) -- Javassist (from http://www.javassist.org/) -- RabbitMQ Java Client (from http://www.rabbitmq.com) -- okhttp (from https://square.github.io/okhttp/) - -======================================================================== -MS-RL -======================================================================== -The following software have components provided under the terms of this license: - -- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) +- OkHttp (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp, https://square.github.io/okhttp/) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) ======================================================================== PHP-3.01 @@ -1161,14 +1137,14 @@ PHP-3.01 The following software have components provided under the terms of this license: - Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) -- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api) +- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec) ======================================================================== Public-Domain ======================================================================== The following software have components provided under the terms of this license: -- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) +- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) - Spongy Castle (from http://rtyley.github.io/spongycastle/) - jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common) - jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) @@ -1179,14 +1155,14 @@ SAX-PD ======================================================================== The following software have components provided under the terms of this license: -- Apache Ant Core (from https://ant.apache.org/) +- Apache Ant Core ======================================================================== SPL-1.0 ======================================================================== The following software have components provided under the terms of this license: -- Servlet Specification 2.5 API (from http://jetty.mortbay.org) +- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5) ======================================================================== SunPro @@ -1200,18 +1176,26 @@ W3C ======================================================================== The following software have components provided under the terms of this license: -- Apache Ant Core (from https://ant.apache.org/) +- Apache Ant Core - JTidy (from http://jtidy.sourceforge.net) -- Xerces2-j (from https://xerces.apache.org/xerces2-j/) -- jaxen (from http://jaxen.codehaus.org/) -- xml-apis (from ) +- Xerces2-j (from https://repo1.maven.org/maven2/xerces/xercesImpl, https://xerces.apache.org/xerces2-j/) +- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen) +- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) +- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) +- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) +- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) +- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering) +- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) +- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) +- xml-apis ======================================================================== WTFPL ======================================================================== The following software have components provided under the terms of this license: -- Reflections (from http://github.com/ronmamo/reflections) +- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections) ======================================================================== public-domain @@ -1219,50 +1203,53 @@ public-domain The following software have components provided under the terms of this license: - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava) +- Apache Groovy (from http://groovy-lang.org, http://groovy.codehaus.org/, https://groovy-lang.org) - Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client) -- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html) -- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) -- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) +- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from https://www.bouncycastle.org/java.html) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html) +- Guava: Google Core Libraries for Java (from http://code.google.com/p/guava-libraries, https://github.com/google/guava, https://repo1.maven.org/maven2/com/google/guava/guava) - HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/) - JTidy (from http://jtidy.sourceforge.net) -- Joda-Time (from https://www.joda.org/joda-time/) +- Joda-Time (from http://joda-time.sourceforge.net, http://www.joda.org/joda-time/, https://www.joda.org/joda-time/) - LatencyUtils (from http://latencyutils.github.io/LatencyUtils/) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) -- Project Lombok (from http://projectlombok.org) -- RabbitMQ Java Client (from http://www.rabbitmq.com) +- PostgreSQL JDBC Driver +- Project Lombok (from http://projectlombok.org, https://projectlombok.org) +- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com) - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) -- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) -- Spring Web (from https://github.com/spring-projects/spring-framework) -- azure-documentdb (from http://azure.microsoft.com/en-us/services/documentdb/) +- Spring Web (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-web) +- azure-documentdb (from http://azure.microsoft.com/en-us/services/documentdb/, https://azure.microsoft.com/en-us/services/cosmos-db/) +- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common) +- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - reactive-streams (from http://www.reactive-streams.org/) -- xml-apis (from ) +- xml-apis ======================================================================== unknown ======================================================================== The following software have components provided under the terms of this license: -- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html) -- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) -- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) +- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from https://www.bouncycastle.org/java.html) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html) - Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy) - Checker Qual (from https://checkerframework.org) -- Checker Qual (from https://checkerframework.org) - JSON in Java (from https://github.com/douglascrockford/JSON-java) - JTidy (from http://jtidy.sourceforge.net) +- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) +- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) +- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) - Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) -- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api) +- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec) - Spongy Castle (from http://rtyley.github.io/spongycastle/) -- System Rules (from http://stefanbirkner.github.io/system-rules/) -- jaxen (from http://jaxen.codehaus.org/) +- System Rules (from http://stefanbirkner.github.com/system-rules/) +- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen) - jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet) - jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core) - jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client) @@ -1271,8 +1258,7 @@ The following software have components provided under the terms of this license: - jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2) - jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) +- jersey-spring5 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring5) - jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core) -- jts-io-common (from ) -- xml-apis (from ) - - +- jts-io-common (from https://repo1.maven.org/maven2/org/locationtech/jts/io/jts-io-common) +- xml-apis diff --git a/README.md b/README.md index 8bfcb90b5e426329e082c09d3ef9aeb0ad8b23dc..cf670738a775199c38be2b6ecc223b1880948a40 100644 --- a/README.md +++ b/README.md @@ -9,4 +9,8 @@ running the `os-indexer` Azure implementation ## GCP Implementation -All documentation for the GCP implementation of `os-indexer` lives [here](./provider/indexer-gcp/README.md) \ No newline at end of file +All documentation for the GCP implementation of `os-indexer` lives [here](./provider/indexer-gcp/README.md) + +## AWS Implementation + +All documentation for the AWS implementation of `os-indexer` lives [here](./provider/indexer-aws/README.md) \ No newline at end of file diff --git a/devops/azure/chart/templates/virtual-service.yaml b/devops/azure/chart/templates/virtual-service.yaml index 8d3a2cc1b23edee0a353b22fa8e6d8d3f09e53bb..93a8d8af9b645f3db447b8c15cab1a7e149e3f22 100644 --- a/devops/azure/chart/templates/virtual-service.yaml +++ b/devops/azure/chart/templates/virtual-service.yaml @@ -1,5 +1,6 @@ --- # Source: /devops/azure/chart/templates/virtual-service.yaml +{{- if .Values.istioDnsHost }} apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: @@ -34,4 +35,5 @@ spec: - PATCH - DELETE allowOrigins: - - prefix: "*" \ No newline at end of file + - prefix: "*" +{{- end }} \ No newline at end of file diff --git a/devops/gcp/configmap/templates/indexer-variables.yml b/devops/gcp/configmap/templates/indexer-variables.yml index 3db35220adcd84c70c91f585bce3cd4f33963833..d9b96cdcea4089f64637a1634e3fab3730348adc 100644 --- a/devops/gcp/configmap/templates/indexer-variables.yml +++ b/devops/gcp/configmap/templates/indexer-variables.yml @@ -6,23 +6,18 @@ metadata: name: "{{ .Values.conf.configmap }}" namespace: "{{ .Release.Namespace }}" data: - KEY_RING: "{{ .Values.data.key_ring }}" - KMS_KEY: "{{ .Values.data.kms_key }}" LOG_LEVEL: "{{ .Values.data.log_level }}" SCHEMA_HOST: "{{ .Values.data.schema_host }}" AUTHORIZE_API: "{{ .Values.data.authorize_api }}" - GOOGLE_CLOUD_PROJECT: "{{ .Values.data.google_cloud_project }}" REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}" REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}" SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}" STORAGE_QUERY_RECORD_HOST: "{{ .Values.data.storage_query_record_host }}" STORAGE_SCHEMA_HOST: "{{ .Values.data.storage_schema_host }}" STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "{{ .Values.data.storage_query_record_for_conversion_host }}" - STORAGE_HOSTNAME: "{{ .Values.data.storage_hostname }}" STORAGE_RECORDS_BATCH_SIZE: "{{ .Values.data.storage_records_batch_size }}" STORAGE_QUERY_KINDS_HOST: "{{ .Values.data.storage_query_kinds_host }}" INDEXER_QUEUE_HOST: "{{ .Values.data.indexer_queue_host }}" - LEGALTAG_API: "{{ .Values.data.legaltag_api }}" - CRS_API: "{{ .Values.data.crs_api }}" PARTITION_API: "{{ .Values.data.partition_api }}" GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}" + SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}" diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml index a423c75f0818b2e638467c912ba3f83a4c509d37..21247d9775534523bd1a1390374a9eb7d2cf1a0f 100644 --- a/devops/gcp/configmap/values.yaml +++ b/devops/gcp/configmap/values.yaml @@ -3,26 +3,21 @@ # Declare variables to be passed into your templates. data: - key_ring: "csqp" - kms_key: "searchService" log_level: "INFO" schema_host: "" authorize_api: "http://entitlements/api/entitlements/v2/" - google_cloud_project: "" redis_search_host: "" redis_group_host: "" security_https_certificate_trust: "true" storage_query_record_host: "http://storage/api/storage/v2/query/records" storage_schema_host: "http://storage/api/storage/v2/schemas" storage_query_record_for_conversion_host: "http://storage/api/storage/v2/query/records:batch" - storage_hostname: "storage" storage_records_batch_size: "20" storage_query_kinds_host: "http://storage/api/storage/v2/query/kinds" indexer_queue_host: "" - legaltag_api: "http://legal/api/legal/v1" - crs_api: "" partition_api: "http://partition/api/partition/v1/" google_audiences: "" + spring_profiles_active: "gcp" conf: diff --git a/devops/gcp/deploy/templates/indexer-deploy.yml b/devops/gcp/deploy/templates/indexer-deploy.yml index 3c7a1fa1e3b592e90b829a7cc350ca95c5f5489e..fcbe1a8708b76eb20517324fe46cec9bf1eaa6b0 100644 --- a/devops/gcp/deploy/templates/indexer-deploy.yml +++ b/devops/gcp/deploy/templates/indexer-deploy.yml @@ -20,6 +20,7 @@ spec: containers: - name: "{{ .Values.conf.app_name }}" image: "{{ .Values.data.image }}" + imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" envFrom: - configMapRef: name: "{{ .Values.conf.configmap }}" diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml index bbe858fe8409edf4485099409d8908056cfc8a63..886588bcd32f711c53c1c27fbe4ff75bc303bd01 100644 --- a/devops/gcp/deploy/values.yaml +++ b/devops/gcp/deploy/values.yaml @@ -8,6 +8,7 @@ data: limits_cpu: "1" limits_memory: "1G" serviceAccountName: "" + imagePullPolicy: "IfNotPresent" image: "" conf: diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml index 7a4e8e9213d2cfafdd3de581a0d01ba39bae11dc..84e8a9e69da845786e9df5d6f2ac181c6d5975b6 100644 --- a/indexer-core/pom.xml +++ b/indexer-core/pom.xml @@ -4,21 +4,36 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <name>indexer-core</name> <description>Indexer Service Core</description> <packaging>jar</packaging> <properties> <commons-beanutils.version>1.9.4</commons-beanutils.version> - <osdu.oscorecommon.version>0.13.0-rc4</osdu.oscorecommon.version> + <osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version> + <spring-security-web.version>5.4.9</spring-security-web.version> + <gson.version>2.8.9</gson.version> + <netty.version>4.1.70.Final</netty.version> </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-bom</artifactId> + <version>${netty.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> <dependency> @@ -61,6 +76,17 @@ <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-security</artifactId> + <exclusions> + <exclusion> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-web</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-web</artifactId> + <version>${spring-security-web.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> @@ -111,7 +137,7 @@ <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> - <version>2.8.5</version> + <version>${gson.version}</version> </dependency> <dependency> <groupId>com.google.http-client</groupId> @@ -201,18 +227,6 @@ <scope>test</scope> </dependency> - <!-- swagger dependencies --> - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-swagger2</artifactId> - <version>${springfox-version}</version> - </dependency> - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-swagger-ui</artifactId> - <version>${springfox-version}</version> - </dependency> - </dependencies> <build> diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java index 1123e5ce21712f4ceac5a47cc78b322f4ecca5a9..cff663d6ca875c6c3bddbbd0c7ba68653ac77ac1 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java @@ -47,6 +47,7 @@ public class IndexerConfigurationProperties { private String storageQueryRecordForConversionHost; private String storageQueryRecordHost; private Integer storageRecordsBatchSize; + private Integer storageRecordsByKindBatchSize; private String storageSchemaHost; private String schemaHost; private String entitlementsHost; @@ -122,4 +123,12 @@ public class IndexerConfigurationProperties { public final Boolean isSmartSearchCcsDisabled() { return Boolean.TRUE.toString().equalsIgnoreCase(smartSearchCcsDisabled); } + + public Integer getStorageRecordsByKindBatchSize() { + if (this.storageRecordsByKindBatchSize != null) { + return this.storageRecordsByKindBatchSize; + } + // if property is not set, fall back to storageRecordsBatchSize property which is used by all CSPs to set batch size. + return this.storageRecordsBatchSize; + } } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/CronServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/CronServiceImpl.java index b6a61d47c0f41ec13cc9cbf3635a859d49baf4bb..1ec074481b13495868ca5c16608b213d7a8ccdfd 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/CronServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/CronServiceImpl.java @@ -1,5 +1,3 @@ - - // Copyright 2017-2019, Schlumberger // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +20,6 @@ import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.search.IndexInfo; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.springframework.beans.factory.annotation.Value; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java similarity index 87% rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java rename to indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java index efbda282b271ebef8814a293c4751b4ecc81953e..689e7a654ae9fcbb6acc0fa659a51f804151ad03 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java @@ -15,14 +15,17 @@ package org.opengroup.osdu.indexer.service; import org.elasticsearch.client.RestHighLevelClient; -import org.opengroup.osdu.core.common.search.IMappingService; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import java.io.IOException; import java.util.Map; import java.util.Set; -public interface IndexerMappingService extends IMappingService { +public interface IMappingService { + + String getIndexMapping(RestHighLevelClient client, String index) throws Exception; + + String getIndexSchema(String index) throws Exception; String createMapping(RestHighLevelClient client, IndexSchema schema, String index, boolean merge) throws IOException; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImpl.java index 1a02c456ded766a73da95636ed4e7ae8e6718768..23027e20654fcaae75e8aafd58cf51972565c4e7 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImpl.java @@ -30,7 +30,6 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.core.common.search.Preconditions; import org.opengroup.osdu.indexer.logging.AuditLogger; import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; @@ -55,7 +54,7 @@ public class IndexCopyServiceImpl implements IndexCopyService { @Inject private IElasticSettingService elasticSettingService; @Inject - private IndexerMappingService mappingService; + private IMappingService mappingService; @Inject private DpsHeaders headersInfo; @Inject diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java index 009247ba48e40da753c5c10a38b8a3d265e325e9..7787fba463d36e76566575bc9e949bbb3ea92727 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java @@ -30,7 +30,6 @@ import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; import org.opengroup.osdu.core.common.model.storage.Schema; import org.opengroup.osdu.core.common.model.storage.SchemaItem; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException; import org.opengroup.osdu.indexer.util.ElasticClientHandler; @@ -61,7 +60,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { @Inject private ElasticIndexNameResolver elasticIndexNameResolver; @Inject - private IndexerMappingService mappingService; + private IMappingService mappingService; @Inject private IndicesService indicesService; @Inject diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java index 1bfda7e9797a8ed1105d9006d2784432d9139a07..329ebd3245eef95724ac38a84c3cd15de49a89bc 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java @@ -36,7 +36,6 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; -import org.opengroup.osdu.core.common.search.IMappingService; import org.opengroup.osdu.core.common.search.Preconditions; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; @@ -49,7 +48,7 @@ import java.lang.reflect.Type; import java.util.*; @Service -public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService { +public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMappingService { @Inject private JaxRsDpsLog log; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java index c7d1301a000f39f6d6f6daf94cfce95323ee60df..2280acf76d8738cdf857014041c816eda76c5e99 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java @@ -41,7 +41,6 @@ import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.logging.AuditLogger; import org.opengroup.osdu.indexer.provider.interfaces.IPublisher; import org.opengroup.osdu.indexer.util.ElasticClientHandler; @@ -81,7 +80,7 @@ public class IndexerServiceImpl implements IndexerService { @Inject private IndicesService indicesService; @Inject - private IndexerMappingService mappingService; + private IMappingService mappingService; @Inject private IPublisher progressPublisher; @Inject diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesService.java new file mode 100644 index 0000000000000000000000000000000000000000..c09ff3e5744cb5ad52016499abdb8e55aff0b271 --- /dev/null +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesService.java @@ -0,0 +1,39 @@ +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.service; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.settings.Settings; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.search.IndexInfo; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface IndicesService { + + boolean createIndex(RestHighLevelClient client, String index, Settings settings, String type, Map<String, Object> mapping) throws ElasticsearchStatusException, IOException; + + boolean isIndexExist(RestHighLevelClient client, String index) throws IOException; + + boolean deleteIndex(RestHighLevelClient client, String index) throws ElasticsearchException, IOException, AppException; + + boolean deleteIndex(String index) throws ElasticsearchException, IOException, AppException; + + List<IndexInfo> getIndexInfo(RestHighLevelClient client, String indexPattern) throws IOException; +} diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java index bd3a90009edc62e105323b2586678083cc094267..50f58e9d9c38bc0ccd4e155904b868d8ae5d3cb9 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; import org.opengroup.osdu.core.common.model.search.IndexInfo; import org.opengroup.osdu.indexer.util.ElasticClientHandler; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java index 005e8c28a7844d4f14ef3ce8d41f1d60af111106..566de1d5bc7aff30d3c166bf69fa529834e48403 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java @@ -24,8 +24,6 @@ import org.elasticsearch.client.indices.GetMappingsResponse; import org.elasticsearch.common.unit.TimeValue; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; -import org.opengroup.osdu.core.common.search.IMappingService; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.core.common.search.Preconditions; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.springframework.beans.factory.annotation.Autowired; @@ -39,7 +37,7 @@ import java.util.Objects; @Service @RequestScope -public class MappingServiceImpl implements IMappingService { +public abstract class MappingServiceImpl implements IMappingService { @Autowired private IndicesService indicesService; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java index 70fa630e6ca33621db37c7b4f1a15344925fb3e1..af15d671f8f1cbc4ad1907daf99be0cf0e9a0297 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java @@ -88,7 +88,7 @@ public class ReindexServiceImpl implements ReindexService { // don't call reindex-worker endpoint if it's the last batch // previous storage query result size will be less then requested (limit param) - if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()) { + if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsByKindBatchSize()) { String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build()); this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers); return newPayLoad; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java index 950774d23c9ebeb55bc6f04bc0779bdcff04b712..95d88bc77e1b7f4fc85607fa47e6c28bcc1f4ba3 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.client.http.HttpMethods; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -43,7 +42,6 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; import java.io.UnsupportedEncodingException; -import java.lang.reflect.Type; import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -189,7 +187,7 @@ public class StorageServiceImpl implements StorageService { public RecordQueryResponse getRecordsByKind(RecordReindexRequest reindexRequest) throws URISyntaxException { Map<String, String> queryParams = new HashMap<>(); queryParams.put(RecordMetaAttribute.KIND.getValue(), reindexRequest.getKind()); - queryParams.put("limit", configurationProperties.getStorageRecordsBatchSize().toString()); + queryParams.put("limit", configurationProperties.getStorageRecordsByKindBatchSize().toString()); if (!Strings.isNullOrEmpty(reindexRequest.getCursor())) { queryParams.put("cursor", reindexRequest.getCursor()); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/HomeController.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/HomeController.java index eb1576dfb15d87cdfd9f7fc733a1546380c247f5..9638ba469d37d45fa83d27b614e4743f975daae6 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/HomeController.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/HomeController.java @@ -7,7 +7,6 @@ import org.springframework.web.bind.annotation.RequestMapping; public class HomeController { @RequestMapping(value = "/swagger") public String swagger() { - System.out.println("swagger-ui.html"); - return "redirect:swagger-ui.html"; + return "redirect:swagger-ui/index.html"; } } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SpringfoxSwaggerHostResolver.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SpringfoxSwaggerHostResolver.java new file mode 100644 index 0000000000000000000000000000000000000000..ba3701ff9f9710030c27805d69d8c6f825da9e01 --- /dev/null +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SpringfoxSwaggerHostResolver.java @@ -0,0 +1,35 @@ +package org.opengroup.osdu.indexer.swagger; + +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.servers.Server; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import springfox.documentation.oas.web.OpenApiTransformationContext; +import springfox.documentation.oas.web.WebMvcOpenApiTransformationFilter; +import springfox.documentation.spi.DocumentationType; + +import javax.servlet.http.HttpServletRequest; + +@Component +@Order(Ordered.LOWEST_PRECEDENCE) +public class SpringfoxSwaggerHostResolver implements WebMvcOpenApiTransformationFilter { + + @Override + public boolean supports(DocumentationType delimiter) { + return delimiter == DocumentationType.OAS_30; + } + + @Override + public OpenAPI transform(OpenApiTransformationContext<HttpServletRequest> context) { + OpenAPI swagger = context.getSpecification(); + + Server server = swagger.getServers().get(0); + if (server.getUrl().contains(":443")) { + // via the gateway + server.setUrl(server.getUrl().replace(":443", "")); + } + + return swagger; + } +} \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerDocumentationConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerDocumentationConfig.java index 5811f6bb8b7e7239c81de143d1329c04ed98864b..1581a366b7392e361ca159b87d9447fe2b61421c 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerDocumentationConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerDocumentationConfig.java @@ -3,67 +3,63 @@ package org.opengroup.osdu.indexer.swagger; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import springfox.documentation.builders.ParameterBuilder; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; -import springfox.documentation.schema.ModelRef; +import springfox.documentation.builders.RequestParameterBuilder; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.service.ApiKey; import springfox.documentation.service.AuthorizationScope; +import springfox.documentation.service.ParameterType; +import springfox.documentation.service.RequestParameter; import springfox.documentation.service.SecurityReference; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spi.service.contexts.SecurityContext; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; -import springfox.documentation.service.Parameter; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @Configuration -@EnableSwagger2 +@EnableOpenApi public class SwaggerDocumentationConfig { - public static final String AUTHORIZATION_HEADER = "Authorization"; - public static final String DEFAULT_INCLUDE_PATTERN = "/.*"; + public static final String AUTHORIZATION_HEADER = "Authorization"; + public static final String DEFAULT_INCLUDE_PATTERN = "/.*"; - @Bean - public Docket api() { - ParameterBuilder builder = new ParameterBuilder(); - List<Parameter> parameters = new ArrayList<>(); - builder.name(DpsHeaders.DATA_PARTITION_ID) - .description("data partition id") - .defaultValue("common") - .modelRef(new ModelRef("string")) - .parameterType("header") - .required(true) - .build(); - parameters.add(builder.build()); - return new Docket(DocumentationType.SWAGGER_2) - .globalOperationParameters(parameters) - .select() - .apis(RequestHandlerSelectors.basePackage("org.opengroup.osdu.indexer.api")) - .build() - .securityContexts(Collections.singletonList(securityContext())) - .securitySchemes(Collections.singletonList(apiKey())); - } + @Bean + public Docket api() { + RequestParameterBuilder builder = new RequestParameterBuilder(); + List<RequestParameter> parameters = new ArrayList<>(); + builder.name(DpsHeaders.DATA_PARTITION_ID) + .description("data partition id") + .in(ParameterType.HEADER) + .required(true) + .build(); + parameters.add(builder.build()); + return new Docket(DocumentationType.OAS_30) + .globalRequestParameters(parameters) + .select() + .apis(RequestHandlerSelectors.basePackage("org.opengroup.osdu.indexer.api")) + .build() + .securityContexts(Collections.singletonList(securityContext())) + .securitySchemes(Collections.singletonList(apiKey())); + } - private ApiKey apiKey() { - return new ApiKey("JWT", AUTHORIZATION_HEADER, "header"); - } - - private SecurityContext securityContext() { - return SecurityContext.builder() - .securityReferences(defaultAuth()) - .forPaths(PathSelectors.regex(DEFAULT_INCLUDE_PATTERN)) - .build(); - } + private ApiKey apiKey() { + return new ApiKey(AUTHORIZATION_HEADER, AUTHORIZATION_HEADER, "header"); + } + private SecurityContext securityContext() { + return SecurityContext.builder() + .securityReferences(defaultAuth()) + .operationSelector(o -> PathSelectors.regex(DEFAULT_INCLUDE_PATTERN).test(o.requestMappingPattern())) + .build(); + } List<SecurityReference> defaultAuth() { AuthorizationScope authorizationScope = new AuthorizationScope("global", "accessEverything"); AuthorizationScope[] authorizationScopes = new AuthorizationScope[]{authorizationScope}; return Collections.singletonList( - new SecurityReference("JWT", authorizationScopes)); + new SecurityReference(AUTHORIZATION_HEADER, authorizationScopes)); } } diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java index a7dca6b6897f5900a9fd6200146a7afc52774ec0..b52abb7c8598bb3a7e8ed3a1c064cf8c445ee1b1 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java @@ -28,7 +28,6 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; -import org.opengroup.osdu.core.common.search.IMappingService; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; import org.powermock.api.mockito.PowerMockito; diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java index 705f8e98ad4e733377abc03948fbe0fb22227c5c..8b1a7e0a28b1524b8056986bfc664efc67c22a2d 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java @@ -28,7 +28,6 @@ import org.opengroup.osdu.core.common.model.http.RequestStatus; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.indexer.OperationType; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException; import org.opengroup.osdu.indexer.util.ElasticClientHandler; @@ -75,7 +74,7 @@ public class IndexerSchemaServiceTest { @Mock private ElasticIndexNameResolver elasticIndexNameResolver; @Mock - private IndexerMappingService mappingService; + private IMappingService mappingService; @Mock private IndicesService indicesService; @Mock diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java index c3c3d3a7a09e86a35c96fe224397aabf60ccfcf1..026ff32cb2e04a3da21b765baefc282d056915bf 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java @@ -13,7 +13,6 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.opengroup.osdu.core.common.model.indexer.RecordInfo; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.springframework.test.context.junit4.SpringRunner; diff --git a/pom.xml b/pom.xml index 7e269cedec18e5a7b8ccf95d5be9fbbfdb0ab456..157543a4672741449bc5b59340a52a8423a863ed 100644 --- a/pom.xml +++ b/pom.xml @@ -2,24 +2,18 @@ <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-parent</artifactId> - <version>2.1.18.RELEASE</version> - <relativePath /> - </parent> - <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> <packaging>pom</packaging> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <description>Indexer Service</description> <properties> <java.version>1.8</java.version> - <springfox-version>2.7.0</springfox-version> + <maven.compiler.target>1.8</maven.compiler.target> + <maven.compiler.source>1.8</maven.compiler.source> <spring-cloud.version>Greenwich.SR2</spring-cloud.version> - <os-core-common.version>0.13.0-rc4</os-core-common.version> + <os-core-common.version>0.13.0</os-core-common.version> <snakeyaml.version>1.26</snakeyaml.version> <hibernate-validator.version>6.1.5.Final</hibernate-validator.version> <jackson.version>2.11.4</jackson.version> @@ -29,15 +23,16 @@ <netty.version>4.1.51.Final</netty.version> <reactor-netty.version>0.8.20.RELEASE</reactor-netty.version> <woodstox-core.version>6.2.3</woodstox-core.version> - <log4j2.version>2.17.0</log4j2.version> - <!-- <maven.compiler.target>1.8</maven.compiler.target>--> -<!-- <maven.compiler.source>1.8</maven.compiler.source>--> + <log4j2.version>2.17.1</log4j2.version> + <spring-boot.version>2.1.18.RELEASE</spring-boot.version> + <json-smart.version>2.4.7</json-smart.version> <!-- <maven.war.plugin>2.6</maven.war.plugin>--> <!-- <appengine.maven.plugin>1.0.0</appengine.maven.plugin>--> <!-- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>--> <!-- <failOnMissingWebXml>false</failOnMissingWebXml>--> <!-- <spring-cloud.version>Greenwich.SR2</spring-cloud.version>--> <!-- <springfox-version>2.7.0</springfox-version>--> + <springfox.version>3.0.0</springfox.version> </properties> <licenses> @@ -50,6 +45,13 @@ <dependencyManagement> <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>${spring-boot.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-to-slf4j</artifactId> @@ -127,17 +129,54 @@ <artifactId>woodstox-core</artifactId> <version>${woodstox-core.version}</version> </dependency> + <dependency> + <groupId>net.minidev</groupId> + <artifactId>json-smart</artifactId> + <version>${json-smart.version}</version> + </dependency> </dependencies> </dependencyManagement> - <dependencies> - <dependency> - <groupId>org.projectlombok</groupId> - <artifactId>lombok</artifactId> - <version>1.18.16</version> - <scope>provided</scope> - </dependency> - </dependencies> + <dependencies> + <dependency> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + <version>1.18.16</version> + <scope>provided</scope> + </dependency> + <!--Swagger --> + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-boot-starter</artifactId> + <version>${springfox.version}</version> + <exclusions> + <exclusion> + <groupId>org.springframework.plugin</groupId> + <artifactId>spring-plugin-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.springframework.plugin</groupId> + <artifactId>spring-plugin-metadata</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework.plugin</groupId> + <artifactId>spring-plugin-core</artifactId> + <version>2.0.0.RELEASE</version> + </dependency> + <dependency> + <groupId>org.springframework.plugin</groupId> + <artifactId>spring-plugin-metadata</artifactId> + <version>2.0.0.RELEASE</version> + <exclusions> + <exclusion> + <groupId>org.springframework.plugin</groupId> + <artifactId>spring-plugin-core</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> <modules> <module>indexer-core</module> diff --git a/provider/indexer-aws/README.md b/provider/indexer-aws/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3c2ad5d3ac00b155f30c9cf2fcd2bc7b77b2d9b9 --- /dev/null +++ b/provider/indexer-aws/README.md @@ -0,0 +1,202 @@ +# Indexer Service +indexer-aws is a [Spring Boot](https://spring.io/projects/spring-boot) service that provides a set of APIs to index storage records against Elasticsearch. It's not user-facing, all APIs are used internally by the platform. + +## Running Locally + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. + +### Prerequisites +Pre-requisites + +* JDK 8 (https://docs.aws.amazon.com/corretto/latest/corretto-8-ug/downloads-list.html) +* Maven 3.8.3 or later +* Lombok 1.16 or later +* OSDU Instance deployed on AWS + +### Service Configuration +In order to run the service locally or remotely, you will need to have the following environment variables defined. + +| name | example value | required | description | sensitive? | +| --- | --- | --- | --- | --- | +| `LOCAL_MODE` | `true` | yes | Set to 'true' to use env vars in place of the k8s variable resolver | no | +| `APPLICATION_PORT` | `8080` | yes | The port the service will be hosted on. | no | +| `AWS_REGION` | `us-east-1` | yes | The region where resources needed by the service are deployed | no | +| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | yes | The AWS Access Key for a user with access to Backend Resources required by the service | yes | +| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | yes | The AWS Secret Key for a user with access to Backend Resources required by the service | yes | +| `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxxx` | no | AWS Session token needed if using an SSO user session to authenticate | yes | +| `ENVIRONMENT` | `osdu-prefix` | yes | The Resource Prefix defined during deployment | no | +| `LOG_LEVEL` | `DEBUG` | yes | The Log Level severity to use (https://www.tutorialspoint.com/log4j/log4j_logging_levels.htm) | no | +| `SSM_ENABLED` | `true` | yes | Set to 'true' to use SSM to resolve config properties, otherwise use env vars | no | +| `SSL_ENABLED` | `false` | no | Set to 'false' to disable SSL for local development | no | +| `ENTITLEMENTS_BASE_URL` | `http://localhost:8081` or `https://some-hosted-url` | yes | Specify the base url for an entitlements service instance. Can be run locally or remote | no | +| `PARTITION_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no | +| `STORAGE_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no | +| `SCHEMA_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no | +| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - | +| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - | + + +### Run Locally +Check that maven is installed: + +example: +```bash +$ mvn --version +Apache Maven 3.8.3 (ff8e977a158738155dc465c6a97ffaf31982d739) +Maven home: /usr/local/Cellar/maven/3.8.3/libexec +Java version: 1.8.0_312, vendor: Amazon.com Inc., runtime: /Library/Java/JavaVirtualMachines/amazon-corretto-8.jdk/Contents/Home/jre +... +``` + +You may need to configure access to the remote maven repository that holds the OSDU dependencies. Copy one of the below files' content to your .m2 folder +* For development against the OSDU GitLab environment, leverage: `<REPO_ROOT>~/.mvn/community-maven.settings.xml` +* For development in an AWS Environment, leverage: `<REPO_ROOT>/provider/indexer-aws/maven/settings.xml` + +* Navigate to the service's root folder and run: + +```bash +mvn clean package -pl indexer-core,provider/indexer-aws +``` + +* If you wish to build the project without running tests + +```bash +mvn clean package -pl indexer-core,provider/indexer-aws -DskipTests +``` + +After configuring your environment as specified above, you can follow these steps to run the application. These steps should be invoked from the *repository root.* +<br/> +<br/> +NOTE: If not on osx/linux: Replace `*` with version numbers as defined in the provider/indexer-aws/pom.xml file + +```bash +java -jar provider/indexer-aws/target/indexer-aws-*.*.*-SNAPSHOT-spring-boot.jar +``` + +## Running Elasticsearch locally +For indexer to index anything, it needs to have access to an Elasticsearch cluster. The easiest way to do this is to spin one up locally. +You can spin one up locally using Docker or Kubernetes Helm. What's detailed below is simply downloading executable and running directly. + +Instructions copied from here for longevity: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/getting-started-install.html + +1. Download a distribution from here: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.20.tar.gz + +2. Extract the archive +Linux and macOS: `tar -xvf elasticsearch-6.8.20.tar.gz` +Windows PowerShell: `Expand-Archive elasticsearch-6.8.20-windows-x86_64.zip` + +3. Start Elasticsearch from the bin directory: +Linux and macOS: + ``` + cd elasticsearch-6.8.20/bin + ./elasticsearch + ``` + + Windows: + + ``` + cd %PROGRAMFILES%\Elastic\Elasticsearch\bin + .\elasticsearch.exe + ``` + +You should see in the logs that pop up what url and port it runs on. By default you should see http://localhost with port 9300 + +## Testing + + ### Running Integration Tests + This section describes how to run OSDU Integration tests (testing/indexer-test-aws). + + IMPORTANT: You need to setup indexer queue locally first using README in indexer-queue-aws. + + You will need to have the following environment variables defined. + export AWS_COGNITO_AUTH_FLOW=USER_PASSWORD_AUTH + export AWS_COGNITO_AUTH_PARAMS_PASSWORD=$ADMIN_PASSWORD + export AWS_COGNITO_AUTH_PARAMS_USER=$ADMIN_USER + export DEFAULT_DATA_PARTITION_ID_TENANT1=opendes + export DEFAULT_DATA_PARTITION_ID_TENANT2=common + export ENTITLEMENTS_DOMAIN=example.com + export OTHER_RELEVANT_DATA_COUNTRIES=US + export STORAGE_HOST=$STORAGE_URL + export HOST=$SCHEMA_URL + export ELASTIC_HOST=$ELASTIC_HOST + export ELASTIC_PORT=$ELASTIC_PORT + export ELASTIC_PASSWORD=$ELASTIC_PASSWORD + export ELASTIC_USER_NAME=$ELASTIC_USERNAME + + | name | example value | description | sensitive? + | --- | --- | --- | --- | + | `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes | + | `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes | + | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes | + | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no | + | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no | + | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no | + | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no | + | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition| no | + | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no | + | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no | + | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes | + | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no | + | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no | + | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no | + | `HOST` | `http://localhost:8080` | Base url for deployment | no | + | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no | + | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no | + | `ELASTIC_PASSWORD` | `xxxxxxxxxxxxxxx` | Password for user to access elasticsearch | yes | + | `ELASTIC_USER_NAME` | `xxxxxxxxxxxxxxxx` | Username for user to access elasticsearch | yes | + + + **Creating a new user to use for integration tests** + ``` + aws cognito-idp admin-create-user --user-pool-id ${AWS_COGNITO_USER_POOL_ID} --username ${AWS_COGNITO_AUTH_PARAMS_USER} --user-attributes Name=email,Value=${AWS_COGNITO_AUTH_PARAMS_USER} Name=email_verified,Value=True --message-action SUPPRESS + + aws cognito-idp initiate-auth --auth-flow ${AWS_COGNITO_AUTH_FLOW} --client-id ${AWS_COGNITO_CLIENT_ID} --auth-parameters USERNAME=${AWS_COGNITO_AUTH_PARAMS_USER},PASSWORD=${AWS_COGNITO_AUTH_PARAMS_PASSWORD} + ``` + + **Entitlements group configuration for integration accounts** + <br/> + In order to add user entitlements, run entitlements bootstrap scripts in the entitlements project + + | AWS_COGNITO_AUTH_PARAMS_USER | + | --- | + | service.indexer.admin | + | service.legal.admin | + | service.storage.admin | + + Execute following command to build code and run all the integration tests: + +### Run Tests simulating Pipeline + +* Prior to running tests, scripts must be executed locally to generate pipeline env vars + +```bash +testing/indexer-test-aws/build-aws/prepare-dist.sh + +#Set Neccessary ENV Vars here as defined in run-tests.sh + +dist/testing/integration/build-aws/run-tests.sh +``` + +### Run Tests using mvn +Set required env vars and execute the following: +``` +mvn clean package -f testing/pom.xml -pl indexer-test-core,indexer-test-aws -DskipTests +mvn test -f testing/indexer-test-aws/pom.xml +``` + + + +## License +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml index 983cb5530daf93e03e969306cf1c6e295979419d..1a8ba1dba23253591437777f227794591c9b5fb0 100644 --- a/provider/indexer-aws/pom.xml +++ b/provider/indexer-aws/pom.xml @@ -18,7 +18,7 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> @@ -26,13 +26,34 @@ <artifactId>indexer-aws</artifactId> <description>Indexer service on AWS</description> <packaging>jar</packaging> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <properties> <aws.version>1.11.1018</aws.version> <deployment.environment>dev</deployment.environment> + <netty.version>4.1.51.Final</netty.version> + <jackson.version>2.11.4</jackson.version> </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-bom</artifactId> + <version>${netty.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson</groupId> + <artifactId>jackson-bom</artifactId> + <version>${jackson.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> <!-- Internal packages --> <dependency> @@ -42,12 +63,12 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <dependency> <groupId>org.opengroup.osdu.core.aws</groupId> <artifactId>os-core-lib-aws</artifactId> - <version>0.13.0-rc3</version> + <version>0.13.0</version> </dependency> <!-- AWS managed packages --> diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md index bc6a63ec3e33284229283fc5ba59caab611971a6..9f120e4504136755a842219c45d2b80138e6d007 100644 --- a/provider/indexer-azure/README.md +++ b/provider/indexer-azure/README.md @@ -47,6 +47,8 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA | `STORAGE_SCHEMA_HOST` | `${storage_service_url}/schemas` | Endpoint of schema API | no | - | | `STORAGE_QUERY_RECORD_HOST` | `${storage_service_url}/query/records` | Endpoint of records API | no | - | | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | `${storage_service_url}/query/records:batch` | Endpoint of records batch API | no | - | +| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - | +| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | 100 | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - | | `KEYVAULT_URI` | ex `https://foo-kv.vault.azure.net/` | . | . | . | | `appinsights_key` | `********` | App Insights key | yes | output of infrastructure deployments | | `aad_client_id` | `********` | AAD client application ID | yes | output of infrastructure deployment | diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml index a90cd035b1033dca96a2ff9b82cc8be5a85ccf2e..fa94c668f3d2f2ccb599785d8fbf347b540bb865 100644 --- a/provider/indexer-azure/pom.xml +++ b/provider/indexer-azure/pom.xml @@ -21,12 +21,12 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>indexer-azure</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <name>indexer-azure</name> <description>Indexer Service Azure</description> <packaging>jar</packaging> @@ -37,16 +37,16 @@ <azure.appservice.plan /> <azure.appservice.appname /> <azure.appservice.subscription /> - <log4j.version>2.17.0</log4j.version> + <log4j.version>2.17.1</log4j.version> <nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version> - <indexer-core.version>0.13.0-SNAPSHOT</indexer-core.version> + <indexer-core.version>0.14.0-SNAPSHOT</indexer-core.version> <spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version> <osdu.corelibazure.version>0.13.0-rc6</osdu.corelibazure.version> - <osdu.oscorecommon.version>0.13.0-rc4</osdu.oscorecommon.version> + <osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version> <reactor-netty.version>0.9.12.RELEASE</reactor-netty.version> <java-jwt.version>3.8.1</java-jwt.version> <powermock.version>2.0.2</powermock.version> - <mockito.version>3.0.0</mockito.version> + <mockito.version>3.11.2</mockito.version> <cobertura-maven-plugin.version>2.7</cobertura-maven-plugin.version> <spring-security-oauth2.version>2.3.6.RELEASE</spring-security-oauth2.version> <spring-boot.version>2.1.18.RELEASE</spring-boot.version> @@ -59,18 +59,6 @@ <dependencyManagement> <dependencies> <!-- Override version defined in parent's POM --> - <dependency> - <groupId>io.projectreactor.netty</groupId> - <artifactId>reactor-netty</artifactId> - <version>${reactor-netty.version}</version> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-dependencies</artifactId> - <version>${spring-boot.version}</version> - <type>pom</type> - <scope>import</scope> - </dependency> <!-- Inherit managed dependencies from core-lib-azure --> <dependency> <groupId>org.opengroup.osdu</groupId> @@ -79,11 +67,6 @@ <type>pom</type> <scope>import</scope> </dependency> - <dependency> - <groupId>org.opengroup.osdu</groupId> - <artifactId>os-core-common</artifactId> - <version>${osdu.oscorecommon.version}</version> - </dependency> </dependencies> </dependencyManagement> @@ -249,6 +232,20 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.junit</groupId> + <artifactId>junit</artifactId> + </exclusion> + <exclusion> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.springframework.security</groupId> diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java index ce59f2e5c65b6aa3aea5361b7bb1a11c2cd1f690..fd1150ce646395d0c0029a812fd643f056fbe2ef 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java @@ -41,11 +41,13 @@ public class AADSecurityConfig extends WebSecurityConfigurerAdapter { "/index-worker", "/_dps/task-handlers", "/_dps/task-handlers/**", "/reindex", "/v2/api-docs", + "/v3/api-docs", "/swagger-resources/**", "/configuration/security", "/swagger", "/info", "/swagger-ui.html", + "/swagger-ui/**", "/webjars/**").permitAll() .anyRequest().authenticated() .and().addFilterBefore(appRoleAuthFilter, UsernamePasswordAuthenticationFilter.class); diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java index f36df01e3d05572696c0590442eb03bd57fba33e..6ee0289c6e1526316a71f3b94f372dcb6ed0c1bc 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java @@ -128,7 +128,7 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder { String recordChangedMessagePayload = gson.toJson(recordChangedMessages); createTask(recordChangedMessagePayload, headers); } - } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()); + } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsByKindBatchSize()); } catch (AppException e) { throw e; diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties index 6f65e89ecc3c8f14aa59007ff610e2bd5c7aa0b0..8f479afcd82f6acbe812168eae39a0980b076310 100644 --- a/provider/indexer-azure/src/main/resources/application.properties +++ b/provider/indexer-azure/src/main/resources/application.properties @@ -46,6 +46,8 @@ STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records STORAGE_QUERY_KINDS_HOST=${storage_service_url}/query/kinds STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 +STORAGE_RECORDS_BY_KIND_BATCH_SIZE=100 + INDEXER_QUEUE_HOST=http://127.0.0.1:9000 diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/CronServiceImplTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/CronServiceImplTest.java index 2ec6cfdf20306468e69d78d70450c9def671bfe2..4e14ed7038f7073a798c5c89a7addfd50c20ea81 100644 --- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/CronServiceImplTest.java +++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/CronServiceImplTest.java @@ -28,7 +28,7 @@ import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.service.CronServiceImpl; import org.opengroup.osdu.core.common.model.search.IndexInfo; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.service.IndicesService; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexCopyServiceImplTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexCopyServiceImplTest.java index e03a77601ca1297aa4cf73cd6022da094ef5c6e2..6fa376a3f6d5a14fe151f2c881202809c220f700 100644 --- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexCopyServiceImplTest.java +++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexCopyServiceImplTest.java @@ -32,13 +32,13 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.indexer.logging.AuditLogger; +import org.opengroup.osdu.indexer.service.IMappingService; import org.opengroup.osdu.indexer.service.IndexCopyServiceImpl; -import org.opengroup.osdu.indexer.service.IndexerMappingService; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.indexer.service.IndicesService; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -74,7 +74,7 @@ public class IndexCopyServiceImplTest { @Mock private IndicesService indicesService; @Mock - private IndexerMappingService mappingService; + private IMappingService mappingService; @Mock private ElasticClientHandler elasticClientHandler; @Mock diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md index 842139cfd3e3ec77b6ecf15559f63beb6f7c290c..c4ea917bc3ffe5d2887485c75dd0bd0b3542bed3 100644 --- a/provider/indexer-gcp/README.md +++ b/provider/indexer-gcp/README.md @@ -1,42 +1,49 @@ # Indexer Service os-indexer-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU R2 domain searches against Elasticsearch. +## Table of Contents <a name="TOC"></a> +* [Getting started](#Getting-started) +* [Mappers](#Mappers) +* [Settings and Configuration](#Settings-and-Configuration) +* [Run service](#Run-service) +* [Testing](#Testing) +* [Deployment](#Deployment) +* [Entitlements groups](#Entitlements-groups) +* [Licence](#License) + ## Getting Started These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. -### Prerequisites -Pre-requisites +## Mappers -* GCloud SDK with java (latest version) -* JDK 8 -* Lombok 1.16 or later -* Maven +This is a universal solution created using EPAM OQM mappers technology. It allows you to work with various +implementations of message brokers. -### Installation -In order to run the service locally or remotely, you will need to have the following environment variables defined. +For more information about mappers: +- [OQM Readme](https://community.opengroup.org/osdu/platform/system/lib/cloud/gcp/oqm/-/blob/master/README.md) -| name | value | description | sensitive? | source | -| --- | --- | --- | --- | --- | -| `LOG_PREFIX` | `service` | Logging prefix | no | - | -| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - | -| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | -| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment | -| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment | -| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances | -| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment | -| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | -| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | -| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | -| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | -| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | -| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | -| `GOOGLE_CLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment | -| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | -| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | -| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | -| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - | -| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment | -| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment | +### Limitations of the current version + +In the current version, the mappers are equipped with several drivers to the stores and the message broker: + +- OQM (mapper to message brokers): Google PubSub; RabbitMQ + +## Settings and Configuration + +### Prerequisites +1. Mandatory + - JDK 8 + - Lombok 1.16 or later + - Maven +2. For Google Cloud only + - GCloud SDK with java (latest version) + +### Anthos Service Configuration: +[Anthos service configuration ](docs/anthos/README.md) +### GCP Service Configuration: +[Gcp service configuration ](docs/gcp/README.md) + +## Run service ### Run Locally Check that maven is installed: @@ -181,26 +188,6 @@ $ (cd testing/indexer-test-gcp/ && mvn clean test) or * Google Documentation: https://cloud.google.com/cloud-build/docs/deploying-builds/deploy-appengine -#### Cloud KMS Setup - -Enable cloud KMS on master project - -Create king ring and key in the ***master project*** - -```bash - gcloud services enable cloudkms.googleapis.com - export KEYRING_NAME="csqp" - export CRYPTOKEY_NAME="searchService" - gcloud kms keyrings create $KEYRING_NAME --location global - gcloud kms keys create $CRYPTOKEY_NAME --location global \ - --keyring $KEYRING_NAME \ - --purpose encryption -``` - -Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab - -Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab - #### Memory Store (Redis Instance) Setup Create a new Standard tier Redis instance on the ***service project*** @@ -211,6 +198,14 @@ The Redis instance must be created under the same region with the App Engine app gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD ``` +## Entitlements groups +Storage service account should have entitlements groups listed below: +- service.entitlements.user +- users +- service.storage.viewer +- service.schema-service.viewers +- data.default.viewers + ## Licence Copyright © Google LLC Copyright © EPAM Systems diff --git a/provider/indexer-gcp/docs/anthos/README.md b/provider/indexer-gcp/docs/anthos/README.md new file mode 100644 index 0000000000000000000000000000000000000000..90cb1a8b9ed4e39f7e9f80be0ad42524b77d7ace --- /dev/null +++ b/provider/indexer-gcp/docs/anthos/README.md @@ -0,0 +1,188 @@ +## Service Configuration for Anthos + +## Environment variables: + +Define the following environment variables. + +Must have: + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for GCP environment | false | - | +| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` |yes| - | +| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens| yes | - | +| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - | + +Defined in default application property file but possible to override: + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `LOG_PREFIX` | `service` | Logging prefix | no | - | +| `LOG_LEVEL` | `****` | Logging level | no | - | +| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | +| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment | +| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment | +| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment | +| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment | +| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment | + +These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested. +Usage of spring profiles is preferred. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - | +| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - | +| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - | + +## Elasticsearch configuration + +### Properties set in Partition service: + +**prefix:** `elasticsearch` + +It can be overridden by: + +- through the Spring Boot property `elastic-search-properties-prefix` +- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX` + +**Propertyset:** + +| Property | Description | +| --- | --- | +| elasticsearch.host | server URL | +| elasticsearch.port | server port | +| elasticsearch.configuration | username and password | + +<details><summary>Example of a definition for a single tenant</summary></details> + +``` + +curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{ + "properties": { + "elasticsearch.host": { + "sensitive": false, + "value": "elastic.us-central1.gcp.cloud.es.io" + }, + "elasticsearch.port": { + "sensitive": false, + "value": "9243" + }, + "elasticsearch.configuration": { + "sensitive": true, + "value": "elasticuser:elasticpassword" + } + } +}' + +``` + +### For OQM RabbitMQ + +**prefix:** `oqm.rabbitmq` +It can be overridden by: + +- through the Spring Boot property `oqm.rabbitmq.partition-properties-prefix` +- environment variable `OQM_RABBITMQ_PARTITION_PROPERTIES_PREFIX`` + +**Propertyset** (for two types of connection: messaging and admin operations): + +| Property | Description | +| --- | --- | +| oqm.rabbitmq.amqp.host | messaging hostnameorIP | +| oqm.rabbitmq.amqp.port | - port | +| oqm.rabbitmq.amqp.path | - path | +| oqm.rabbitmq.amqp.username | - username | +| oqm.rabbitmq.amqp.password | - password | +| oqm.rabbitmq.admin.schema | admin host schema | +| oqm.rabbitmq.admin.host | - host name | +| oqm.rabbitmq.admin.port | - port | +| oqm.rabbitmq.admin.path | - path | +| oqm.rabbitmq.admin.username | - username | +| oqm.rabbitmq.admin.password | - password | + +<details><summary>Example of a single tenant definition</summary> + +``` + +curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{ + "properties": { + "oqm.rabbitmq.amqp.host": { + "sensitive": false, + "value": "localhost" + }, + "oqm.rabbitmq.amqp.port": { + "sensitive": false, + "value": "5672" + }, + "oqm.rabbitmq.amqp.path": { + "sensitive": false, + "value": "" + }, + "oqm.rabbitmq.amqp.username": { + "sensitive": false, + "value": "guest" + }, + "oqm.rabbitmq.amqp.password": { + "sensitive": true, + "value": "guest" + }, + + "oqm.rabbitmq.admin.schema": { + "sensitive": false, + "value": "http" + }, + "oqm.rabbitmq.admin.host": { + "sensitive": false, + "value": "localhost" + }, + "oqm.rabbitmq.admin.port": { + "sensitive": false, + "value": "9002" + }, + "oqm.rabbitmq.admin.path": { + "sensitive": false, + "value": "/api" + }, + "oqm.rabbitmq.admin.username": { + "sensitive": false, + "value": "guest" + }, + "oqm.rabbitmq.admin.password": { + "sensitive": true, + "value": "guest" + } + } +}' + +``` + +</details> + +#### Exchanges and queues configuration + +At RabbitMq should be created exchange with name: + +**name:** `indexing-progress` + + + +## Keycloak configuration + +[Keycloak service accounts setup](https://www.keycloak.org/docs/latest/server_admin/#_service_accounts) + +Configure Clients. One Client per OSDU service. Set them “confidential”. + + + +Each Client has embedded Service Account (SA) option. Enable SAs for Clients, make “Authorization enabled”: + + + +Add `partition-and-entitlements` scope to `Default Client Scopes` and generate Keys. + +Give `client-id` and `client-secret` to services, which should be authorized within the platform. \ No newline at end of file diff --git a/provider/indexer-gcp/docs/anthos/pics/client.png b/provider/indexer-gcp/docs/anthos/pics/client.png new file mode 100644 index 0000000000000000000000000000000000000000..8a2014e672b65ddf644558e2c9e8cb8823f549dd Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/client.png differ diff --git a/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG b/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG new file mode 100644 index 0000000000000000000000000000000000000000..e1c67734b91a536659fdf0b940a93208d749e3c8 Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG differ diff --git a/provider/indexer-gcp/docs/anthos/pics/sa.png b/provider/indexer-gcp/docs/anthos/pics/sa.png new file mode 100644 index 0000000000000000000000000000000000000000..b9c8124047b5834a2cf79baecbf751dc106feeff Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/sa.png differ diff --git a/provider/indexer-gcp/docs/gcp/README.md b/provider/indexer-gcp/docs/gcp/README.md new file mode 100644 index 0000000000000000000000000000000000000000..494169b1634af2e093c39038688987e78de6d726 --- /dev/null +++ b/provider/indexer-gcp/docs/gcp/README.md @@ -0,0 +1,94 @@ +## Service Configuration for GCP + +## Environment variables: + +Define the following environment variables. + +Must have: + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | +| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - | + +Defined in default application property file but possible to override: + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `LOG_PREFIX` | `service` | Logging prefix | no | - | +| `LOG_LEVEL` | `****` | Logging level | no | - | +| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | +| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment | +| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment | +| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment | +| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment | +| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment | +| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | + +These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested. +Usage of spring profiles is preferred. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - | +| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - | +| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - | + +## Pubsub configuration: + +At Pubsub should be created topic with name: + +**name:** `indexing-progress` + +## Elasticsearch configuration + +### Properties set in Partition service: + +**prefix:** `elasticsearch` + +It can be overridden by: + +- through the Spring Boot property `elastic-search-properties-prefix` +- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX` + +**Propertyset:** + +| Property | Description | +| --- | --- | +| elasticsearch.host | server URL | +| elasticsearch.port | server port | +| elasticsearch.configuration | username and password | + +<details><summary>Example of a definition for a single tenant</summary></details> + +``` + +curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{ + "properties": { + "elasticsearch.host": { + "sensitive": false, + "value": "elastic.us-central1.gcp.cloud.es.io" + }, + "elasticsearch.port": { + "sensitive": false, + "value": "9243" + }, + "elasticsearch.configuration": { + "sensitive": true, + "value": "elasticuser:elasticpassword" + } + } +}' + +``` + +## Google cloud service account configuration : +TBD + +| Required roles | +| --- | +| - | diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml index 394e63367d99c3bdabbc00c85674308752b134f3..3f7bb4987478522f50a8d113ae6094c2997c4583 100644 --- a/provider/indexer-gcp/pom.xml +++ b/provider/indexer-gcp/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>indexer-gcp</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <name>indexer-gcp</name> <description>Indexer Service GCP App Engine</description> <packaging>jar</packaging> @@ -19,12 +19,12 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-gcp</artifactId> - <version>0.13.0-rc4</version> + <version>0.14.0-rc1</version> </dependency> <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <!-- excluded due to runtime conflict with latest core-lib-gcp transient dependencies --> <exclusions> <exclusion> @@ -179,6 +179,4 @@ </plugin> </plugins> </build> - - </project> diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java deleted file mode 100644 index 6a3b330aad0dcc5a7ba09687a82e0c5e1f53548c..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.cache; - -import com.google.auth.oauth2.AccessToken; -import org.opengroup.osdu.core.common.cache.RedisCache; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -public class DatastoreCredentialCache extends RedisCache<String, AccessToken> { - - @Autowired - public DatastoreCredentialCache(final IndexerConfigurationProperties configurationProperties) { - super(configurationProperties.getRedisSearchHost(), Integer.parseInt(configurationProperties.getRedisSearchPort()), 58 * 60, String.class, AccessToken.class); - } -} \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java index 206787c8d06dd825dabf1b165c9c2f52f742a735..83ee124a964a148dcaba28b7b0e88d7c7a111b2f 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java @@ -18,29 +18,23 @@ package org.opengroup.osdu.indexer.cache; import com.google.gson.Gson; -import java.io.IOException; import java.util.Objects; import javax.inject.Inject; -import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.cache.RedisCache; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; -import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.stereotype.Component; @Component public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable { - private IKmsClient kmsClient; private RedisCache<String, String> cache; @Inject - public ElasticCredentialsCache(final IndexerConfigurationProperties properties, final IKmsClient kmsClient) { + public ElasticCredentialsCache(final IndexerConfigurationProperties properties) { this.cache = new RedisCache<>(properties.getRedisSearchHost(), Integer.parseInt(properties.getRedisSearchPort()), properties.getElasticCacheExpiration() * 60, String.class, String.class); - this.kmsClient = kmsClient; } @Override @@ -50,27 +44,17 @@ public class ElasticCredentialsCache implements IElasticCredentialsCache<String, @Override public void put(String s, ClusterSettings o) { - try { - String jsonSettings = new Gson().toJson(o); - String encryptString = kmsClient.encryptString(jsonSettings); - this.cache.put(s, encryptString); - } catch (IOException e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal server error", "Unable to encrypt settings before being cached", e); - } + String jsonSettings = new Gson().toJson(o); + this.cache.put(s, jsonSettings); } @Override public ClusterSettings get(String s) { - try { - String encryptedSettings = this.cache.get(s); - if (Objects.isNull(encryptedSettings) || encryptedSettings.isEmpty()) { - return null; - } - String jsonSettings = this.kmsClient.decryptString(encryptedSettings); - return new Gson().fromJson(jsonSettings, ClusterSettings.class); - } catch (IOException e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal server error", "Unable to decrypt settings from cache", e); + String jsonSettings = this.cache.get(s); + if (Objects.isNull(jsonSettings) || jsonSettings.isEmpty()) { + return null; } + return new Gson().fromJson(jsonSettings, ClusterSettings.class); } @Override diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java deleted file mode 100644 index 6f4d9b7505504f259ceca3d3b8e1834dc53f91ba..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2020 Google LLC - Copyright 2020 EPAM Systems, Inc - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package org.opengroup.osdu.indexer.di; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.core.common.cache.VmCache; -import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential; -import org.springframework.beans.factory.config.AbstractFactoryBean; -import org.springframework.stereotype.Component; - -@Component -public class DatastoreCredentialsCacheFactory extends - AbstractFactoryBean<ICache<String, DatastoreCredential>> { - - @Override - public Class<?> getObjectType() { - return ICache.class; - } - - @Override - protected ICache<String, DatastoreCredential> createInstance() throws Exception { - return new VmCache<>(5 * 60, 20); - } -} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..72cb1f6a3a1cedfc3f408853d99d10a9c80cca74 --- /dev/null +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020-2022 Google LLC + * Copyright 2020-2022 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + +import org.opengroup.osdu.core.common.partition.IPartitionProvider; +import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; +import org.opengroup.osdu.core.destination.elastic.ElasticSearchDestinationResolver; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ElasticSearchConfig { + + @Bean + public IElasticRepository elasticRepository(ElasticSearchConfigurationProperties properties, IPartitionProvider partitionProvider) { + return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider); + } +} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..586ce1ab9e9127282066a2e0fddf651836d12f18 --- /dev/null +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020-2022 Google LLC + * Copyright 2020-2022 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class ElasticSearchConfigurationProperties { + + private String elasticsearchPropertiesPrefix = "elasticsearch"; +} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java deleted file mode 100644 index c0995677c674dc1192b4c9533903454af441cfeb..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2020 Google LLC - Copyright 2020 EPAM Systems, Inc - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package org.opengroup.osdu.indexer.di; - -import org.opengroup.osdu.core.gcp.PubSub.PubSubExtensions; -import org.springframework.beans.factory.config.AbstractFactoryBean; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -@Component -@RequestScope -public class PubSubExtensionsFactory extends AbstractFactoryBean<PubSubExtensions> { - - - @Override - public Class<?> getObjectType() { - return PubSubExtensions.class; - } - - @Override - protected PubSubExtensions createInstance() throws Exception { - return new PubSubExtensions(); - } -} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java deleted file mode 100644 index fa51ef212c0427bd21e4a522a99891d38163ccc4..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.di; - -import lombok.extern.java.Log; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory; -import org.springframework.beans.factory.config.AbstractFactoryBean; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -@Log -@Component -@RequestScope -public class TenantFactoryService extends AbstractFactoryBean<ITenantFactory> { - - @Override - protected ITenantFactory createInstance() throws Exception { - return new TenantFactory(); - } - - @Override - public Class<?> getObjectType() { - return ITenantFactory.class; - } -} \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java deleted file mode 100644 index 9a6dc9ef7e3920668aa2155c1c39dc8531ed3529..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2021 Google LLC - * Copyright 2021 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.persistence; - -import com.google.api.gax.retrying.RetrySettings; -import com.google.cloud.TransportOptions; -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.http.HttpTransportOptions; -import java.util.HashMap; -import java.util.Map; -import javax.inject.Inject; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential; -import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; -import org.springframework.stereotype.Component; -import org.threeten.bp.Duration; - -@Component("indexerDatastoreFactory") -public class DatastoreFactory { - - @Inject - private DatastoreCredentialCache cache; - - private static Map<String, Datastore> datastoreClients = new HashMap<>(); - - private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() - .setMaxAttempts(6) - .setInitialRetryDelay(Duration.ofSeconds(10)) - .setMaxRetryDelay(Duration.ofSeconds(32)) - .setRetryDelayMultiplier(2.0) - .setTotalTimeout(Duration.ofSeconds(50)) - .setInitialRpcTimeout(Duration.ofSeconds(50)) - .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofSeconds(50)) - .build(); - - private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder() - .setReadTimeout(30000) - .build(); - - public Datastore getDatastoreInstance(TenantInfo tenantInfo) { - if (datastoreClients.get(tenantInfo.getName()) == null) { - Datastore googleDatastore = DatastoreOptions.newBuilder() - .setCredentials(new DatastoreCredential(tenantInfo)) - .setRetrySettings(RETRY_SETTINGS) - .setTransportOptions(TRANSPORT_OPTIONS) - .setNamespace(tenantInfo.getName()) - .setProjectId(tenantInfo.getProjectId()) - .build().getService(); - datastoreClients.put(tenantInfo.getName(), googleDatastore); - } - return datastoreClients.get(tenantInfo.getName()); - } -} diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java deleted file mode 100644 index a8e3373f44926527a271d4d4390b5f4d5e647177..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.persistence; - -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.Entity; -import com.google.cloud.datastore.Key; -import lombok.extern.java.Log; -import org.apache.http.HttpStatus; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.search.ClusterSettings; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; -import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; -import org.opengroup.osdu.core.common.search.Preconditions; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; - -@Log -@Component -public class ElasticRepositoryDatastore implements IElasticRepository { - - static final String HOST = "host"; - static final String PORT = "port"; - static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration"; - - @Inject - private IKmsClient kmsClient; - @Inject - private DatastoreFactory datastoreFactory; - - @Inject - private IndexerConfigurationProperties properties; - - - @Override - public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) { - - Datastore googleDatastore = this.datastoreFactory.getDatastoreInstance(tenantInfo); - Key key = googleDatastore.newKeyFactory().setKind(properties.getElasticDatastoreKind()).newKey(properties.getElasticDatastoreId()); - Entity datastoreEntity = googleDatastore.get(key); - - if (datastoreEntity == null) { - throw new AppException(HttpStatus.SC_NOT_FOUND, "Cluster setting not found", "The requested cluster setting was not found in datastore.", String.format("Cluster setting with key: '%s' does not exist in datastore.", key.getName())); - } - - String encryptedHost = null; - String encryptedPort = null; - String encryptedConfiguration = null; - - try { - encryptedHost = datastoreEntity.getString(HOST); - encryptedPort = datastoreEntity.getString(PORT); - encryptedConfiguration = datastoreEntity.getString(XPACK_RESTCLIENT_CONFIGURATION); - - String host = this.kmsClient.decryptString(encryptedHost); - String portString = this.kmsClient.decryptString(encryptedPort); - String usernameAndPassword = this.kmsClient.decryptString(encryptedConfiguration); - - Preconditions.checkNotNullOrEmpty(host, "host cannot be null"); - Preconditions.checkNotNullOrEmpty(portString, "port cannot be null"); - Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null"); - - int port = Integer.parseInt(portString); - - return new ClusterSettings(host, port, usernameAndPassword); - } catch (GoogleJsonResponseException e) { - String debuggingInfo = String.format("Host: %s | port: %s | configuration: %s", encryptedHost, encryptedPort, encryptedConfiguration); - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting decryption error", "An error has occurred decrypting cluster settings.", debuggingInfo, e); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", "An error has occurred fetching cluster settings from the datastore.", e); - } - } -} \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java index 31d242cd46515f9b37907e458b510dd7a37557bd..2a35a96f5684f9e0c1227d549c0419da067c8118 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java @@ -14,129 +14,58 @@ package org.opengroup.osdu.indexer.publish; -import com.google.api.gax.retrying.RetrySettings; -import com.google.cloud.pubsub.v1.Publisher; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; -import com.google.gson.reflect.TypeToken; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.PubsubMessage; +import java.util.HashMap; +import java.util.Map; +import lombok.RequiredArgsConstructor; import lombok.extern.java.Log; -import org.apache.http.HttpStatus; -import org.elasticsearch.common.Strings; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.indexer.JobStatus; -import org.opengroup.osdu.core.common.model.indexer.RecordStatus; -import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.core.gcp.PubSub.PubSubExtensions; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver; +import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination; +import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage; +import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic; import org.opengroup.osdu.indexer.provider.interfaces.IPublisher; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import org.threeten.bp.Duration; - -import javax.inject.Inject; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.List; -import java.util.Map; @Log @Component @RequestScope +@RequiredArgsConstructor public class PublisherImpl implements IPublisher { - private static final Map<String, Publisher> PUBSUB_CLIENTS = new HashMap<>(); - private static final String TOPIC_ID = "indexing-progress"; + private final OqmDriver driver; - @Inject - private ITenantFactory tenantStorageFactory; - - @Inject - private PubSubExtensions pubSubExtensions; - - @Inject - private IndexerConfigurationProperties properties; + private final OqmTopic oqmTopic = OqmTopic.builder().name(TOPIC_ID).build(); @Override - public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception { - - // Don't publish to pubsub when testing locally - if (properties.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) { - return; - } - - String tenant = headers.getPartitionId(); - if(Strings.isNullOrEmpty(tenant)) - tenant = headers.getAccountId(); - - Publisher publisher = this.getPublisher(tenant); - if (publisher == null) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", "A fatal internal error occurred creating publisher client."); - } - PubsubMessage pubsubMessage = getPubsubMessage(headers, indexerBatchStatus); - - pubSubExtensions.publishAndCreateTopicIfNotExist(publisher, pubsubMessage); - - } - - private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() - .setTotalTimeout(Duration.ofSeconds(30)) - .setInitialRetryDelay(Duration.ofSeconds(2)) - .setRetryDelayMultiplier(2) - .setMaxRetryDelay(Duration.ofSeconds(5)) - .setInitialRpcTimeout(Duration.ofSeconds(10)) - .setRpcTimeoutMultiplier(2) - .setMaxRpcTimeout(Duration.ofSeconds(10)) + public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) { + OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId()) .build(); + String json = generatePubSubMessage(indexerBatchStatus); - - private PubsubMessage getPubsubMessage(DpsHeaders headers, JobStatus indexerBatchStatus) { - - Gson gson = new GsonBuilder().create(); - Type listType = new TypeToken<List<RecordStatus>>() {}.getType(); - JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType); - ByteString statusChangedTagsData = ByteString.copyFromUtf8(statusChangedTagsJson.toString()); - - PubsubMessage.Builder builder = PubsubMessage.newBuilder(); - String tenant = headers.getPartitionId(); - //This code it to provide backward compatibility to slb-account-id - if(!Strings.isNullOrEmpty(tenant)) { - builder.putAttributes(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionId()); - } else { - builder.putAttributes(DpsHeaders.ACCOUNT_ID, headers.getAccountId()); - } - - builder.putAttributes(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); - // TODO: uncomment when deploying to production -// builder.putAttributes( AppEngineHeaders.CLOUD_TRACE_CONTEXT, headers.getHeaders().get(AppEngineHeaders.CLOUD_TRACE_CONTEXT)); - builder.setData(statusChangedTagsData); - - return builder.build(); + Map<String, String> attributes = getAttributes(headers); + OqmMessage oqmMessage = OqmMessage.builder().data(json).attributes(attributes).build(); + driver.publish(oqmMessage, oqmTopic, oqmDestination); } - private Publisher getPublisher(String tenantName) throws IOException { - TenantInfo info = this.tenantStorageFactory.getTenantInfo(tenantName); - if (info == null) { - return null; - } else { - if (PUBSUB_CLIENTS.containsKey(tenantName)) return PUBSUB_CLIENTS.get(tenantName); - - ProjectTopicName topicName = ProjectTopicName.newBuilder().setProject(info.getProjectId()).setTopic(TOPIC_ID).build(); - Publisher publisher = Publisher.newBuilder(topicName).setRetrySettings(RETRY_SETTINGS).build(); - - if (publisher == null) return null; + private Map<String, String> getAttributes(DpsHeaders headers) { + Map<String, String> attributes = new HashMap<>(); + attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); + attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + headers.addCorrelationIdIfMissing(); + attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + return attributes; + } - PUBSUB_CLIENTS.put(tenantName, publisher); - return publisher; - } + private String generatePubSubMessage(JobStatus jobStatus) { + Gson gson = new GsonBuilder().create(); + JsonElement statusChangedTagsJson = gson.toJsonTree(jobStatus, JobStatus.class); + return statusChangedTagsJson.toString(); } } \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java similarity index 95% rename from provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java rename to provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java index 0f70a40bc444ae9fd7e1553c319684df1286273b..47b42552ccdef36a19103027cdde783bd4d50359 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java @@ -27,7 +27,7 @@ import org.springframework.security.config.annotation.web.configuration.WebSecur @Configuration @EnableWebSecurity @EnableGlobalMethodSecurity(prePostEnabled = true) -public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter { +public class GcpSecurityConfig extends WebSecurityConfigurerAdapter { @Override protected void configure(HttpSecurity http) throws Exception { diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java index 9950bb31d3eb14da96c466aeb283eea7b2f2110b..360f576f68021a2fdb398db4d5c368284069bc2e 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java @@ -25,6 +25,7 @@ import java.util.logging.Level; import javax.inject.Inject; import lombok.extern.java.Log; import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.auth.TokenProvider; import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.model.entitlements.AuthorizationResponse; import org.opengroup.osdu.core.common.model.http.AppException; @@ -34,7 +35,6 @@ import org.opengroup.osdu.core.common.model.search.SearchServiceRole; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.stereotype.Component; @@ -50,7 +50,7 @@ public class RequestInfoImpl implements IRequestInfo { private DpsHeaders dpsHeaders; @Inject - private IServiceAccountJwtClient serviceAccountJwtClient; + private TokenProvider tokenProvider; @Inject private TenantInfo tenantInfo; @@ -99,12 +99,12 @@ public class RequestInfoImpl implements IRequestInfo { @Override public boolean isTaskQueueRequest() { - if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){ - log.log(Level.INFO,"Request confirmed as cloud task, token validation in progress"); + if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)) { + log.log(Level.INFO, "Request confirmed as cloud task, token validation in progress"); return isCloudTaskRequest(); } - if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){ - log.log(Level.INFO,"Request confirmed as AppEngine, headers validation in progress"); + if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) { + log.log(Level.INFO, "Request confirmed as AppEngine, headers validation in progress"); return isAppEngineTaskRequest(); } return false; @@ -116,7 +116,7 @@ public class RequestInfoImpl implements IRequestInfo { return true; } - private boolean isAppEngineTaskRequest(){ + private boolean isAppEngineTaskRequest() { if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) { return false; } @@ -136,7 +136,7 @@ public class RequestInfoImpl implements IRequestInfo { } return authHeader; } else { - return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName()); + return "Bearer " + this.tokenProvider.getIdToken(); } } } diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java deleted file mode 100644 index 6c57527219c328d75ab5acaffbee7d85f7c558e8..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.util; - -import com.auth0.jwt.JWT; -import com.google.cloud.iam.credentials.v1.GenerateIdTokenResponse; -import com.google.cloud.iam.credentials.v1.IamCredentialsClient; -import com.google.cloud.iam.credentials.v1.ServiceAccountName; -import java.util.Collections; -import javax.inject.Inject; -import org.apache.http.HttpStatus; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.model.search.IdToken; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; -import org.opengroup.osdu.core.gcp.multitenancy.credentials.IamCredentialsProvider; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -@Primary -@Component -@RequestScope -public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient { - - private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/-/serviceAccounts/%s"; - - private final IamCredentialsProvider iamCredentialsProvider = new IamCredentialsProvider(); - - @Value("${GOOGLE_AUDIENCES}") - private String audiences; - - @Inject - private ITenantFactory tenantInfoServiceProvider; - @Inject - private IJwtCache cacheService; - @Inject - private JaxRsDpsLog log; - @Inject - private DpsHeaders dpsHeaders; - @Inject - private IndexerConfigurationProperties properties; - - public String getIdToken(String tenantName) { - this.log.info("Tenant name received for auth token is: " + tenantName); - TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName); - if (tenant == null) { - this.log.error("Invalid tenant name receiving from pubsub"); - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub"); - } - try { - - IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); - // Add the user to DpsHeaders directly - this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); - - if (!IdToken.refreshToken(cachedToken)) { - return cachedToken.getTokenValue(); - } - - try (IamCredentialsClient iamCredentialsClient = iamCredentialsProvider.getIamCredentialsClient()) { - ServiceAccountName serviceAccountName = ServiceAccountName.parse(String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getServiceAccount())); - GenerateIdTokenResponse idTokenResponse = iamCredentialsClient.generateIdToken(serviceAccountName, Collections.emptyList(), audiences, true); - String token = idTokenResponse.getToken(); - IdToken idToken = IdToken.builder().tokenValue(token).expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build(); - this.cacheService.put(tenant.getServiceAccount(), idToken); - return token; - } - } catch (AppException e) { - throw e; - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", e); - } - } -} diff --git a/provider/indexer-gcp/src/main/resources/application-anthos.properties b/provider/indexer-gcp/src/main/resources/application-anthos.properties new file mode 100644 index 0000000000000000000000000000000000000000..9e553661404fae2ae946a221d506b5b56d16f8a0 --- /dev/null +++ b/provider/indexer-gcp/src/main/resources/application-anthos.properties @@ -0,0 +1,6 @@ +oqmDriver=rabbitmq +service.token.provider=OPENID +partition-auth-enabled=false +openid.provider-url= +openid.provider-client-id= +openid.provider-client-secret= \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-dev.properties b/provider/indexer-gcp/src/main/resources/application-dev.properties deleted file mode 100644 index 6c49ae23e07fddb7668a52d645d70268579ce9a9..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/resources/application-dev.properties +++ /dev/null @@ -1,21 +0,0 @@ -google-cloud-project=opendes - -indexer-host=indexer-jvmvia5dea-uc.a.run.app -STORAGE_HOSTNAME=storage-jvmvia5dea-uc.a.run.app - -storage-schema-host=http://127.0.0.1:8081/api/storage/v2/schemas -storage-query-record-host=http://127.0.0.1:8081/api/storage/v2/query/records -storage-query-record-for-conversion-host=https://storage-jvmvia5dea-uc.a.run.app/api/storage/v2/query/records:batch -storage-records-batch-size=20 - -indexer-queue-host=https://indexer-queue-jvmvia5dea-uc.a.run.app/_dps/task-handlers/enqueue - -AUTHORIZE_API=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1 -LEGALTAG_API=https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1 -CRS_API=https://crs-converter-jvmvia5dea-uc.a.run.app/api/crs/v1 - -## use below values for gcp: opendes -REDIS_GROUP_HOST=127.0.0.1 -redis-search-host=127.0.0.1 - -google-audiences=519000754840-09v7ssbpku7sevhvbtq3bdoi70es16p5.apps.googleusercontent.com \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-gcp.properties b/provider/indexer-gcp/src/main/resources/application-gcp.properties new file mode 100644 index 0000000000000000000000000000000000000000..fd738d7f484bf44fc3b2f2043191259dd0b3a1b8 --- /dev/null +++ b/provider/indexer-gcp/src/main/resources/application-gcp.properties @@ -0,0 +1,3 @@ +oqmDriver=pubsub +service.token.provider=GCP +partition-auth-enabled=true \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-kuber.properties b/provider/indexer-gcp/src/main/resources/application-kuber.properties deleted file mode 100644 index 22d3ba0253112794113a96f90fa094f5468902e1..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/resources/application-kuber.properties +++ /dev/null @@ -1,24 +0,0 @@ -google-cloud-project=${GOOGLE_CLOUD_PROJECT} - -indexer-host=os-indexer-service -STORAGE_HOSTNAME=os-storage-service - -storage-schema-host=http://os-storage-service/api/storage/v2/schemas -storage-query-record-host=http://os-storage-service/api/storage/v2/query/records -storage-query-record-for-conversion-host=http://os-storage-service/api/storage/v2/query/records:batch -storage-records-batch-size=20 - -indexer-queue-host=http://os-storage-service/_dps/task-handlers/enqueue - -AUTHORIZE_API=http://os-entitlement-service/entitlements/v1 -LEGALTAG_API=http://os-legal-service/api/legal/v1 -CRS_API=${CRS_API} - -## use below values for gcp: opendes -REDIS_GROUP_HOST=${REDIS_GROUP_HOST} -redis-search-host=${REDIS_SEARCH_HOST} - -google-audiences=${GOOGLE_AUDIENCES} - -deployment-environment=cloud -disable.appengine.log.factory=true \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-local.properties b/provider/indexer-gcp/src/main/resources/application-local.properties deleted file mode 100644 index a5511b15c0564250bca41b67c6c39646f9512e79..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/resources/application-local.properties +++ /dev/null @@ -1,21 +0,0 @@ -google-cloud-project=osdu - -indexer-host=os-indexer-dot-opendes.appspot.com -STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com - -storage-schema-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas -storage-query-record-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records -storage-query-record-for-conversion-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch -storage-records-batch-size=20 - -indexer-queue-host=https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue - -AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1 -LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1 -CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1 - -## use below values for gcp: opendes -REDIS_GROUP_HOST=127.0.0.1 -redis-search-host=127.0.0.1 - -google-audiences=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com diff --git a/provider/indexer-gcp/src/main/resources/application-testing.properties b/provider/indexer-gcp/src/main/resources/application-testing.properties deleted file mode 100644 index 67941ce47b5fd27c99e487fa59c935d0bcacd827..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/main/resources/application-testing.properties +++ /dev/null @@ -1,21 +0,0 @@ -google-cloud-project=opendes-evt - -indexer-host=os-indexer-dot-opendes-evt.appspot.com -STORAGE_HOSTNAME=os-storage-dot-opendes-evt.appspot.com - -storage-schema-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/schemas -storage-query-record-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records -storage-query-record-for-conversion-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records:batch -storage-records-batch-size=20 - -indexer-queue-host=https://os-indexer-queue-dot-opendes-evt.appspot.com/_dps/task-handlers/enqueue - -AUTHORIZE_API=https://entitlements-dot-opendes-evt.appspot.com/entitlements/v1 -LEGALTAG_API=https://os-legal-dot-opendes-evt.appspot.com/api/legal/v1 -CRS_API=https://crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v1 - -## use below values for gcp: opendes -REDIS_GROUP_HOST=10.253.209.196 -redis-search-host=10.118.2.140 - -google-audiences=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties index e18440679858d1208f1f6f683220e78a7e372978..f8ffcf5aea0314dd8077f87179ceac92a173160d 100644 --- a/provider/indexer-gcp/src/main/resources/application.properties +++ b/provider/indexer-gcp/src/main/resources/application.properties @@ -1,18 +1,15 @@ -LOG_PREFIX=indexer - +#server config +server.port=8080 server.servlet.contextPath=/api/indexer/v2 + +#log config +LOG_PREFIX=indexer logging.level.org.springframework.web=${LOG_LEVEL:DEBUG} -server.port=8080 + +#jvm config JAVA_OPTS=-Xms3072m -Xmx3072m JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 -deployment-environment=CLOUD - -REDIS_GROUP_PORT=6379 -DEFAULT_DATA_COUNTRY=US - -redis-search-port=6379 - #Default Cache Settings schema-cache-expiration=60 index-cache-expiration=60 @@ -22,26 +19,41 @@ cursor-cache-expiration=60 kinds-cache-expiration=2880 # Attributes Cache expiration 2*24*60 attributes-cache-expiration=2880 - kinds-redis-database=1 cron-index-cleanup-threshold-days=3 cron-empty-index-cleanup-threshold-days=7 +#indexer service config +DEFAULT_DATA_COUNTRY=US gae-service=indexer -key-ring=csqp -kms-key=searchService +security.https.certificate.trust=false +storage-records-by-kind-batch-size=20 +storage-records-batch-size=20 -KEY_RING=${key-ring} -KMS_KEY=${kms-key} -GOOGLE_CLOUD_PROJECT=${google-cloud-project} +REDIS_GROUP_PORT=6379 +REDIS_GROUP_HOST=redis-cache-groups -elastic-datastore-kind=SearchSettings -elastic-datastore-id=indexer-service +REDIS_SEARCH_PORT=6379 +REDIS_SEARCH_HOST=redis-cache-search -security.https.certificate.trust=false -indexer.que.service.mail=default@iam.gserviceaccount.com -SCHEMA_HOST=${HOST}/api/schema-service/v1/schema -storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds +PARTITION_HOST=http://partition +PARTITION_PATH=/api/partition/v1 +PARTITION_API=${PARTITION_HOST}${PARTITION_PATH} + +ENTITLEMENTS_HOST=http://entitlements +ENTITLEMENTS_PATH=/api/entitlements/v2 +AUTHORIZE_API=${ENTITLEMENTS_HOST}${ENTITLEMENTS_PATH} + +STORAGE_HOST=http://storage +STORAGE_PATH=/api/storage/v2 +STORAGE_API=${STORAGE_HOST}${STORAGE_PATH} +STORAGE_QUERY_KINDS_HOST=${STORAGE_API}/query/kinds +STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_API}/query/records:batch +STORAGE_QUERY_RECORD_HOST=${STORAGE_API}/query/records +STORAGE_SCHEMA_HOST=${STORAGE_API}/schemas + +SCHEMA_BASE_HOST=http://schema +SCHEMA_PATH=/api/schema-service/v1/schema +SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH} -google.audiences=123.apps.googleusercontent.com -partition.api=http://localhost:8081/api/partition/v1 \ No newline at end of file +INDEXER_QUEUE_HOST=http://indexer-queue \ No newline at end of file diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java index f839c8d2f25e550df8ee3e631a1c96c6b826ff66..1038c1f9376f8a0a877700394633e0340c2e2018 100644 --- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java +++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java @@ -25,7 +25,6 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.IndexInfo; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.core.classloader.annotations.PrepareForTest; diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java index c689c04d3c4fa8d0a4f4c40ead41db09a25df4b7..7b26ad8a3a283adb917ffc00c41dd27d58f40b1a 100644 --- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java +++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java @@ -37,7 +37,6 @@ import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.indexer.logging.AuditLogger; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -73,7 +72,7 @@ public class IndexCopyServiceImplTest { @Mock private IndicesService indicesService; @Mock - private IndexerMappingService mappingService; + private IMappingService mappingService; @Mock private ElasticClientHandler elasticClientHandler; @Mock diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java deleted file mode 100644 index c716cb1a637f5a5d843ebccd73656f2fa1c0752f..0000000000000000000000000000000000000000 --- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java +++ /dev/null @@ -1,337 +0,0 @@ -package org.opengroup.osdu.indexer.service;//// Copyright 2017-2019, Schlumberger -//// -//// Licensed under the Apache License, Version 2.0 (the "License"); -//// you may not use this file except in compliance with the License. -//// You may obtain a copy of the License at -//// -//// http://www.apache.org/licenses/LICENSE-2.0 -//// -//// Unless required by applicable law or agreed to in writing, software -//// distributed under the License is distributed on an "AS IS" BASIS, -//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -//// See the License for the specific language governing permissions and -//// limitations under the License. -// -//package org.opendes.indexer.service; -// -//import com.google.gson.Gson; -//import com.google.gson.reflect.TypeToken; -//import org.elasticsearch.action.bulk.BulkItemResponse; -//import org.elasticsearch.action.bulk.BulkResponse; -//import org.elasticsearch.client.RequestOptions; -//import org.elasticsearch.client.RestHighLevelClient; -//import org.junit.Before; -//import org.junit.Ignore; -//import org.junit.Test; -//import org.junit.runner.RunWith; -//import org.mockito.InjectMocks; -//import org.mockito.Mock; -//import org.mockito.Spy; -//import org.opendes.client.api.DpsHeaders; -//import org.opendes.core.logging.JaxRsDpsLog; -//import org.opendes.core.model.DeploymentEnvironment; -//import org.opendes.core.model.RecordChangedMessages; -//import org.opendes.core.service.IndicesService; -//import org.opendes.core.util.Config; -//import org.opendes.core.util.ElasticClientHandler; -//import org.opendes.core.util.ElasticIndexNameResolver; -//import org.opendes.core.util.HeadersUtil; -//import org.opendes.indexer.logging.AuditLogger; -//import org.opendes.indexer.model.*; -//import org.opendes.indexer.publish.IPublisher; -//import org.opendes.indexer.util.IRequestInfo; -//import org.opendes.indexer.util.IndexerQueueTaskBuilder; -//import org.opendes.indexer.util.JobStatus; -//import org.opendes.indexer.util.RecordInfo; -//import org.powermock.core.classloader.annotations.PrepareForTest; -//import javax.inject.Inject; -//import org.springframework.context.annotation.Lazy; -//import org.springframework.test.context.junit4.SpringRunner; -// -//import java.io.IOException; -//import java.lang.reflect.Type; -//import java.util.*; -// -//import static java.util.Collections.singletonList; -//import static org.junit.Assert.*; -//import static org.mockito.Matchers.any; -//import static org.mockito.Mockito.verify; -//import static org.mockito.Mockito.when; -//import static org.powermock.api.mockito.PowerMockito.mock; -//import static org.powermock.api.mockito.PowerMockito.mockStatic; -// -//@Ignore -//@RunWith(SpringRunner.class) -//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class}) -//public class IndexerServiceTest { -// -// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," + -// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]"; -// private final String kind1 = "tenant1:testindexer1:well:1.0.0"; -// private final String kind2 = "tenant1:testindexer2:well:1.0.0"; -// private final String recordId1 = "tenant1:doc:test1"; -// private final String recordId2 = "tenant1:doc:test2"; -// private final String failureMassage = "test failure"; -// -// @Mock -// private IndexSchemaService indexSchemaService; -// @Mock -// private IndicesService indicesService; -// @Mock -// private IndexerMappingService indexerMappingService; -// @Mock -// private StorageService storageService; -// @Mock -// private IPublisher publisherImpl; -// @Mock -// private RestHighLevelClient restHighLevelClient; -// @Mock -// private ElasticClientHandler elasticClientHandler; -// @Mock -// private BulkResponse bulkResponse; -// @Mock -// private IRequestInfo requestInfo; -// @Mock -// private ElasticIndexNameResolver elasticIndexNameResolver; -// @Mock -// private AttributeParsingServiceImpl attributeParsingServiceImpl; -// @Mock -// private IndexerQueueTaskBuilder indexerQueueTaskBuilder; -// @Mock -// private JaxRsDpsLog log; -// @Mock -// private AuditLogger auditLogger; -// @InjectMocks -// private IndexerServiceImpl sut; -// @InjectMocks @Spy -// private JobStatus jobStatus = new JobStatus(); -// -// @Inject -// @Lazy -// private DpsHeaders dpsHeaders; -// private RecordChangedMessages recordChangedMessages; -// private List<RecordInfo> recordInfos; -// -// @Before -// public void setup() throws IOException { -// -// mockStatic(StorageAcl.class); -// mockStatic(Config.class); -// -// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); -// when(Config.getElasticClusterName()).thenReturn("CLUSTER"); -// when(Config.getElasticServerAddress()).thenReturn("testsite"); -// -// dpsHeaders = new DpsHeaders(); -// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1"); -// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); -// when(requestInfo.getHeaders()).thenReturn(dpsHeaders); -// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders()); -// -// Type listType = new TypeToken<List<RecordInfo>>() {}.getType(); -// recordInfos = (new Gson()).fromJson(pubsubMsg, listType); -// -// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); -// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse); -// -// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()}; -// when(bulkResponse.getItems()).thenReturn(responses); -// Map<String, String> attr = new HashMap<>(); -// attr.put(DpsHeaders.ACCOUNT_ID, "slb"); -// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build(); -// when(StorageAcl.flattenAcl(any())).thenReturn(null); -// } -// -// @Test -// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception { -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>()); -// -// assertNull(jobStatus); -// } -// -// @Test -// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() { -// try { -// indexSchemaServiceMock(kind1, null); -// indexSchemaServiceMock(kind2, null); -// List<ConversionStatus> conversionStatus = new LinkedList<>(); -// List<Records.Entity> validRecords = new ArrayList<>(); -// Map<String, Object> storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() { -// try { -// indexSchemaServiceMock(kind1, null); -// indexSchemaServiceMock(kind2, null); -// List<ConversionStatus> conversionStatuses = new LinkedList<>(); -// List<String> status=new ArrayList<>(); -// status.add("crs bla bla"); -// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build(); -// conversionStatuses.add(conversionStatus); -// List<Records.Entity> validRecords = new ArrayList<>(); -// Map<String, Object> storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla")); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() { -// try { -// List<Records.Entity> validRecords = new ArrayList<>(); -// List<ConversionStatus> conversionStatus = new LinkedList<>(); -// Map<String, Object> storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// -// Map<String, String> schema = createSchema(); -// indexSchemaServiceMock(kind1, schema); -// indexSchemaServiceMock(kind2, null); -// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0"); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop()); -// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() { -// try { -// Map<String, Object> storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// List<ConversionStatus> conversionStatus = new LinkedList<>(); -// List<Records.Entity> validRecords = new ArrayList<>(); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// Map<String, String> schema = createSchema(); -// indexSchemaServiceMock(kind2, schema); -// indexSchemaServiceMock(kind1, null); -// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() { -// try { -// Map<String, Object> storageData = new HashMap<>(); -// List<ConversionStatus> conversionStatus = new LinkedList<>(); -// -// storageData.put("schema1", "test-value"); -// List<Records.Entity> validRecords = new ArrayList<>(); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// Map<String, String> schema = createSchema(); -// indexSchemaServiceMock(kind2, schema); -// indexSchemaServiceMock(kind1, null); -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size()); -// -// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)")); -// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)")); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// private BulkItemResponse prepareResponseFail() { -// BulkItemResponse responseFail = mock(BulkItemResponse.class); -// when(responseFail.isFailed()).thenReturn(true); -// when(responseFail.getFailureMessage()).thenReturn(failureMassage); -// when(responseFail.getId()).thenReturn(recordId1); -// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure"))); -// return responseFail; -// } -// -// private BulkItemResponse prepareResponseSuccess() { -// BulkItemResponse responseSuccess = mock(BulkItemResponse.class); -// when(responseSuccess.getId()).thenReturn(recordId2); -// return responseSuccess; -// } -// -// private void indexSchemaServiceMock(String kind, Map<String, String> schema) { -// if (schema == null) { -// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build(); -// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema); -// } else { -// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build(); -// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema); -// } -// } -// -// private Map<String, String> createSchema() { -// Map<String, String> schema = new HashMap<>(); -// schema.put("schema1", "keyword"); -// schema.put("schema2", "boolean"); -// schema.put("schema3", "date"); -// schema.put("schema6", "object"); -// return schema; -// } -//} diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml index c2e59a976e30efbef42951d5e2d95313d953bc8f..378bf7dace7c028b53b55110d81fced5a63d35e3 100644 --- a/provider/indexer-ibm/pom.xml +++ b/provider/indexer-ibm/pom.xml @@ -21,7 +21,7 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> @@ -31,7 +31,7 @@ <packaging>jar</packaging> <properties> - <os-core-lib-ibm.version>0.13.0-rc6</os-core-lib-ibm.version> + <os-core-lib-ibm.version>0.13.0</os-core-lib-ibm.version> </properties> <profiles> @@ -53,34 +53,45 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-codec</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-codec</artifactId> + <version>4.1.68.Final</version> </dependency> - <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>os-core-lib-ibm</artifactId> <version>${os-core-lib-ibm.version}</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-codec</artifactId> + </exclusion> + </exclusions> </dependency> - <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-security</artifactId> </dependency> - <dependency> + <!-- <dependency> <groupId>org.springframework.security.oauth</groupId> <artifactId>spring-security-oauth2</artifactId> <version>2.3.6.RELEASE</version> - </dependency> + </dependency> --> - <dependency> + <!-- <dependency> <groupId>org.springframework.security</groupId> <artifactId>spring-security-jwt</artifactId> <version>1.0.10.RELEASE</version> - </dependency> - <dependency> - <groupId>org.springframework.security</groupId> - <artifactId>spring-security-oauth2-client</artifactId> - </dependency> + </dependency> --> <dependency> <groupId>org.springframework.data</groupId> <artifactId>spring-data-commons</artifactId> @@ -89,6 +100,17 @@ <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + <version>8.10.0</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java index 02ee673a2d681c10861453dbafa828374232fc57..56fa2033b16112284ff0817a21606484a0397a1c 100644 --- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java +++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java @@ -17,7 +17,7 @@ import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.service.CronServiceImpl; import org.opengroup.osdu.core.common.model.search.IndexInfo; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.service.IndicesService; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java index bfa8e4103cbca3d54955201fe1a633e36d65ee91..a12e905f8d6602f52701806e51c58cffecf66688 100644 --- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java +++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java @@ -21,13 +21,13 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.indexer.logging.AuditLogger; +import org.opengroup.osdu.indexer.service.IMappingService; import org.opengroup.osdu.indexer.service.IndexCopyServiceImpl; -import org.opengroup.osdu.indexer.service.IndexerMappingService; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.indexer.service.IndicesService; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -63,7 +63,7 @@ public class IndexCopyServiceImplTest { @Mock private IndicesService indicesService; @Mock - private IndexerMappingService mappingService; + private IMappingService mappingService; @Mock private ElasticClientHandler elasticClientHandler; @Mock diff --git a/provider/indexer-reference/README.md b/provider/indexer-reference/README.md index 9fa7b2bb954f311e570265d8734c2fb0e989fe01..13529b214dccc308fa01f9439efeb5bc07274047 100644 --- a/provider/indexer-reference/README.md +++ b/provider/indexer-reference/README.md @@ -27,6 +27,8 @@ In order to run the service locally or remotely, you will need to have the follo | `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | | `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - | +| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - | | `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | | `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | | `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml index 0a0e1d0964271cc66be97fbf0c528956036e818a..3e1359fa94dae1c660471bfd797f0dc482d8b961 100644 --- a/provider/indexer-reference/pom.xml +++ b/provider/indexer-reference/pom.xml @@ -22,12 +22,12 @@ <parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-service</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>indexer-reference</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <name>indexer-reference</name> <description>Indexer Service GCP Anthos</description> <packaging>jar</packaging> @@ -36,7 +36,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <dependency> diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java index 5fc2d11920d2e2b39bf43a2fcde771f21fa90517..81da502c69dc5dc20160417bd1ff1085f823b139 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java @@ -60,7 +60,6 @@ import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.logging.AuditLogger; import org.opengroup.osdu.indexer.provider.interfaces.IPublisher; import org.opengroup.osdu.indexer.util.ElasticClientHandler; @@ -91,7 +90,7 @@ public class IndexerServiceImpl implements IndexerService { @Inject private IndicesService indicesService; @Inject - private IndexerMappingService mappingService; + private IMappingService mappingService; @Inject private IPublisher progressPublisher; @Inject diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java index e1c53edb8ab313727f54a905a3494a74cb3f6961..cadcb0bf01f4e8516756d2cc2e070b71071178f8 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java @@ -36,7 +36,6 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.IndexInfo; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.core.classloader.annotations.PrepareForTest; diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java index 6a438da28b492752abe329f329c0cbd85ec09977..db855559e8c02acc780da73f5436a4081b8e03b2 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java @@ -48,7 +48,6 @@ import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; -import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.logging.AuditLogger; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -76,7 +75,7 @@ public class IndexCopyServiceImplTest { @Mock private IndicesService indicesService; @Mock - private IndexerMappingService mappingService; + private IMappingService mappingService; @Mock private ElasticClientHandler elasticClientHandler; @Mock diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml index 1fd89ff029cac32e6c9580dd231add4c010c9ec3..538651fb4407d563794d76bc9f5259b443484991 100644 --- a/testing/indexer-test-aws/pom.xml +++ b/testing/indexer-test-aws/pom.xml @@ -21,13 +21,13 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-aws</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <packaging>jar</packaging> <properties> @@ -42,7 +42,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <!-- AWS specific packages --> diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml index 2b157b63a7ad882852ac5e43e61e56e8ba39cbc7..4f4c6e50ac203625417c450c7fab210604ec13cb 100644 --- a/testing/indexer-test-azure/pom.xml +++ b/testing/indexer-test-azure/pom.xml @@ -21,13 +21,13 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-azure</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <packaging>jar</packaging> <properties> @@ -45,7 +45,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml index fcc2b5e81bc27d8f33b10d1ac5a46f1fcbd33e90..51bf9d386095dbcf92745592525ebaa1b360f105 100644 --- a/testing/indexer-test-core/pom.xml +++ b/testing/indexer-test-core/pom.xml @@ -5,13 +5,13 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <properties> <maven.compiler.target>1.8</maven.compiler.target> diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java index 51502c68198ab4c42c236e052c148c11287bda0e..b37e7daa324d2f73d3c85598dcb64a5317abaf5b 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java @@ -17,8 +17,10 @@ package org.opengroup.osdu.util; +import com.google.api.client.util.Strings; import com.google.gson.Gson; +import java.net.URI; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @@ -411,14 +413,22 @@ public class ElasticUtils { restHighLevelClient = new RestHighLevelClient(builder); } catch (Exception e) { - throw new AssertionError("Setup elastic error"); + throw new AssertionError("Setup elastic error: %s" + e.getMessage()); } return restHighLevelClient; } - public RestClientBuilder createClientBuilder(String host, String usernameAndPassword, int port) { + public RestClientBuilder createClientBuilder(String url, String usernameAndPassword, int port) throws Exception { String scheme = this.sslEnabled ? "https" : "http"; - RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, scheme)); + + url = url.trim().replaceAll("^(?i)(https?)://",""); + URI uri = new URI(scheme + "://" + url); + + RestClientBuilder builder = RestClient.builder(new HttpHost(uri.getHost(), port, uri.getScheme())); + if (!Strings.isNullOrEmpty(uri.getPath())) { + builder.setPathPrefix(uri.getPath()); + } + builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT) .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT)); diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml index e86a33cbd846f37586836b850bc340f176093c5b..c32245324cbf5bf9290df99dab3b0ff2886c3bce 100644 --- a/testing/indexer-test-gcp/pom.xml +++ b/testing/indexer-test-gcp/pom.xml @@ -6,13 +6,13 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-gcp</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <packaging>jar</packaging> <properties> @@ -37,7 +37,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <!-- Cucumber --> diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml index ce81d0fff255e43e061f97828393d31465a031c9..4531fdba9cfa2a310062e6161c6a32b359282d7f 100644 --- a/testing/indexer-test-ibm/pom.xml +++ b/testing/indexer-test-ibm/pom.xml @@ -6,20 +6,20 @@ <parent> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-ibm</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <packaging>jar</packaging> <properties> <maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.source>1.8</maven.compiler.source> <cucumber.version>1.2.5</cucumber.version> - <os-core-lib-ibm.version>0.13.0-rc5</os-core-lib-ibm.version> + <os-core-lib-ibm.version>0.13.0</os-core-lib-ibm.version> </properties> <dependencies> @@ -38,7 +38,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-test-core</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> </dependency> <dependency> diff --git a/testing/pom.xml b/testing/pom.xml index 679e6dcc19bb2beca2e96243c9cbe5105db0dc47..08218f0dd1511159b46ec16efe599ca5d66ca7e2 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -18,7 +18,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-test</artifactId> - <version>0.13.0-SNAPSHOT</version> + <version>0.14.0-SNAPSHOT</version> <description>Indexer Service Integration Test Root Project</description> <properties> <spring.version>5.1.19.RELEASE</spring.version>