diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1182faefe1d1f5ebb80fc170aeddb798082b453f..65239865d0e935ceb993f742f82136bf75a976d9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,16 +16,14 @@ variables:
GCP_DOMAIN: cloud.slb-ds.com
GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/
- OSDU_GCP_APPLICATION_NAME: os-indexer
OSDU_GCP_SERVICE: indexer
OSDU_GCP_VENDOR: gcp
OSDU_GCP_QUEUE_SA_EMAIL: pub-sub-indexer-queue@nice-etching-277309.iam.gserviceaccount.com
- OSDU_GCP_SERVICE_ACCOUNT: osdu-gcp-sa@nice-etching-277309.iam.gserviceaccount.com
- OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true'
- OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE: 20
- OSDU_GCP_DATA_GROUP: osdu
- OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_V2_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST,STORAGE_QUERY_RECORD_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_HOST,STORAGE_SCHEMA_HOST=$OSDU_GCP_STORAGE_SCHEMA_HOST,STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST,STORAGE_HOSTNAME=$OSDU_GCP_STORAGE_HOSTNAME,STORAGE_RECORDS_BATCH_SIZE=$OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE,INDEXER_QUEUE_HOST=$OSDU_GCP_INDEXER_QUEUE_HOST,LEGALTAG_API=$OSDU_GCP_LEGALTAG_API,CRS_API=$OSDU_GCP_CRS_API,DATA_GROUP=$OSDU_GCP_DATA_GROUP,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,PARTITION_API=$OSDU_GCP_PARTITION_API,INDEXER_QUE_SERVICE_MAIL=$OSDU_GCP_QUEUE_SA_EMAIL,SCHEMA_HOST=$OSDU_GCP_SCHEMA_URL/api/schema-service/v1/schema --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
- OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
+ OSDU_GCP_K8S_ACCOUNT: workload-identity-indexer
+ OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_URL --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL"
+ OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_K8S_ACCOUNT --set data.limits_cpu=1 --set data.limits_memory=1G"
+ OSDU_GCP_HELM_CONFIG_SERVICE: indexer-config
+ OSDU_GCP_HELM_DEPLOYMENT_SERVICE: indexer-deploy
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
IBM_BUILD_SUBDIR: provider/indexer-ibm
@@ -62,7 +60,7 @@ include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: "master"
- file: 'cloud-providers/osdu-gcp-cloudrun.yml'
+ file: 'cloud-providers/osdu-gcp-gke.yml'
aws-test-java:
diff --git a/NOTICE b/NOTICE
index e9ff93311a1cce1c8fe9203f582f72e8de098154..318af935077e87aac709ca934981383e9c28d550 100644
--- a/NOTICE
+++ b/NOTICE
@@ -16,7 +16,7 @@ The following software have components provided under the terms of this license:
- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
- Cobertura (from http://cobertura.sourceforge.net)
-- Plexus :: Default Container (from )
+- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- oro (from )
@@ -27,8 +27,8 @@ The following software have components provided under the terms of this license:
- AMQP 1.0 JMS Spring Boot AutoConfiguration (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-autoconfigure)
- AMQP 1.0 JMS Spring Boot Starter (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-starter)
-- ASM Core (from )
-- ASM Core (from )
+- ASM Core (from https://repo1.maven.org/maven2/org/ow2/asm/asm)
+- ASM Core (from https://repo1.maven.org/maven2/org/ow2/asm/asm)
- ASM based accessors helper used by json-smart (from )
- AWS Event Stream (from https://github.com/awslabs/aws-eventstream-java)
- AWS Java SDK :: AWS Core (from https://aws.amazon.com/sdkforjava)
@@ -179,10 +179,10 @@ The following software have components provided under the terms of this license:
- Apache Ant + JUnit (from http://ant.apache.org/)
- Apache Ant Core (from http://ant.apache.org/)
- Apache Ant Launcher (from http://ant.apache.org/)
-- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/)
- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
- Apache Commons Codec (from https://commons.apache.org/proper/commons-codec/)
-- Apache Commons Collections (from https://commons.apache.org/proper/commons-collections/)
+- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/)
+- Apache Commons IO (from https://commons.apache.org/proper/commons-io/)
- Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/)
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
@@ -191,37 +191,32 @@ The following software have components provided under the terms of this license:
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
- Apache HttpClient (from http://hc.apache.org/httpcomponents-client)
- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client)
-- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga)
-- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga)
- Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api)
- Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core)
- Apache Log4j JUL Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul)
- Apache Log4j SLF4J Binding (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl)
-- Apache Log4j to SLF4J Adapter (from )
+- Apache Log4j to SLF4J Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-to-slf4j)
- Apache Maven Invoker (from )
-- Apache Maven Reporting Implementation (from )
-- Apache Maven Wagon :: API (from )
-- Apache Maven Wagon :: Providers :: File Provider (from )
-- Apache Maven Wagon :: Providers :: HTTP Shared Library (from )
-- Apache Maven Wagon :: Providers :: Lightweight HTTP Provider (from )
-- Apache Maven Wagon :: Providers :: SSH Common Library (from )
-- Apache Maven Wagon :: Providers :: SSH External Provider (from )
-- Apache Maven Wagon :: Providers :: SSH Provider (from )
+- Apache Maven Reporting Implementation (from https://repo1.maven.org/maven2/org/apache/maven/reporting/maven-reporting-impl)
- Apache Velocity (from http://velocity.apache.org/engine/devel/)
- AssertJ fluent assertions (from https://repo1.maven.org/maven2/org/assertj/assertj-core)
- Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client)
- Asynchronous Http Client Netty Utils (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client-netty-utils)
- AutoValue Annotations (from https://github.com/google/auto/tree/master/value)
+- AutoValue Annotations (from https://github.com/google/auto/tree/master/value)
- AutoValue Processor (from https://github.com/google/auto/tree/master/value)
- Azure Spring Boot Starter for Azure AD Spring Security Integration (from https://github.com/Azure/azure-sdk-for-java)
- BSON (from https://bsonspec.org)
- Bean Validation API (from http://beanvalidation.org)
-- Brave Instrumentation: Http Adapters (from )
+- Bean Validation API (from http://beanvalidation.org)
+- BeanUtils (from http://jakarta.apache.org/commons/beanutils/)
+- Brave (from https://repo1.maven.org/maven2/io/zipkin/brave/brave)
+- Brave Instrumentation: Http Adapters (from https://repo1.maven.org/maven2/io/zipkin/brave/brave-instrumentation-http)
- Brave instrumentation for Reactor Netty HTTP (from https://github.com/reactor/reactor-netty)
- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
-- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
+- Byte Buddy agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
-- Cloud Storage JSON API v1-rev20181109-1.28.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
+- Cloud Storage JSON API v1-rev20200927-1.30.10 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
- CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava)
- Cobertura (from http://cobertura.sourceforge.net)
- Collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections)
@@ -230,60 +225,57 @@ The following software have components provided under the terms of this license:
- Converter: Jackson (from https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson)
- Core functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty)
- Data Mapper for Jackson (from http://jackson.codehaus.org)
-- Doxia :: APT Module (from )
-- Doxia :: Core (from )
-- Doxia :: Decoration Model (from http://maven.apache.org/doxia/doxia-sitetools/doxia-decoration-model/)
-- Doxia :: FML Module (from )
-- Doxia :: Sink API (from )
-- Doxia :: Site Renderer (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/)
-- Doxia :: XDoc Module (from )
-- Doxia :: XHTML Module (from )
-- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
+- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default)
+- Doxia :: APT Module (from http://maven.apache.org/doxia/doxia/doxia-modules/doxia-module-apt/)
+- Doxia :: Core (from http://maven.apache.org/doxia/doxia/doxia-core/)
+- Doxia :: FML Module (from http://maven.apache.org/doxia/doxia/doxia-modules/doxia-module-fml/)
+- Doxia :: Sink API (from http://maven.apache.org/doxia/doxia/doxia-sink-api/)
+- Doxia :: XDoc Module (from http://maven.apache.org/doxia/doxia/doxia-modules/doxia-module-xdoc/)
+- Doxia :: XHTML Module (from http://maven.apache.org/doxia/doxia/doxia-modules/doxia-module-xhtml/)
+- Doxia Sitetools :: Decoration Model (from http://maven.apache.org/doxia/doxia-sitetools/doxia-decoration-model/)
+- Doxia Sitetools :: Site Renderer Component (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Expression Language 3.0 (from http://uel.java.net)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson)
+- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson)
- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
- Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine)
- Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine)
-- Google Cloud Common Protos for Java (from https://github.com/googleapis/common-protos-java)
-- Google Cloud Common Protos for Java (from https://github.com/googleapis/common-protos-java)
-- Google Cloud Common Protos for Java (from https://github.com/googleapis/common-protos-java)
-- Google Cloud Core (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core)
-- Google Cloud Core HTTP (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http)
-- Google Cloud Core HTTP (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http)
+- Google Cloud Core (from https://github.com/googleapis/java-core)
+- Google Cloud Core HTTP (from https://github.com/googleapis/java-core)
+- Google Cloud Core HTTP (from https://github.com/googleapis/java-core)
- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc)
- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc)
-- Google Cloud Datastore (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-datastore)
-- Google Cloud Datastore (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-datastore)
+- Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore)
+- Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore)
- Google Cloud IAM Service Account Credentials (from https://github.com/googleapis/java-iamcredentials)
- Google Cloud Key Management Service (KMS) API v1-rev9-1.22.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
-- Google Cloud Logging (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-logging)
-- Google Cloud Logging (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-logging)
-- Google Cloud Pub/Sub (from https://github.com/googleapis/java-pubsub)
-- Google Cloud Pub/Sub (from https://github.com/googleapis/java-pubsub)
-- Google Cloud Storage (from https://github.com/googleapis/java-storage)
+- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging)
+- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging)
+- Google Cloud Pub/Sub (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub)
+- Google Cloud Pub/Sub (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub)
+- Google Cloud Storage (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-storage)
- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client)
- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client)
- Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client)
- Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client)
-- Gson (from https://github.com/google/gson)
-- Gson (from https://github.com/google/gson)
+- Gson (from https://repo1.maven.org/maven2/com/google/code/gson/gson)
+- Gson (from https://repo1.maven.org/maven2/com/google/code/gson/gson)
- Guava InternalFutureFailureAccess and InternalFutures (from https://repo1.maven.org/maven2/com/google/guava/failureaccess)
- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava)
- Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava)
- HPPC Collections (from https://repo1.maven.org/maven2/com/carrotsearch/hppc)
-- HPPC Collections (from https://repo1.maven.org/maven2/com/carrotsearch/hppc)
- HTTP functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty)
-- Hibernate Validator Engine (from )
+- Hibernate Validator Engine (from https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
+- HttpCore (NIO extensions module) (from http://hc.apache.org/httpcomponents-core/)
+- HttpCore (base module) (from http://hc.apache.org/httpcomponents-core/)
- IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java)
-- IO (from http://jakarta.apache.org/commons/${pom.artifactId.substring(8)}/)
-- Identity and Access Management (IAM) API v1-rev247-1.23.0 (from )
-- IntelliJ IDEA Annotations (from http://www.jetbrains.org)
+- Identity and Access Management (IAM) API v1-rev307-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam)
- J2ObjC Annotations (from https://github.com/google/j2objc/)
- J2ObjC Annotations (from https://github.com/google/j2objc/)
- JBoss Logging 3 (from http://www.jboss.org)
@@ -291,7 +283,7 @@ The following software have components provided under the terms of this license:
- JDOM (from http://www.jdom.org)
- JMES Path Query library (from https://aws.amazon.com/sdkforjava)
- JSON Small and Fast Parser (from https://repo1.maven.org/maven2/net/minidev/json-smart)
-- JSON Web Token support for the JVM (from https://github.com/jwtk/jjwt.git)
+- JSON Web Token support for the JVM (from https://github.com/jwtk/jjwt)
- JSON library from Android SDK (from http://developer.android.com/sdk)
- JSON.simple (from http://code.google.com/p/json-simple/)
- JSONassert (from https://github.com/skyscreamer/JSONassert)
@@ -307,19 +299,18 @@ The following software have components provided under the terms of this license:
- Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8)
- Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8)
- Jackson extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson)
-- Jackson module: Afterburner (from https://github.com/FasterXML/jackson-modules-base)
-- Jackson module: JAXB-annotations (from http://github.com/FasterXML/jackson-module-jaxb-annotations)
-- Jackson module: JAXB-annotations (from http://github.com/FasterXML/jackson-module-jaxb-annotations)
+- Jackson module: JAXB Annotations (from https://github.com/FasterXML/jackson-modules-base)
+- Jackson module: JAXB Annotations (from https://github.com/FasterXML/jackson-modules-base)
- Jackson-annotations (from http://github.com/FasterXML/jackson)
- Jackson-annotations (from http://github.com/FasterXML/jackson)
-- Jackson-core (from https://github.com/FasterXML/jackson-core)
-- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding)
+- Jackson-core (from http://wiki.fasterxml.com/JacksonHome)
+- Jackson-dataformat-XML (from https://github.com/FasterXML/jackson-dataformat-xml)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text)
- Jackson-datatype-Joda (from http://wiki.fasterxml.com/JacksonModuleJoda)
+- Jackson-module-Afterburner (from http://wiki.fasterxml.com/JacksonHome)
- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names)
- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names)
-- Jakarta Bean Validation API (from https://beanvalidation.org)
- Java Libraries for Amazon Simple WorkFlow (from https://aws.amazon.com/sdkforjava)
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
@@ -327,64 +318,56 @@ The following software have components provided under the terms of this license:
- Java UUID Generator (from http://wiki.fasterxml.com/JugHome)
- Javassist (from http://www.javassist.org/)
- Javassist (from http://www.javassist.org/)
+- JetBrains Java Annotations (from https://github.com/JetBrains/java-annotations)
- Jetty Server (from https://repo1.maven.org/maven2/org/mortbay/jetty/jetty)
- Jetty Utilities (from https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util)
- Joda-Time (from https://www.joda.org/joda-time/)
-- Json Path (from https://github.com/jayway/JsonPath)
- KeePassJava2 :: All (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2)
- KeePassJava2 :: DOM (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-dom)
- KeePassJava2 :: JAXB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-jaxb)
- KeePassJava2 :: KDB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdb)
- KeePassJava2 :: KDBX (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdbx)
- KeePassJava2 :: Simple (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-simple)
-- Logback Contrib :: JSON :: Classic (from )
-- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
+- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
- Logback Contrib :: Jackson (from )
- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common)
-- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common)
-- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
- Lucene Grouping (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping)
-- Lucene Grouping (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping)
-- Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter)
- Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter)
- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
-- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
-- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
-- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
-- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
-- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries)
- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries)
- Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser)
-- Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser)
-- Lucene Sandbox (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox)
- Lucene Sandbox (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox)
-- Lucene Spatial (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial)
- Lucene Spatial 3D (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial3d)
-- Lucene Spatial 3D (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial3d)
-- Lucene Spatial Extras (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial-extras)
- Lucene Spatial Extras (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial-extras)
- Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest)
-- Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest)
-- MapStruct Core (from http://mapstruct.org/mapstruct/)
+- MapStruct Core (from https://repo1.maven.org/maven2/org/mapstruct/mapstruct)
- Maven Artifact (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact)
-- Maven Artifact Manager (from )
+- Maven Artifact Manager (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact-manager)
- Maven Core (from https://repo1.maven.org/maven2/org/apache/maven/maven-core)
-- Maven Error Diagnostics (from )
+- Maven Error Diagnostics (from https://repo1.maven.org/maven2/org/apache/maven/maven-error-diagnostics)
- Maven Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-model)
-- Maven Monitor (from )
+- Maven Monitor (from https://repo1.maven.org/maven2/org/apache/maven/maven-monitor)
- Maven Plugin API (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-api)
-- Maven Plugin Descriptor Model (from )
-- Maven Plugin Parameter Documenter API (from )
-- Maven Plugin Registry Model (from )
-- Maven Profile Model (from )
-- Maven Project (from )
-- Maven Reporting API (from )
+- Maven Plugin Descriptor Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-descriptor)
+- Maven Plugin Parameter Documenter API (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-parameter-documenter)
+- Maven Plugin Registry Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-registry)
+- Maven Profile Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-profile)
+- Maven Project Builder (from https://repo1.maven.org/maven2/org/apache/maven/maven-project)
+- Maven Reporting API (from https://repo1.maven.org/maven2/org/apache/maven/reporting/maven-reporting-api)
- Maven Repository Metadata Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-repository-metadata)
- Maven Settings (from https://repo1.maven.org/maven2/org/apache/maven/maven-settings)
+- Maven Wagon API (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-provider-api)
+- Maven Wagon File Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-file)
+- Maven Wagon HTTP Shared Library (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-http-shared)
+- Maven Wagon Lightweight HTTP Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-http-lightweight)
+- Maven Wagon SSH Common Library (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh-common)
+- Maven Wagon SSH External Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh-external)
+- Maven Wagon SSH Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh)
- Metrics Core (from https://repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -432,14 +415,9 @@ The following software have components provided under the terms of this license:
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
-- OAuth2 for Spring Security (from )
-- OAuth2 for Spring Security (from )
+- OAuth2 for Spring Security (from https://repo1.maven.org/maven2/org/springframework/security/oauth/spring-security-oauth2)
+- OAuth2 for Spring Security (from https://repo1.maven.org/maven2/org/springframework/security/oauth/spring-security-oauth2)
- Objenesis (from http://objenesis.org)
-- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp)
-- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp)
-- OkHttp Logging Interceptor (from https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor)
-- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection)
-- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection)
- Okio (from https://github.com/square/okio/)
- Okio (from https://github.com/square/okio/)
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
@@ -451,83 +429,97 @@ The following software have components provided under the terms of this license:
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- PWDB :: Database (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/database)
-- Plexus :: Default Container (from )
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- Plexus Velocity Component (from )
- PowerMock (from http://www.powermock.org)
+- PowerMock (from http://www.powermock.org)
+- PowerMock (from http://www.powermock.org)
+- PowerMock (from http://www.powermock.org)
+- PowerMock (from http://www.powermock.org)
- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf)
- Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j)
- QpidJMS Client (from )
- RabbitMQ Java Client (from https://www.rabbitmq.com)
-- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty)
-- Retrofit (from https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit)
-- Servlet Specification 2.5 API (from )
-- SnakeYAML (from http://code.google.com/p/snakeyaml/)
-- Spatial4J (from https://projects.eclipse.org/projects/locationtech.spatial4j)
+- Reactor Netty with all modules (from https://github.com/reactor/reactor-netty)
+- Retrofit (from https://github.com/square/retrofit)
+- Servlet Specification 2.5 API (from http://jetty.mortbay.org)
+- SnakeYAML (from http://www.snakeyaml.org)
+- Spatial4J (from http://www.locationtech.org/projects/locationtech.spatial4j)
- Spring AMQP Core (from https://github.com/spring-projects/spring-amqp)
-- Spring Boot AMQP Starter (from http://projects.spring.io/spring-boot/)
+- Spring AOP (from https://github.com/spring-projects/spring-framework)
+- Spring Beans (from https://github.com/spring-projects/spring-framework)
+- Spring Boot (from http://projects.spring.io/spring-boot/)
+- Spring Boot Actuator (from http://projects.spring.io/spring-boot/)
+- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure)
+- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/)
+- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/)
+- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/)
+- Spring Boot Data MongoDB Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-data-mongodb)
+- Spring Boot Dependencies (from https://projects.spring.io/spring-boot/#)
+- Spring Boot Logging Starter (from http://projects.spring.io/spring-boot/)
+- Spring Boot Reactor Netty Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-reactor-netty)
+- Spring Boot Starter (from http://projects.spring.io/spring-boot/)
+- Spring Boot Test (from http://projects.spring.io/spring-boot/)
+- Spring Boot Test Auto-Configure (from http://projects.spring.io/spring-boot/)
+- Spring Boot Validation Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-validation)
+- Spring Boot WebFlux Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-webflux)
- Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework)
+- Spring Context (from https://github.com/spring-projects/spring-framework)
+- Spring Core (from https://github.com/spring-projects/spring-framework)
- Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons)
- Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons)
- Spring Data MongoDB - Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-mongodb)
- Spring Expression Language (SpEL) (from https://github.com/spring-projects/spring-framework)
-- Spring Framework: AOP (from http://www.springframework.org)
-- Spring Framework: Context (from http://www.springframework.org)
-- Spring Framework: WebMVC (from http://www.springframework.org)
- Spring JMS (from https://github.com/spring-projects/spring-framework)
- Spring Messaging (from https://github.com/spring-projects/spring-framework)
-- Spring Plugin - Core (from )
- Spring Plugin - Metadata Extension (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-metadata)
+- Spring Plugin Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core)
- Spring RabbitMQ Support (from https://github.com/spring-projects/spring-amqp)
-- Spring Retry (from https://www.springsource.org)
+- Spring Retry (from http://www.springsource.org)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
-- Spring TestContext Framework (from https://github.com/spring-projects/spring-framework)
+- Spring TestContext Framework (from https://github.com/SpringSource/spring-framework)
- Spring Transaction (from https://github.com/spring-projects/spring-framework)
+- Spring Web (from https://github.com/spring-projects/spring-framework)
+- Spring Web MVC (from https://github.com/spring-projects/spring-framework)
- Spring WebFlux (from https://github.com/spring-projects/spring-framework)
-- Spring beans (from https://repo1.maven.org/maven2/org/springframework/spring-beans)
-- Spring core (from https://repo1.maven.org/maven2/org/springframework/spring-core)
-- Spring web (from https://repo1.maven.org/maven2/org/springframework/spring-web)
- T-Digest (from https://github.com/tdunning/t-digest)
- Vavr (from http://vavr.io)
- Vavr Match (from http://vavr.io)
- Woodstox (from https://github.com/FasterXML/woodstox)
-- Xerces2-j (from https://xerces.apache.org/xerces2-j/)
+- Zipkin Core Library (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin)
- Zipkin Reporter Brave (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter-brave)
-- Zipkin Reporter: Core (from )
-- Zipkin v2 (from )
+- Zipkin Reporter: Core (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter)
- aalto-xml (from )
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
-- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
- aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client)
-- brave (from )
-- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
-- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
-- com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis)
-- com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis)
-- compiler (from http://github.com/spullara/mustache.java)
+- com.google.api.grpc:proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:proto-google-common-protos (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:proto-google-common-protos (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:proto-google-common-protos (from https://github.com/googleapis/googleapis)
- compiler (from http://github.com/spullara/mustache.java)
-- datastore-v1-proto-client (from )
+- datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client)
- elasticsearch-cli (from https://github.com/elastic/elasticsearch)
-- elasticsearch-cli (from https://github.com/elastic/elasticsearch)
-- elasticsearch-core (from https://github.com/elastic/elasticsearch)
- elasticsearch-core (from https://github.com/elastic/elasticsearch)
- elasticsearch-geo (from https://github.com/elastic/elasticsearch)
- elasticsearch-secure-sm (from https://github.com/elastic/elasticsearch)
-- elasticsearch-secure-sm (from https://github.com/elastic/elasticsearch)
-- elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
- elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
+- grpc-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/grpc-google-cloud-pubsub-v1)
+- grpc-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/grpc-google-cloud-pubsub-v1)
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-core (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-core (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-core (from https://github.com/grpc/grpc-java)
@@ -553,21 +545,22 @@ The following software have components provided under the terms of this license:
- javatuples (from http://www.javatuples.org)
- javax.inject (from http://code.google.com/p/atinject/)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
-- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
-- jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core)
-- jersey-core-client (from git://java.net/jersey~code/jersey-client)
+- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
+- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
+- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common)
-- jersey-core-server (from git://java.net/jersey~code/jersey-server)
-- jersey-ext-bean-validation (from )
-- jersey-ext-entity-filtering (from )
-- jersey-inject-hk2 (from )
-- jersey-media-jaxb (from )
-- jersey-media-json-jackson (from git://java.net/jersey~code/project/jersey-media-json-jackson)
+- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server)
+- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation)
+- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering)
+- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2)
+- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb)
+- jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson)
- jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4)
- jose4j (from https://bitbucket.org/b_c/jose4j/)
-- lang-mustache (from https://github.com/elastic/elasticsearch)
+- json-path (from http://code.google.com/p/json-path/)
- lang-mustache (from https://github.com/elastic/elasticsearch)
- lettuce (from http://github.com/mp911de/lettuce/wiki)
+- logging-interceptor (from https://github.com/square/okhttp)
- mapper-extras (from https://github.com/elastic/elasticsearch)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
@@ -576,64 +569,45 @@ The following software have components provided under the terms of this license:
- mockito-core (from https://github.com/mockito/mockito)
- nio-multipart-parser (from )
- nio-stream-storage (from https://github.com/synchronoss/nio-stream-storage)
+- okhttp (from https://square.github.io/okhttp/)
+- okhttp (from https://square.github.io/okhttp/)
+- okhttp-urlconnection (from https://github.com/square/okhttp)
+- okhttp-urlconnection (from https://github.com/square/okhttp)
- org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/)
-- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
-- parent-join (from https://github.com/elastic/elasticsearch)
+- org.xmlunit:xmlunit-core (from https://www.xmlunit.org/)
- parent-join (from https://github.com/elastic/elasticsearch)
- perfmark:perfmark-api (from https://github.com/perfmark/perfmark)
-- powermock-api-support (from )
-- powermock-core (from http://www.powermock.org)
-- powermock-module-junit4 (from http://www.powermock.org)
-- powermock-module-junit4-common (from )
-- powermock-reflect (from )
-- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
-- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
+- powermock-module-junit4-common (from https://repo1.maven.org/maven2/org/powermock/powermock-module-junit4-common)
- proto-google-cloud-iamcredentials-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-iamcredentials-v1)
-- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
-- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
-- proto-google-iam-v1 (from https://github.com/googleapis/api-client-staging)
-- rank-eval (from https://github.com/elastic/elasticsearch)
+- proto-google-cloud-logging-v2 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2)
+- proto-google-cloud-logging-v2 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2)
+- proto-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-pubsub-v1)
+- proto-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-pubsub-v1)
+- proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1)
- rank-eval (from https://github.com/elastic/elasticsearch)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- rest (from https://github.com/elastic/elasticsearch)
-- rest (from https://github.com/elastic/elasticsearch)
-- rest-high-level (from https://github.com/elastic/elasticsearch)
- rest-high-level (from https://github.com/elastic/elasticsearch)
- rxjava (from https://github.com/ReactiveX/RxJava)
- server (from https://github.com/elastic/elasticsearch)
-- server (from https://github.com/elastic/elasticsearch)
-- spring-boot (from https://spring.io/projects/spring-boot)
-- spring-boot-actuator (from https://spring.io/projects/spring-boot)
-- spring-boot-actuator-autoconfigure (from https://spring.io/projects/spring-boot)
-- spring-boot-autoconfigure (from https://spring.io/projects/spring-boot)
-- spring-boot-configuration-processor (from https://spring.io/projects/spring-boot)
-- spring-boot-dependencies (from https://spring.io/projects/spring-boot)
-- spring-boot-starter (from https://spring.io/projects/spring-boot)
-- spring-boot-starter-actuator (from https://spring.io/projects/spring-boot)
-- spring-boot-starter-data-mongodb (from https://spring.io/projects/spring-boot)
+- spring-boot-starter-amqp (from https://spring.io/projects/spring-boot)
- spring-boot-starter-jersey (from https://spring.io/projects/spring-boot)
- spring-boot-starter-json (from https://spring.io/projects/spring-boot)
- spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot)
-- spring-boot-starter-logging (from https://spring.io/projects/spring-boot)
-- spring-boot-starter-reactor-netty (from https://spring.io/projects/spring-boot)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
- spring-boot-starter-test (from https://spring.io/projects/spring-boot)
- spring-boot-starter-tomcat (from https://spring.io/projects/spring-boot)
-- spring-boot-starter-validation (from https://spring.io/projects/spring-boot)
- spring-boot-starter-web (from https://spring.io/projects/spring-boot)
-- spring-boot-starter-webflux (from https://spring.io/projects/spring-boot)
-- spring-boot-test (from https://spring.io/projects/spring-boot)
-- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
-- spring-security-config (from http://spring.io/spring-security)
-- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
+- spring-security-config (from https://spring.io/projects/spring-security)
+- spring-security-core (from https://spring.io/projects/spring-security)
- spring-security-oauth2-client (from http://spring.io/spring-security)
- spring-security-oauth2-core (from http://spring.io/spring-security)
- spring-security-oauth2-jose (from http://spring.io/spring-security)
- spring-security-oauth2-jose (from http://spring.io/spring-security)
- spring-security-oauth2-resource-server (from http://spring.io/spring-security)
-- spring-security-test (from http://spring.io/spring-security)
-- spring-security-web (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-web)
+- spring-security-test (from https://spring.io/projects/spring-security)
+- spring-security-web (from https://spring.io/projects/spring-security)
- springfox-core (from https://github.com/springfox/springfox)
- springfox-schema (from https://github.com/springfox/springfox)
- springfox-spi (from https://github.com/springfox/springfox)
@@ -643,11 +617,12 @@ The following software have components provided under the terms of this license:
- springfox-swagger2 (from https://github.com/springfox/springfox)
- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations)
- swagger-jaxrs (from )
-- swagger-models (from )
+- swagger-models (from https://repo1.maven.org/maven2/io/swagger/swagger-models)
- tomcat-annotations-api (from https://tomcat.apache.org/)
- tomcat-embed-core (from http://tomcat.apache.org/)
- tomcat-embed-el (from https://tomcat.apache.org/)
- tomcat-embed-websocket (from https://tomcat.apache.org/)
+- xercesImpl (from https://repo1.maven.org/maven2/xerces/xercesImpl)
- xml-apis (from )
========================================================================
@@ -657,22 +632,19 @@ The following software have components provided under the terms of this license:
- API Common (from https://github.com/googleapis/api-common-java)
- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
-- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
-- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common)
-- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common)
-- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- Reflections (from http://github.com/ronmamo/reflections)
- Stax2 API (from http://github.com/FasterXML/stax2-api)
- ThreeTen backport (from https://www.threeten.org/threetenbp)
-- ThreeTen backport (from https://www.threeten.org/threetenbp)
-- jersey-ext-bean-validation (from )
+- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation)
- jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4)
========================================================================
@@ -683,15 +655,14 @@ The following software have components provided under the terms of this license:
- API Common (from https://github.com/googleapis/api-common-java)
- API Common (from https://github.com/googleapis/api-common-java)
- ASM Analysis (from )
-- ASM Commons (from )
-- ASM Core (from )
-- ASM Core (from )
+- ASM Commons (from https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
+- ASM Core (from https://repo1.maven.org/maven2/org/ow2/asm/asm)
+- ASM Core (from https://repo1.maven.org/maven2/org/ow2/asm/asm)
- ASM Tree (from )
- ASM Util (from )
-- ASM library repackaged as OSGi bundle (from )
- Apache Commons Codec (from https://commons.apache.org/proper/commons-codec/)
-- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
-- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
- GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java)
@@ -701,14 +672,9 @@ The following software have components provided under the terms of this license:
- Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials)
- Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials)
- Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http)
-- Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http)
-- JSch (from http://www.jcraft.com/jsch/)
- JavaBeans Activation Framework API jar (from )
- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common)
-- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common)
- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
-- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
-- Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest)
- Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -723,20 +689,21 @@ The following software have components provided under the terms of this license:
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
- Reflections (from http://github.com/ronmamo/reflections)
-- SnakeYAML (from http://code.google.com/p/snakeyaml/)
-- Spring core (from https://repo1.maven.org/maven2/org/springframework/spring-core)
-- ThreeTen backport (from https://www.threeten.org/threetenbp)
+- SnakeYAML (from http://www.snakeyaml.org)
+- Spring Core (from https://github.com/spring-projects/spring-framework)
- ThreeTen backport (from https://www.threeten.org/threetenbp)
+- asm-all-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged)
- classworlds (from http://classworlds.codehaus.org/)
- jakarta.xml.bind-api (from )
- jaxen (from http://jaxen.codehaus.org/)
-- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
-- jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core)
-- jersey-core-client (from git://java.net/jersey~code/jersey-client)
-- jersey-core-server (from git://java.net/jersey~code/jersey-server)
-- jersey-ext-entity-filtering (from )
-- jersey-inject-hk2 (from )
-- jersey-media-jaxb (from )
+- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
+- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
+- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
+- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server)
+- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering)
+- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2)
+- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb)
+- jsch (from https://repo1.maven.org/maven2/com/jcraft/jsch)
========================================================================
CC-BY-2.5
@@ -768,29 +735,29 @@ CDDL-1.0
========================================================================
The following software have components provided under the terms of this license:
-- ASM library repackaged as OSGi bundle (from )
-- Class Model for Hk2 (from )
+- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model)
- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
-- HK2 Spring Bridge (from )
-- HK2 config types (from )
-- HK2 configuration module (from )
-- HK2 core module (from )
-- HK2 module of HK2 itself (from )
+- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge)
+- HK2 config types (from https://repo1.maven.org/maven2/org/glassfish/hk2/config-types)
+- HK2 configuration module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-config)
+- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core)
+- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2)
- JavaBeans Activation Framework API jar (from )
-- Run Level Service (from )
+- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel)
- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
-- Servlet Specification 2.5 API (from )
+- Servlet Specification 2.5 API (from http://jetty.mortbay.org)
- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
+- asm-all-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
-- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
-- jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core)
-- jersey-core-client (from git://java.net/jersey~code/jersey-client)
+- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
+- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
+- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common)
-- jersey-core-server (from git://java.net/jersey~code/jersey-server)
-- jersey-ext-entity-filtering (from )
-- jersey-inject-hk2 (from )
-- jersey-media-jaxb (from )
-- jersey-media-json-jackson (from git://java.net/jersey~code/project/jersey-media-json-jackson)
+- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server)
+- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering)
+- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2)
+- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb)
+- jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson)
========================================================================
CDDL-1.1
@@ -798,15 +765,15 @@ CDDL-1.1
The following software have components provided under the terms of this license:
- Expression Language 3.0 (from http://uel.java.net)
-- Java Architecture for XML Binding (from http://jaxb.java.net/)
- Java Servlet API (from http://servlet-spec.java.net)
- JavaBeans Activation Framework (from )
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- JavaMail API (from https://repo1.maven.org/maven2/com/sun/mail/javax.mail)
- OSGi resource locator (from )
-- Servlet Specification 2.5 API (from )
+- Servlet Specification 2.5 API (from http://jetty.mortbay.org)
- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
-- jersey-ext-bean-validation (from )
+- jaxb-api (from https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
+- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation)
- jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4)
- tomcat-embed-core (from http://tomcat.apache.org/)
@@ -825,7 +792,6 @@ The following software have components provided under the terms of this license:
- JDOM (from http://www.jdom.org)
- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
-- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
- Woodstox (from https://github.com/FasterXML/woodstox)
========================================================================
@@ -834,14 +800,14 @@ EPL-1.0
The following software have components provided under the terms of this license:
- Logback Classic Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-classic)
-- Logback Contrib :: JSON :: Classic (from )
-- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
+- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
- Logback Contrib :: Jackson (from )
- Logback Core Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-core)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
-- SnakeYAML (from http://code.google.com/p/snakeyaml/)
+- SnakeYAML (from http://www.snakeyaml.org)
- jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core)
- jts-io-common (from )
@@ -850,39 +816,39 @@ GPL-2.0-only
========================================================================
The following software have components provided under the terms of this license:
-- ASM library repackaged as OSGi bundle (from )
-- Class Model for Hk2 (from )
+- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model)
- Cobertura (from http://cobertura.sourceforge.net)
- Cobertura Limited Runtime (from http://cobertura.sourceforge.net)
- Commons Lang (from http://commons.apache.org/lang/)
- Expression Language 3.0 (from http://uel.java.net)
- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
-- HK2 Spring Bridge (from )
-- HK2 config types (from )
-- HK2 configuration module (from )
-- HK2 core module (from )
-- HK2 module of HK2 itself (from )
-- Java Architecture for XML Binding (from http://jaxb.java.net/)
+- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge)
+- HK2 config types (from https://repo1.maven.org/maven2/org/glassfish/hk2/config-types)
+- HK2 configuration module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-config)
+- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core)
+- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2)
- Java Servlet API (from http://servlet-spec.java.net)
- JavaBeans Activation Framework (from )
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
- OSGi resource locator (from )
- RabbitMQ Java Client (from https://www.rabbitmq.com)
-- Run Level Service (from )
+- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel)
- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
+- asm-all-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged)
- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
-- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
-- jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core)
-- jersey-core-client (from git://java.net/jersey~code/jersey-client)
+- jaxb-api (from https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
+- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
+- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
+- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common)
-- jersey-core-server (from git://java.net/jersey~code/jersey-server)
-- jersey-ext-bean-validation (from )
-- jersey-ext-entity-filtering (from )
-- jersey-inject-hk2 (from )
-- jersey-media-jaxb (from )
-- jersey-media-json-jackson (from git://java.net/jersey~code/project/jersey-media-json-jackson)
+- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server)
+- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation)
+- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering)
+- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2)
+- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb)
+- jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson)
- jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4)
- tomcat-embed-core (from http://tomcat.apache.org/)
@@ -891,7 +857,7 @@ GPL-2.0-or-later
========================================================================
The following software have components provided under the terms of this license:
-- SnakeYAML (from http://code.google.com/p/snakeyaml/)
+- SnakeYAML (from http://www.snakeyaml.org)
========================================================================
GPL-2.0-with-classpath-exception
@@ -902,7 +868,6 @@ The following software have components provided under the terms of this license:
- Cobertura (from http://cobertura.sourceforge.net)
- Expression Language 3.0 (from http://uel.java.net)
- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
-- Java Architecture for XML Binding (from http://jaxb.java.net/)
- Java Servlet API (from http://servlet-spec.java.net)
- JavaBeans Activation Framework (from )
- OSGi resource locator (from )
@@ -911,16 +876,17 @@ The following software have components provided under the terms of this license:
- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
-- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
-- jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core)
-- jersey-core-client (from git://java.net/jersey~code/jersey-client)
+- jaxb-api (from https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
+- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
+- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
+- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common)
-- jersey-core-server (from git://java.net/jersey~code/jersey-server)
-- jersey-ext-bean-validation (from )
-- jersey-ext-entity-filtering (from )
-- jersey-inject-hk2 (from )
-- jersey-media-jaxb (from )
-- jersey-media-json-jackson (from git://java.net/jersey~code/project/jersey-media-json-jackson)
+- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server)
+- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation)
+- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering)
+- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2)
+- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb)
+- jersey-media-json-jackson (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-json-jackson)
- jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4)
- tomcat-embed-core (from http://tomcat.apache.org/)
@@ -929,7 +895,7 @@ GPL-3.0-only
========================================================================
The following software have components provided under the terms of this license:
-- Project Lombok (from https://projectlombok.org)
+- Project Lombok (from http://projectlombok.org)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
========================================================================
@@ -962,14 +928,13 @@ The following software have components provided under the terms of this license:
- Cobertura (from http://cobertura.sourceforge.net)
- Commons Lang (from http://commons.apache.org/lang/)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
-- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Javassist (from http://www.javassist.org/)
- Javassist (from http://www.javassist.org/)
- Logback Classic Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-classic)
-- Logback Contrib :: JSON :: Classic (from )
-- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
+- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
- Logback Contrib :: Jackson (from )
- Logback Core Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-core)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -983,7 +948,7 @@ The following software have components provided under the terms of this license:
- Javassist (from http://www.javassist.org/)
- Javassist (from http://www.javassist.org/)
-- SnakeYAML (from http://code.google.com/p/snakeyaml/)
+- SnakeYAML (from http://www.snakeyaml.org)
========================================================================
LGPL-3.0-only
@@ -1011,13 +976,12 @@ The following software have components provided under the terms of this license:
- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
+- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default)
- Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions)
-- JOpt Simple (from http://pholser.github.io/jopt-simple)
+- JOpt Simple (from http://pholser.github.com/jopt-simple)
- JUL to SLF4J bridge (from http://www.slf4j.org)
- Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
-- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
-- Lucene Sandbox (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox)
- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -1040,24 +1004,22 @@ The following software have components provided under the terms of this license:
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
-- Plexus :: Default Container (from )
- Plexus Default Interactivity Handler (from )
-- Project Lombok (from https://projectlombok.org)
+- Project Lombok (from http://projectlombok.org)
- SLF4J API Module (from http://www.slf4j.org)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
- Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos)
- adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java)
- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/)
- documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/)
-- java jwt (from http://www.jwt.io)
+- java jwt (from https://github.com/auth0/java-jwt)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java)
-- server (from https://github.com/elastic/elasticsearch)
-- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
+- spring-security-core (from https://spring.io/projects/spring-security)
========================================================================
MPL-1.1
@@ -1076,8 +1038,8 @@ The following software have components provided under the terms of this license:
- Javassist (from http://www.javassist.org/)
- Javassist (from http://www.javassist.org/)
-- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp)
- RabbitMQ Java Client (from https://www.rabbitmq.com)
+- okhttp (from https://square.github.io/okhttp/)
========================================================================
MS-RL
@@ -1102,7 +1064,7 @@ The following software have components provided under the terms of this license:
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common)
-- jersey-core-server (from git://java.net/jersey~code/jersey-server)
+- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server)
- jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core)
========================================================================
@@ -1119,14 +1081,13 @@ The following software have components provided under the terms of this license:
- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
-- Servlet Specification 2.5 API (from )
+- Servlet Specification 2.5 API (from http://jetty.mortbay.org)
========================================================================
SunPro
========================================================================
The following software have components provided under the terms of this license:
-- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
========================================================================
@@ -1143,8 +1104,8 @@ The following software have components provided under the terms of this license:
- Apache Ant Core (from http://ant.apache.org/)
- JTidy (from http://jtidy.sourceforge.net)
-- Xerces2-j (from https://xerces.apache.org/xerces2-j/)
- jaxen (from http://jaxen.codehaus.org/)
+- xercesImpl (from https://repo1.maven.org/maven2/xerces/xercesImpl)
- xml-apis (from )
========================================================================
@@ -1159,7 +1120,6 @@ X11
========================================================================
The following software have components provided under the terms of this license:
-- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
========================================================================
@@ -1184,11 +1144,11 @@ The following software have components provided under the terms of this license:
- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
-- Project Lombok (from https://projectlombok.org)
+- Project Lombok (from http://projectlombok.org)
- RabbitMQ Java Client (from https://www.rabbitmq.com)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
-- Spring web (from https://repo1.maven.org/maven2/org/springframework/spring-web)
+- Spring Web (from https://github.com/spring-projects/spring-framework)
- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- reactive-streams (from http://www.reactive-streams.org/)
@@ -1213,13 +1173,13 @@ The following software have components provided under the terms of this license:
- System Rules (from http://stefanbirkner.github.io/system-rules/)
- jakarta.xml.bind-api (from )
- jaxen (from http://jaxen.codehaus.org/)
-- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
-- jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core)
-- jersey-core-client (from git://java.net/jersey~code/jersey-client)
-- jersey-ext-bean-validation (from )
-- jersey-ext-entity-filtering (from )
-- jersey-inject-hk2 (from )
-- jersey-media-jaxb (from )
+- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
+- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
+- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
+- jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation)
+- jersey-ext-entity-filtering (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-entity-filtering)
+- jersey-inject-hk2 (from https://repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2)
+- jersey-media-jaxb (from https://repo1.maven.org/maven2/org/glassfish/jersey/media/jersey-media-jaxb)
- jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4)
- jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core)
- jts-io-common (from )
diff --git a/README.md b/README.md
index 724f7df4cd11d492bcdffbc8005551e83c4a1218..8bfcb90b5e426329e082c09d3ef9aeb0ad8b23dc 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,4 @@ running the `os-indexer` Azure implementation
## GCP Implementation
-All documentation for the GCP implementation of `os-indexer` lives [here](./provider/indexer-gcp/README.md)
-
-
+All documentation for the GCP implementation of `os-indexer` lives [here](./provider/indexer-gcp/README.md)
\ No newline at end of file
diff --git a/devops/azure/chart/helm-config.yaml b/devops/azure/chart/helm-config.yaml
index cf8438efec51d9e3aef1b1edd2c4ff7f4e17ca0b..89157496f323e000d9c149870ed0b7cc70c1fa05 100644
--- a/devops/azure/chart/helm-config.yaml
+++ b/devops/azure/chart/helm-config.yaml
@@ -3,6 +3,8 @@ global:
# Service(s) Replica Count
replicaCount: 2
+ nodepool: services
+ isAutoscalingEnabled: false
################################################################################
# Specify the Gitlab branch being used for image creation
diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml
index 20fac08cff7a776c18f4daacf8b23b4815b3ef90..b7baad43d37c4a2d2613da1b2b5f12c56817f663 100644
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -28,6 +28,10 @@ spec:
app: {{ .Chart.Name }}
aadpodidbinding: osdu-identity
spec:
+{{- if .Values.global.isAutoscalingEnabled }}
+ nodeSelector:
+ nodepool: {{ .Values.global.nodepool }}
+{{- end }}
volumes:
- name: azure-keyvault
csi:
diff --git a/devops/azure/chart/values.yaml b/devops/azure/chart/values.yaml
index d427d2a7b7ee63453f8206b7c32e8f049f1961ca..7ad7ec09d9d78519cf118719ce60207873234536 100644
--- a/devops/azure/chart/values.yaml
+++ b/devops/azure/chart/values.yaml
@@ -14,6 +14,8 @@
global:
replicaCount: 1
+ nodepool: services
+ isAutoscalingEnabled: false
image:
repository: community.opengroup.org:5555/osdu/platform/system/indexer-service
diff --git a/devops/gcp/configmap/templates/indexer-variables.yml b/devops/gcp/configmap/templates/indexer-variables.yml
index 9925de5e5c4bee6655c4739cc3d494a4417a64fd..7ef012d59b5b84986c87850a8a95688b5c3f6ca7 100644
--- a/devops/gcp/configmap/templates/indexer-variables.yml
+++ b/devops/gcp/configmap/templates/indexer-variables.yml
@@ -6,6 +6,8 @@ metadata:
name: "{{ .Values.conf.configmap }}"
namespace: "{{ .Release.Namespace }}"
data:
+ KEY_RING: "{{ .Values.data.key_ring }}"
+ KMS_KEY: "{{ .Values.data.kms_key }}"
LOG_LEVEL: "{{ .Values.data.log_level }}"
SCHEMA_HOST: "{{ .Values.data.schema_host }}"
AUTHORIZE_API: "{{ .Values.data.authorize_api }}"
@@ -13,7 +15,6 @@ data:
REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}"
- INDEXER_HOST: "{{ .Values.data.indexer_host }}"
STORAGE_QUERY_RECORD_HOST: "{{ .Values.data.storage_query_record_host }}"
STORAGE_SCHEMA_HOST: "{{ .Values.data.storage_schema_host }}"
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "{{ .Values.data.storage_query_record_for_conversion_host }}"
diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml
index 5708b9046abd95fb5d8007696dc243f13489c991..fc7fbb3f0f88cd1d60992489de89557f5e882bc1 100644
--- a/devops/gcp/configmap/values.yaml
+++ b/devops/gcp/configmap/values.yaml
@@ -3,23 +3,24 @@
# Declare variables to be passed into your templates.
data:
- log_level: ""
+ key_ring: "csqp"
+ kms_key: "searchService"
+ log_level: "INFO"
schema_host: ""
- authorize_api: ""
+ authorize_api: "http://entitlements/api/entitlements/v2/"
google_cloud_project: ""
redis_search_host: ""
redis_group_host: ""
security_https_certificate_trust: "true"
- indexer_host: ""
- storage_query_record_host: ""
- storage_schema_host: ""
- storage_query_record_for_conversion_host: ""
- storage_hostname: ""
+ storage_query_record_host: "http://storage/api/storage/v2/query/records"
+ storage_schema_host: "http://storage/api/storage/v2/schemas"
+ storage_query_record_for_conversion_host: "http://storage/api/storage/v2/query/records:batch"
+ storage_hostname: "storage"
storage_records_batch_size: "20"
indexer_queue_host: ""
- legaltag_api: ""
+ legaltag_api: "http://legal/api/legal/v1"
crs_api: ""
- partition_api: ""
+ partition_api: "http://partition/api/partition/v1/"
google_audiences: ""
indexer_que_service_mail: ""
diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml
index d7627dae6d7ba8ba245e2be3399e9eda98f1f1c2..bbe858fe8409edf4485099409d8908056cfc8a63 100644
--- a/devops/gcp/deploy/values.yaml
+++ b/devops/gcp/deploy/values.yaml
@@ -4,7 +4,7 @@
data:
requests_cpu: "0.25"
- requests_memory: "128M"
+ requests_memory: "256M"
limits_cpu: "1"
limits_memory: "1G"
serviceAccountName: ""
diff --git a/docs/tutorial/IndexerService.md b/docs/tutorial/IndexerService.md
index 0873557d9b48c398bdb17e8aeae3b6a6426f1f38..9966f7cbbfc0a23fcc0fac4bc326431f23801b2f 100644
--- a/docs/tutorial/IndexerService.md
+++ b/docs/tutorial/IndexerService.md
@@ -2,13 +2,11 @@
### Table of contents <a name="TOC"></a>
- [Indexer service](#indexer-service)
- - [Table of contents <a name="TOC"></a>](#table-of-contents)
- [Get indexing status <a name="get-indexing-status"></a>](#get-indexing-status)
- [Reindex <a name="reindex"></a>](#reindex)
-- [Copy Index <a name="copy-index"></a>](#copy-index)
-- [Get task status <a name="get-task-status"></a>](#get-task-status)
- [Schema Service adoption <a name="schema-service-adoption"></a>](#schema-service-adoption)
- [R3 Schema Support <a name="r3-schema-support"></a>](#r3-schema-support)
+- [Version info endpoint](#version-info-endpoint)
##Introduction <a name="introduction"></a>
@@ -56,20 +54,22 @@ The indexer is indexes attributes defined in the schema. Schema can be created a
## Get indexing status <a name="get-indexing-status"></a>
-Indexer service adds internal meta data to each record which registers the status of the indexing. The meta data includes the status and the last indexing date and time. This additional meta block helps to see the details of indexing. The format of the index meta block is as follows:
+Indexer service adds internal metadata to each record which registers the status of the indexing. The meta data includes the status and the last indexing date and time. This additional meta block helps to see the details of indexing. The format of the index meta block is as follows:
-```
-"index": {
- "trace": [
- String,
- String
- ],
- "statusCode": Integer,
- "lastUpdateTime": Datetime
+```json
+{
+ "index": {
+ "trace": [
+ String,
+ String
+ ],
+ "statusCode": Integer,
+ "lastUpdateTime": Datetime
+ }
}
```
Example:
-```
+```json
{
"results": [
{
@@ -97,7 +97,7 @@ Details of the index block:
You can query the index status using the following example query:
-```
+```bash
curl --request POST \
--url /api/search/v2/query \
--header 'Authorization: Token' \
@@ -128,7 +128,7 @@ POST /api/indexer/v2/reindex
<details><summary>**Curl**</summary>
-```
+```bash
curl --request POST \
--url '/api/indexer/v2/reindex' \
--header 'accept: application/json' \
@@ -143,121 +143,11 @@ curl --request POST \
[Back to table of contents](#TOC)
-## Copy Index <a name="copy-index"></a>
-
-Copy Index API can be used copy `kind` index from `common` to a private `data partition` search backend. To call it, kind from `common` partition should be provided as path parameter and private partition-id should be specified in OSDU-Data-Partition-Id header.
-
-__Note__: Copy Index API is intended for __only__ copying `kind` index from `common` cluster to private `partition` cluster, no other combination of data partitions are honored at this time.
-
-```
-POST /api/indexer/v2/copyIndex/copy/{kind}
-OSDU-Data-Partition-Id:OSDU
-```
-
-<details><summary>**Curl**</summary>
-
-```
-curl --request POST \
- --url '/api/indexer/v2/copyIndex/copy/common:welldb:wellbore:1.0.0' \
- --header 'accept: application/json' \
- --header 'authorization: Bearer <JWT>' \
- --header 'content-type: application/json' \
- --header 'OSDU-Data-Partition-Id: OSDU'
-```
-</details>
-
-The successful response from the above request will be a `task-id`, this can be later used to track the status of the task via task status API(#get-task-status).
-
-```
-{
- "task": "CrOX4STSQF6kgtSRdERhbw:92863567"
-}
-```
-
-[Back to table of contents](#TOC)
-
-## Get task status <a name="get-task-status"></a>
-
-Status of ongoing or completed index copy request for given `taskId` can retrieved via GET task status api.
-
-```
-GET /api/indexer/v2/copyIndex/taskStatus/{taskId}
-```
-
-<details><summary>**Curl**</summary>
-
-```
-curl --request GET \
- --url '/api/indexer/v2/copyIndex/taskStatus/[taskid]]' \
- --header 'accept: application/json' \
- --header 'authorization: Bearer <JWT>' \
- --header 'content-type: application/json' \
- --header 'OSDU-Data-Partition-Id: OSDU'
-```
-</details>
-
-API will respond with status of task.
-
-```
-{
- "completed": true,
- "task": {
- "node": "[nodeid]",
- "id": 113159669,
- "type": "transport",
- "action": "indices:data/write/reindex",
- "status": {
- "total": 1530,
- "updated": 0,
- "created": 1530,
- "deleted": 0,
- "batches": 1,
- "version_conflicts": 0,
- "noops": 0,
- "retries": {
- "bulk": 0,
- "search": 0
- },
- "throttled_millis": 0,
- "requests_per_second": -1,
- "throttled_until_millis": 0
- },
- "description": "reindex from [scheme=https host=host-id port=9243 query={\n \"match_all\" : {\n \"boost\" : 1.0\n }\n}][common:welldb:wellbore:1.0.0] to [common:welldb:wellbore:1.0.0]",
- "start_time_in_millis": 1539735233086,
- "running_time_in_nanos": 1094744315,
- "cancellable": true,
- "headers": {}
- },
- "response": {
- "took": 1084,
- "timed_out": false,
- "total": 1530,
- "updated": 0,
- "created": 1530,
- "deleted": 0,
- "batches": 1,
- "version_conflicts": 0,
- "noops": 0,
- "retries": {
- "bulk": 0,
- "search": 0
- },
- "throttled_millis": 0,
- "requests_per_second": -1,
- "throttled_until_millis": 0,
- "failures": []
- }
-}
-```
-
-[Back to table of contents](#TOC)
-
##Schema Service adoption <a name="schema-service-adoption"></a>
Indexer service is in adaptation process to use schemas from the Schema service instead of Storage Service.
The Indexer Service retrieves a schema from the Schema Service if the schema is not found on the Storage Service.
-Change affects only Azure implementation so far.
-Later call to the Storage Service will be deprecated and then removed (after the end of the deprecation period).
+Change affects only Azure implementation so far. Later call to the Storage Service will be deprecated and then removed (after the end of the deprecation period).
[Back to table of contents](#TOC)
@@ -296,3 +186,38 @@ data-partition-id: opendes
```
[Back to table of contents](#TOC)
+## Version info endpoint
+For deployment available public `/info` endpoint, which provides build and git related information.
+#### Example response:
+```json
+{
+ "groupId": "org.opengroup.osdu",
+ "artifactId": "storage-gcp",
+ "version": "0.10.0-SNAPSHOT",
+ "buildTime": "2021-07-09T14:29:51.584Z",
+ "branch": "feature/GONRG-2681_Build_info",
+ "commitId": "7777",
+ "commitMessage": "Added copyright to version info properties file",
+ "connectedOuterServices": [
+ {
+ "name": "elasticSearch",
+ "version":"..."
+ },
+ {
+ "name": "postgresSql",
+ "version":"..."
+ },
+ {
+ "name": "redis",
+ "version":"..."
+ }
+ ]
+}
+```
+This endpoint takes information from files, generated by `spring-boot-maven-plugin`,
+`git-commit-id-plugin` plugins. Need to specify paths for generated files to matching
+properties:
+- `version.info.buildPropertiesPath`
+- `version.info.gitPropertiesPath`
+
+[Back to table of contents](#TOC)
\ No newline at end of file
diff --git a/indexer-core/maven/settings.xml b/indexer-core/maven/settings.xml
deleted file mode 100644
index b7a15a01004e3079c32548b080a7c49d2861d9df..0000000000000000000000000000000000000000
--- a/indexer-core/maven/settings.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
- <servers>
- <server>
- <id>os-core</id>
- <username>slb-des-ext-collaboration</username>
- <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
- <!-- The generated token expires on or before 11/14/2019 -->
- <password>${VSTS_FEED_TOKEN}</password>
- </server>
- </servers>
-</settings>
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index a6d2ba058ad3ffe95566caf1071b503b0e8b971b..650b0702f9d6140364f4b8635753a1e0d125ffac 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -16,7 +16,7 @@
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
- <osdu.oscorecommon.version>0.10.0</osdu.oscorecommon.version>
+ <osdu.oscorecommon.version>0.11.0-rc4</osdu.oscorecommon.version>
</properties>
<dependencies>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java
index ce5d01e3c3ee31a9f69465871209cc7f015df5ad..5c58742f89bcf3fad892289ef7e407cbdf25a5eb 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java
@@ -39,6 +39,7 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.annotation.RequestScope;
+import springfox.documentation.annotations.ApiIgnore;
@Log
@RestController
@@ -51,6 +52,7 @@ public class CleanupIndiciesApi {
@Autowired
private AuditLogger auditLogger;
+ @ApiIgnore
@PostMapping(path = "/index-cleanup", consumes = "application/json")
@PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
public ResponseEntity cleanupIndices(@NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/InfoApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/InfoApi.java
new file mode 100644
index 0000000000000000000000000000000000000000..c9e7b6b15b62775df9cca6ecbdb26cbf63f8a58a
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/InfoApi.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.api;
+
+import java.io.IOException;
+import org.opengroup.osdu.core.common.info.VersionInfoBuilder;
+import org.opengroup.osdu.core.common.model.info.VersionInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping
+public class InfoApi {
+
+ @Autowired
+ private VersionInfoBuilder versionInfoBuilder;
+
+ @GetMapping(value = "/info", produces = MediaType.APPLICATION_JSON_VALUE)
+ public VersionInfo info() throws IOException {
+ return versionInfoBuilder.buildVersionInfo();
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
index 6e546e14cf7197a480bf84de084a3cf628373ad0..b6323ac4554024cfc82171b8ea74924812498d3d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
@@ -83,6 +83,12 @@ public class GlobalExceptionMapperCore extends ResponseEntityExceptionHandler {
this.logger.warning(exceptionMsg, e);
}
- return new ResponseEntity<Object>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+ // Support for non standard HttpStatus Codes
+ HttpStatus httpStatus = HttpStatus.resolve(e.getError().getCode());
+ if (httpStatus == null) {
+ return ResponseEntity.status(e.getError().getCode()).body(e);
+ } else {
+ return new ResponseEntity<>(e.getError(), httpStatus);
+ }
}
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
index 04a3211fda1a6a91a71fc7cea639951b41b90642..45a13b37f6b127e064f4c0807b5f3f132f1acf9f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
@@ -14,6 +14,15 @@
package org.opengroup.osdu.indexer.schema.converter;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
+import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
+import org.opengroup.osdu.indexer.schema.converter.tags.*;
+
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -24,18 +33,6 @@ import java.util.Optional;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
-import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
-import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
-import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
-import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
-import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
-import org.opengroup.osdu.indexer.schema.converter.tags.Items;
-import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty;
public class PropertiesProcessor {
@@ -185,9 +182,14 @@ public class PropertiesProcessor {
}
if ("array".equals(entry.getValue().getType())) {
-
Items items = entry.getValue().getItems();
- if(Objects.nonNull(items) && items.isComplexTypeItems()){
+
+ if (items == null) {
+ errors.add(String.format("Invalid array attribute: '%s', missing or null 'items'", entry.getKey()));
+ return Stream.empty();
+ }
+
+ if (Objects.nonNull(items) && items.isComplexTypeItems()) {
return processComplexTypeItems(entry, items);
}
@@ -215,7 +217,7 @@ public class PropertiesProcessor {
if (Objects.nonNull(entry.getValue().getRef())) {
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions
, pathPrefixWithDot + entry.getKey(), new SchemaConverterPropertiesConfig());
- Stream<Map<String, Object>> refResult = propertiesProcessor.processRef(entry.getValue().getRef());
+ Stream<Map<String, Object>> refResult = propertiesProcessor.processRef(entry.getValue().getRef());
errors.addAll(propertiesProcessor.getErrors());
return refResult;
}
@@ -232,28 +234,28 @@ public class PropertiesProcessor {
Map<String, String> indexHint = entry.getValue().getIndexHint();
String indexingType = Objects.isNull(indexHint) ?
- schemaConverterConfig.getDefaultObjectArraysType() :
- indexHint.getOrDefault(TYPE_KEY,schemaConverterConfig.getDefaultObjectArraysType());
+ schemaConverterConfig.getDefaultObjectArraysType() :
+ indexHint.getOrDefault(TYPE_KEY, schemaConverterConfig.getDefaultObjectArraysType());
- if(schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)){
+ if (schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)) {
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, new SchemaConverterPropertiesConfig());
Stream<Map<String, Object>> propertiesStream = Stream.empty();
- if(Objects.nonNull(items.getProperties())){
+ if (Objects.nonNull(items.getProperties())) {
propertiesStream = items.getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry);
}
- if (Objects.nonNull(items.getRef())){
+ if (Objects.nonNull(items.getRef())) {
propertiesStream = Stream.concat(propertiesStream, propertiesProcessor.processRef(items.getRef()));
}
- if(Objects.nonNull(items.getAllOf())){
+ if (Objects.nonNull(items.getAllOf())) {
propertiesStream = Stream.concat(propertiesStream, items.getAllOf().stream().flatMap(propertiesProcessor::processItem));
}
return storageSchemaObjectArrayEntry(
- indexingType,
- entry.getKey(),
- propertiesStream);
- }else {
+ indexingType,
+ pathPrefixWithDot + entry.getKey(),
+ propertiesStream);
+ } else {
return storageSchemaEntry(indexingType, pathPrefixWithDot + entry.getKey());
}
}
@@ -300,14 +302,14 @@ public class PropertiesProcessor {
return Stream.of(map);
}
- private Stream<Map<String, Object>> storageSchemaObjectArrayEntry(String kind, String path,Stream<Map<String, Object>> mapStream) {
+ private Stream<Map<String, Object>> storageSchemaObjectArrayEntry(String kind, String path, Stream<Map<String, Object>> mapStream) {
Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
Preconditions.checkNotNullOrEmpty(path, "path cannot be null or empty");
Map<String, Object> map = new HashMap<>();
map.put("kind", kind);
map.put("path", path);
- map.put(Constants.PROPERTIES,mapStream.collect(Collectors.toList()));
+ map.put(Constants.PROPERTIES, mapStream.collect(Collectors.toList()));
return Stream.of(map);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
index 66d47186931a086c5cf952da892612da185a6fd5..b912825864ea2c780af67fa5fe537ce26f0b78dc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
@@ -17,7 +17,6 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
-import com.google.gson.internal.LinkedTreeMap;
import com.google.gson.reflect.TypeToken;
import java.lang.reflect.Array;
import java.lang.reflect.Type;
@@ -42,10 +41,6 @@ import org.springframework.web.context.annotation.RequestScope;
@RequestScope
public class AttributeParsingServiceImpl implements IAttributeParsingService {
- private static final String GEOJSON = "GeoJSON";
- private static final String GEOMETRY_COLLECTION = "geometrycollection";
- private static final String GEOMETRIES = "geometries";
-
@Inject
private NumberParser numberParser;
@Inject
@@ -159,9 +154,7 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
}
@Override
- public void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap) {
-
- Object attributeVal = storageRecordData.get(attributeName);
+ public void tryParseGeopoint(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
try {
Type type = new TypeToken<Map<String, Double>>() {}.getType();
@@ -174,16 +167,6 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
if (position == null || position.isEmpty()) return;
dataMap.put(attributeName, position);
-
- // check if geo shape is not there and if it is not then create it in the schema as well as create the data.
- LinkedTreeMap<String, Object> map = (LinkedTreeMap) storageRecordData.get(GEOJSON);
- if (map == null || map.isEmpty()) {
- Map<String, Object> geometry = this.geometryConversionService.getGeopointGeoJson(positionMap);
-
- if (geometry == null) return;
-
- dataMap.put(DATA_GEOJSON_TAG, geometry);
- }
} catch (JsonSyntaxException | IllegalArgumentException e) {
String parsingError = String.format("geo-point parsing error: %s attribute: %s | value: %s", e.getMessage(), attributeName, attributeVal);
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java
index 6cefb9bca8b6090d3ff1f807da91050aab6ace3b..46e570767ee8a1676aa3119723fcf147276ebc6b 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java
@@ -14,11 +14,15 @@
package org.opengroup.osdu.indexer.service;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
@@ -31,7 +35,8 @@ public class ElasticSettingServiceImpl implements IElasticSettingService {
@Inject
private TenantInfo tenantInfo;
-
+ @Inject
+ private javax.inject.Provider<ITenantInfoService> tenantInfoServiceProvider;
@Inject
private IElasticRepository elasticRepository;
@Inject
@@ -43,18 +48,33 @@ public class ElasticSettingServiceImpl implements IElasticSettingService {
@Override
public ClusterSettings getElasticClusterInformation() {
+ return getClusterSettingsByTenantInfo(tenantInfo);
+ }
+
+ @Override
+ public Map<String, ClusterSettings> getAllClustersSettings() {
+ List<TenantInfo> tenantInfos = tenantInfoServiceProvider.get().getAllTenantInfos();
+ return tenantInfos.stream()
+ .collect(Collectors.toMap(TenantInfo::getDataPartitionId,
+ this::getClusterSettingsByTenantInfo));
+ }
+
+ private ClusterSettings getClusterSettingsByTenantInfo(TenantInfo tenantInfo) {
+ String cacheKey = String.format("%s-%s", configurationProperties.getGaeService(),
+ tenantInfo.getName());
- String cacheKey = String.format("%s-%s", configurationProperties.getGaeService(), tenantInfo.getName());
ClusterSettings clusterInfo = (ClusterSettings) this.elasticCredentialCache.get(cacheKey);
if (clusterInfo != null) {
return clusterInfo;
}
- log.warning(String.format("elastic-credential cache missed for tenant: %s", tenantInfo.getName()));
+ log.warning(String.format("elastic-credential cache missed for tenant: %s",
+ tenantInfo.getName()));
clusterInfo = this.elasticRepository.getElasticClusterSettings(tenantInfo);
if (clusterInfo == null) {
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Tenant not found", "No information about the given tenant was found");
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Tenant not found",
+ "No information about the given tenant was found");
}
this.elasticCredentialCache.put(cacheKey, clusterInfo);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
index a658cc6a1f289d5140cabb39d94864d4f173ef00..26ab87db89b860fb97c5b9c58a28fe3dc4328cfd 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
@@ -21,7 +21,7 @@ public interface IAttributeParsingService {
void tryParseDate(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
- void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap);
+ void tryParseGeopoint(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index a979c0868f074adc77c7bcaec6d51ed2ab1eac86..009247ba48e40da753c5c10a38b8a3d265e325e9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -236,6 +236,12 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
meta.put(RecordMetaAttribute.LEGAL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.LEGAL));
meta.put(RecordMetaAttribute.ANCESTRY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ANCESTRY));
meta.put(RecordMetaAttribute.INDEX_STATUS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.INDEX_STATUS));
+ meta.put(RecordMetaAttribute.AUTHORITY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.AUTHORITY));
+ meta.put(RecordMetaAttribute.SOURCE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.SOURCE));
+ meta.put(RecordMetaAttribute.CREATE_USER.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_USER));
+ meta.put(RecordMetaAttribute.CREATE_TIME.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_TIME));
+ meta.put(RecordMetaAttribute.MODIFY_USER.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_USER));
+ meta.put(RecordMetaAttribute.MODIFY_TIME.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_TIME));
String kind = schemaObj.getKind();
String type = kind.split(":")[2];
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 6f796030f9d6ec73e8e6675c6c24882c86646a14..fed2822af6d993b407e4d7502f0685d35e42a184 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.service;
+import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.http.HttpStatus;
@@ -129,8 +130,8 @@ public class IndexerServiceImpl implements IndexerService {
}
auditLogger.indexStarted(recordInfos.stream()
- .map(RecordInfo::getKind)
- .collect(Collectors.toList()));
+ .map(RecordInfo::getKind)
+ .collect(Collectors.toList()));
// process schema change messages
Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
@@ -306,10 +307,14 @@ public class IndexerServiceImpl implements IndexerService {
try {
// index individual parts of kind
String[] kindParts = storageRecord.getKind().split(":");
-
+ String authority = kindParts[0];
+ String source = kindParts[1];
+ String type = kindParts[2];
document.setKind(storageRecord.getKind());
- document.setNamespace(kindParts[0] + ":" + kindParts[1]);
- document.setType(kindParts[2]);
+ document.setNamespace(authority + ":" + source);
+ document.setAuthority(authority);
+ document.setSource(source);
+ document.setType(type);
document.setId(storageRecord.getId());
document.setVersion(storageRecord.getVersion());
document.setAcl(storageRecord.getAcl());
@@ -317,6 +322,14 @@ public class IndexerServiceImpl implements IndexerService {
if (storageRecord.getTags() != null) {
document.setTags(storageRecord.getTags());
}
+ document.setCreateUser(storageRecord.getCreateUser());
+ document.setCreateTime(storageRecord.getCreateTime());
+ if (!Strings.isNullOrEmpty(storageRecord.getModifyUser())) {
+ document.setModifyUser(storageRecord.getModifyUser());
+ }
+ if (!Strings.isNullOrEmpty(storageRecord.getModifyTime())) {
+ document.setModifyTime(storageRecord.getModifyTime());
+ }
RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
if (recordStatus.getIndexProgress().getStatusCode() == 0) {
recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
@@ -467,6 +480,8 @@ public class IndexerServiceImpl implements IndexerService {
indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
+ indexerPayload.put(RecordMetaAttribute.AUTHORITY.getValue(), record.getAuthority());
+ indexerPayload.put(RecordMetaAttribute.SOURCE.getValue(), record.getSource());
indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
@@ -478,6 +493,14 @@ public class IndexerServiceImpl implements IndexerService {
if (record.getAncestry() != null) {
indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
}
+ indexerPayload.put(RecordMetaAttribute.CREATE_USER.getValue(), record.getCreateUser());
+ indexerPayload.put(RecordMetaAttribute.CREATE_TIME.getValue(), record.getCreateTime());
+ if (!Strings.isNullOrEmpty(record.getModifyUser())) {
+ indexerPayload.put(RecordMetaAttribute.MODIFY_USER.getValue(), record.getModifyUser());
+ }
+ if (!Strings.isNullOrEmpty(record.getModifyTime())) {
+ indexerPayload.put(RecordMetaAttribute.MODIFY_TIME.getValue(), record.getModifyTime());
+ }
return indexerPayload;
}
@@ -512,13 +535,13 @@ public class IndexerServiceImpl implements IndexerService {
private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) {
List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType);
- if(!succeededRecords.isEmpty()) {
+ if (!succeededRecords.isEmpty()) {
successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList()));
}
List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType);
List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType);
failedRecords.addAll(skippedRecords);
- if(!failedRecords.isEmpty()) {
+ if (!failedRecords.isEmpty()) {
failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList()));
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index e550d4e0949cf2afb6458a341713d3b713106558..55759face83e4ba6cff8b67d3c97b41829eac5d5 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -123,7 +123,7 @@ public class StorageIndexerPayloadMapper {
this.attributeParsingService.tryParseDate(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
break;
case GEO_POINT:
- this.attributeParsingService.tryParseGeopoint(recordId, schemaPropertyName, storageRecordData, dataCollectorMap);
+ this.attributeParsingService.tryParseGeopoint(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
break;
case GEO_SHAPE:
this.attributeParsingService.tryParseGeojson(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerDocumentationConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerDocumentationConfig.java
index a969e816ea23f08d0f9c1dcba30ebb3ff35318c5..5811f6bb8b7e7239c81de143d1329c04ed98864b 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerDocumentationConfig.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerDocumentationConfig.java
@@ -31,21 +31,13 @@ public class SwaggerDocumentationConfig {
ParameterBuilder builder = new ParameterBuilder();
List<Parameter> parameters = new ArrayList<>();
builder.name(DpsHeaders.DATA_PARTITION_ID)
- .description("tenant")
+ .description("data partition id")
.defaultValue("common")
.modelRef(new ModelRef("string"))
.parameterType("header")
.required(true)
.build();
parameters.add(builder.build());
- builder.name("slb-frame-of-reference")
- .description("reference")
- .defaultValue("none")
- .modelRef(new ModelRef("string"))
- .parameterType("header")
- .required(true)
- .build();
- parameters.add(builder.build());
return new Docket(DocumentationType.SWAGGER_2)
.globalOperationParameters(parameters)
.select()
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/CloudConnectedOuterServicesBuilder.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/CloudConnectedOuterServicesBuilder.java
new file mode 100644
index 0000000000000000000000000000000000000000..13c0007bbaacb5a81aa6e2a4d6586652b3f09a25
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/CloudConnectedOuterServicesBuilder.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.info.ConnectedOuterServicesBuilder;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
+import org.opengroup.osdu.core.common.model.info.ConnectedOuterService;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@ConditionalOnMissingBean(type = "ConnectedOuterServicesBuilder")
+@Slf4j
+@RequestScope
+public class CloudConnectedOuterServicesBuilder implements ConnectedOuterServicesBuilder {
+
+ private static final String NAME_PREFIX = "ElasticSearch-";
+ private static final String REDIS_PREFIX = "Redis-";
+ private static final String NOT_AVAILABLE = "N/A";
+
+ private final List<RedisCache> redisCaches;
+ private final ElasticClientHandler elasticClient;
+ private final IElasticSettingService elasticSettingService;
+
+ public CloudConnectedOuterServicesBuilder(
+ List<RedisCache> redisCaches,
+ ElasticClientHandler elasticClient,
+ IElasticSettingService elasticSettingService) {
+ this.redisCaches = redisCaches;
+ this.elasticClient = elasticClient;
+ this.elasticSettingService = elasticSettingService;
+ }
+
+ @Override
+ public List<ConnectedOuterService> buildConnectedOuterServices() {
+ return Stream.concat(redisCaches.stream().map(this::fetchRedisInfo),
+ fetchElasticInfos().stream())
+ .collect(Collectors.toList());
+ }
+
+ private ConnectedOuterService fetchRedisInfo(RedisCache cache) {
+ String redisVersion = StringUtils.substringBetween(cache.info(), ":", "\r");
+ return ConnectedOuterService.builder()
+ .name(REDIS_PREFIX + StringUtils.substringAfterLast(cache.getClass().getName(), "."))
+ .version(redisVersion)
+ .build();
+ }
+
+ private List<ConnectedOuterService> fetchElasticInfos() {
+ try {
+ return elasticSettingService.getAllClustersSettings()
+ .entrySet().stream()
+ .map(entry -> fetchElasticInfo(entry.getKey(), entry.getValue()))
+ .collect(Collectors.toList());
+ } catch (AppException e) {
+ log.error("Can't fetch cluster settings", e.getOriginalException());
+ return Collections.singletonList(ConnectedOuterService.builder()
+ .name(NAME_PREFIX + NOT_AVAILABLE)
+ .version(NOT_AVAILABLE)
+ .build());
+ }
+ }
+
+ private ConnectedOuterService fetchElasticInfo(String partitionId, ClusterSettings settings) {
+ try (RestHighLevelClient client = elasticClient.createRestClient(settings)) {
+ return ConnectedOuterService.builder()
+ .name(NAME_PREFIX + partitionId)
+ .version(client.info(RequestOptions.DEFAULT).getVersion().getNumber())
+ .build();
+ } catch (AppException e) {
+ log.error("Can't create elastic client", e.getOriginalException());
+ return ConnectedOuterService.builder()
+ .name(NAME_PREFIX + partitionId)
+ .version(NOT_AVAILABLE)
+ .build();
+ } catch (IOException e) {
+ log.error("Can't fetch elastic info.", e);
+ return ConnectedOuterService.builder()
+ .name(NAME_PREFIX + partitionId)
+ .version(NOT_AVAILABLE)
+ .build();
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index 6a9a9dc4c30900a76fa811ac63315bce6358efb0..122711c9a2226d292fda4141a9c949dcc286fbf7 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -49,6 +49,12 @@ public class TypeMapper {
metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
+ metaAttributeIndexerType.put(RecordMetaAttribute.AUTHORITY.getValue(), ElasticType.CONSTANT_KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.SOURCE.getValue(), ElasticType.CONSTANT_KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.CREATE_USER.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.MODIFY_USER.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.CREATE_TIME.getValue(), ElasticType.DATE.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.MODIFY_TIME.getValue(), ElasticType.DATE.getValue());
storageToIndexerType.put(StorageType.LINK.getValue(), ElasticType.KEYWORD.getValue());
storageToIndexerType.put(StorageType.LINK_ARRAY.getValue(), ElasticType.KEYWORD_ARRAY.getValue());
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCoreTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCoreTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..aa53ea967f6a2ddfcc7324d46caabdffd82edf5f
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCoreTest.java
@@ -0,0 +1,93 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.error;
+
+import javassist.NotFoundException;
+import org.apache.http.HttpStatus;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.AccessDeniedException;
+
+import javax.validation.ValidationException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(PowerMockRunner.class)
+public class GlobalExceptionMapperCoreTest {
+
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private GlobalExceptionMapperCore sut;
+
+ @Test
+ public void should_useValuesInAppExceptionInResponse_When_AppExceptionIsHandledByGlobalExceptionMapper() {
+ AppException exception = new AppException(409, "any reason", "any message");
+
+ ResponseEntity<Object> response = sut.handleAppException(exception);
+ assertEquals(409, response.getStatusCodeValue());
+ assertEquals(exception.getError(), response.getBody());
+ }
+
+ @Test
+ public void should_use404ValueInResponse_When_NotFoundExceptionIsHandledByGlobalExceptionMapper() {
+ NotFoundException exception = new NotFoundException("any message");
+
+ ResponseEntity<Object> response = sut.handleNotFoundException(exception);
+ assertEquals(404, response.getStatusCodeValue());
+ assertTrue(response.getBody().toString().contains("any message"));
+ }
+
+ @Test
+ public void should_useGenericValuesInResponse_When_ExceptionIsHandledByGlobalExceptionMapper() {
+ Exception exception = new Exception("any message");
+
+ ResponseEntity<Object> response = sut.handleGeneralException(exception);
+ assertEquals(500, response.getStatusCodeValue());
+ assertEquals("AppError(code=500, reason=Server error., message=An unknown error has occurred., errors=null, debuggingInfo=null, originalException=java.lang.Exception: any message)", response.getBody().toString());
+ }
+
+ @Test
+ public void should_useBadRequestInResponse_When_handleValidationExceptionIsHandledByGlobalExceptionMapper() {
+ ValidationException exception = new ValidationException();
+
+ ResponseEntity<Object> response = sut.handleValidationException(exception);
+ assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCodeValue());
+ }
+
+ @Test
+ public void should_useBadRequestInResponse_When_handleAccessDeniedExceptionIsHandledByGlobalExceptionMapper() {
+ AccessDeniedException exception = new AccessDeniedException("Access is denied.");
+
+ ResponseEntity<Object> response = sut.handleAccessDeniedException(exception);
+ assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCodeValue());
+ }
+
+ @Test
+ public void should_useCustomErrorCodeInResponse_When_AppExceptionIsHandledByGlobalExceptionMapper() {
+ AppException exception = new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Successful Storage service response with wrong json");
+
+ ResponseEntity<Object> response = sut.handleAppException(exception);
+ assertEquals(RequestStatus.INVALID_RECORD, response.getStatusCodeValue());
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
index 7947615ebcaac6ce5850a29d87fd4299fe6d2dfb..f928fd5f89faa984683c12c94e6633d171b9f116 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
@@ -60,6 +60,11 @@ public class SchemaToStorageFormatImplTest {
testSingleFile("/converter/bad-schema/wrong-definitions-and-missed-type.json", KIND);
}
+ @Test(expected = SchemaProcessingException.class)
+ public void wrongArrayDefinitions() {
+ testSingleFile("/converter/bad-schema/wrong-array.json", KIND);
+ }
+
@Test
public void firstSchemaPassed() {
testSingleFile("/converter/basic/schema.json", "osdu:osdu:Wellbore:1.0.0");
@@ -120,6 +125,11 @@ public class SchemaToStorageFormatImplTest {
testSingleFile("/converter/tags/mixAllAnyOneOf/mix.json", KIND);
}
+ @Test
+ public void nestedIndexHints() {
+ testSingleFile("/converter/index-hints/nested-type-schema.json", "osdu:osdu:Wellbore:1.0.0");
+ }
+
@Test
public void folderPassed() throws URISyntaxException, IOException {
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
index 67362f02c6c74f2768e825a0f8063d4d9409ee4d..1372a8c45330ee2d8ecb600f664502776d456e6b 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
@@ -303,14 +303,11 @@ public class AttributeParsingServiceImplTest {
positionMap.put("longitude", 10.45);
positionMap.put("latitude", 90.0);
- Map<String, Object> storageData = new HashMap<>();
- storageData.put("location", positionMap);
-
when(this.geometryConversionService.tryGetGeopoint(positionMap)).thenReturn(positionMap);
Map<String, Object> dataMap = new HashMap<>();
- this.sut.tryParseGeopoint("", "location", storageData, dataMap);
+ this.sut.tryParseGeopoint("", "location", positionMap, dataMap);
assertFalse(dataMap.isEmpty());
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..6880026132541a16c108b2b90efa14c6a0c13395
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
@@ -0,0 +1,41 @@
+package org.opengroup.osdu.indexer.util;
+
+import org.junit.Test;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.StorageType;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+
+import static org.junit.Assert.assertEquals;
+
+public class TypeMapperTest {
+
+ @Test
+ public void getIndexerTypeTest() {
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(StorageType.LINK.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.KEYWORD_ARRAY.getValue(), TypeMapper.getIndexerType(StorageType.LINK_ARRAY.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.BOOLEAN.getValue(), TypeMapper.getIndexerType(StorageType.BOOLEAN.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.TEXT.getValue(), TypeMapper.getIndexerType(StorageType.STRING.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.INTEGER.getValue(), TypeMapper.getIndexerType(StorageType.INT.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.FLOAT.getValue(), TypeMapper.getIndexerType(StorageType.FLOAT.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.DOUBLE.getValue(), TypeMapper.getIndexerType(StorageType.DOUBLE.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.DOUBLE_ARRAY.getValue(), TypeMapper.getIndexerType(StorageType.DOUBLE_ARRAY.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.LONG.getValue(), TypeMapper.getIndexerType(StorageType.LONG.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(StorageType.DATETIME.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.GEO_POINT.getValue(), TypeMapper.getIndexerType(StorageType.GEO_POINT.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.GEO_SHAPE.getValue(), TypeMapper.getIndexerType(StorageType.GEO_SHAPE.getValue(), ElasticType.TEXT.getValue()));
+
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.KIND));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TYPE));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ID));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.NAMESPACE));
+ assertEquals(ElasticType.LONG.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.VERSION));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.X_ACL));
+
+ assertEquals(ElasticType.CONSTANT_KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.AUTHORITY));
+ assertEquals(ElasticType.CONSTANT_KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.SOURCE));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_USER));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_USER));
+ assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_TIME));
+ assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_TIME));
+ }
+}
diff --git a/indexer-core/src/test/resources/converter/bad-schema/wrong-array.json b/indexer-core/src/test/resources/converter/bad-schema/wrong-array.json
new file mode 100644
index 0000000000000000000000000000000000000000..741c9a424b8b1e44c0b5b6a7dbe1b04c1ff113e3
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/bad-schema/wrong-array.json
@@ -0,0 +1,312 @@
+{
+ "license": "Copyright 2017-2020, Schlumberger\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\nhttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "description": "The wellbore schema. Used to capture the general information about a wellbore. This information is sometimes called a \"wellbore header\". A wellbore represents the path from surface to a unique bottomhole location. The wellbore object is uniquely identified within the context of one well object.",
+ "title": "Wellbore",
+ "type": "object",
+ "definitions": {
+ "legal": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "legal",
+ "type": "object"
+ },
+ "metaItem": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "metaItem",
+ "type": "object"
+ },
+ "tagDictionary": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "tagDictionary",
+ "type": "object"
+ },
+ "linkList": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "link": "string"
+ }
+ }
+ },
+ "wellboreData": {
+ "description": "The domain specific data container for a wellbore.",
+ "title": "Wellbore Data",
+ "type": "object",
+ "properties": {
+ "WB_NAME": {
+ "description": "TBD",
+ "title": "Wellbore Name",
+ "type": "string",
+ "example": "SMP G09995 001S0B1"
+ },
+ "SPUD_DATE": {
+ "format": "date",
+ "description": "The date and time when activities to drill the borehole begin to create a hole in the earth. For a sidetrack, this is the date kickoff operations began. The format follows ISO 8601 YYYY-MM-DD extended format",
+ "title": "Spud Date",
+ "type": "string",
+ "example": "2013-03-22"
+ },
+ "TVD": {
+ "description": "TBD",
+ "title": "True Vertical Depth",
+ "type": "string",
+ "example": [
+ 20711,
+ "TBD"
+ ]
+ },
+ "PERMIT_NUMBER": {
+ "description": "Ther permit number for the wellbore",
+ "title": "Permit Number",
+ "type": "string",
+ "example": "SMP-09995"
+ },
+ "CRS": {
+ "description": "Wellbore location CRS",
+ "title": "CRS",
+ "type": "string",
+ "example": "World Geodetic System 1984"
+ },
+ "LONGUITUDE": {
+ "description": "TBD",
+ "title": "Longuitude",
+ "type": "number",
+ "example": [
+ -119.2,
+ "TBD"
+ ]
+ },
+ "STATE": {
+ "description": "The state, in which the wellbore is located.",
+ "title": "State",
+ "type": "string",
+ "example": [
+ "Texas"
+ ]
+ },
+ "CLASS": {
+ "description": "The current class of the wellbore",
+ "title": "class",
+ "type": "string",
+ "example": "NEW FIELD WILDCAT"
+ },
+ "WELLBORE_SHAPE": {
+ "description": "The shape of the wellbore",
+ "title": "Wellbore Shape",
+ "type": "string",
+ "example": [
+ "DIRECTIONAL",
+ "VERTICAL"
+ ]
+ },
+ "FORMATION_AT_TD": {
+ "description": "The formation name at the wellbore total depth",
+ "title": "Formation at TD",
+ "type": "string",
+ "example": "MIOCENE LOWER"
+ },
+ "PERMIT_DATE": {
+ "format": "date",
+ "description": "The date and time when the wellbore permit was issued. The format follows ISO 8601 YYYY-MM-DD extended format",
+ "title": "Permit Date",
+ "type": "string",
+ "example": "2013-03-22"
+ },
+ "STATUS": {
+ "description": "The current status of the wellbore",
+ "title": "Status",
+ "type": "string",
+ "example": "DRY & ABANDONED"
+ },
+ "COUNTRY": {
+ "description": "The country, in which the wellbore is located. The country name follows the convention in ISO 3166-1 'English short country name', see https://en.wikipedia.org/wiki/ISO_3166-1",
+ "title": "Country",
+ "type": "string",
+ "example": [
+ "United States of America"
+ ]
+ },
+ "WB_NUMBER": {
+ "description": "TBD",
+ "title": "Wellbore Number",
+ "type": "string",
+ "example": "001S0B1"
+ },
+ "MD": {
+ "description": "TBD",
+ "title": "Measured Depth",
+ "type": "string",
+ "example": "12.20"
+ },
+ "ORIGINAL_OPERATOR": {
+ "description": "The original operator of the wellbore.",
+ "title": "Original Operator",
+ "type": "string",
+ "example": "Anadarko Petroleum"
+ },
+ "BASIN": {
+ "description": "The basin name, to which the wellbore belongs.",
+ "title": "Basin",
+ "type": "string",
+ "example": "ATWATER"
+ },
+ "EPSG_CODE": {
+ "description": "EPSG code of the CRS",
+ "title": "EPSG Code",
+ "type": "string",
+ "example": "4326"
+ },
+ "COUNTY": {
+ "description": "The county, in which the wellbore is located.",
+ "title": "County",
+ "type": "string",
+ "example": [
+ "ATWATER VALLEY"
+ ]
+ },
+ "UNIT_SYSTEM": {
+ "description": "Unit system used for the wellbore measurements",
+ "title": "Unit Sustem",
+ "type": "string",
+ "example": "English"
+ },
+ "UWI": {
+ "description": "The unique wellbore identifier, aka. API number, US well number or UBHI. Codes can have 10, 12 or 14 digits depending on the availability of directional sidetrack (2 digits) and event sequence codes (2 digits).",
+ "title": "Unique Wellbore Identifier",
+ "type": "string",
+ "x-osdu-natural-key": 1,
+ "example": [
+ "SP435844921288",
+ "42-501-20130-01-02"
+ ]
+ },
+ "FIELD": {
+ "description": "The field name, to which the wellbore belongs.",
+ "title": "Field",
+ "type": "string",
+ "example": "ATWATER VLLY B 8"
+ },
+ "INITIAL_COMPLETION_DATE": {
+ "format": "date",
+ "description": "The date and time of the initial completion of the wellbore. The format follows ISO 8601 YYYY-MM-DD extended format",
+ "title": "Initial Completion Date",
+ "type": "string",
+ "example": "2013-03-22"
+ },
+ "ELEVATION": {
+ "description": "TBD",
+ "title": "Elevation",
+ "type": "string",
+ "example": [
+ 84,
+ "TBD"
+ ]
+ },
+ "STATUS_DATE": {
+ "format": "date",
+ "description": "The date and time of the current status of the wellbore. The format follows ISO 8601 YYYY-MM-DD extended format",
+ "title": "Status Date",
+ "type": "string",
+ "example": "2013-03-22"
+ },
+ "OPERATOR": {
+ "description": "The operator of the wellbore.",
+ "title": "Operator",
+ "type": "string",
+ "example": "Anadarko Petroleum"
+ },
+ "LEASE": {
+ "description": "The lease name, to which the wellbore belongs.",
+ "title": "LEASE",
+ "type": "string",
+ "example": "SMP G09995"
+ },
+ "API": {
+ "description": "Second parameter used for relationship tests",
+ "title": "Api relationship test",
+ "type": "string",
+ "example": "test-wellbore-api"
+ },
+ "LATITUDE": {
+ "description": "TBD",
+ "title": "Latitude",
+ "type": "number",
+ "example": [
+ 60.2,
+ "TBD"
+ ]
+ },
+ "ELEVATION_REF": {
+ "description": "Elevation reference used for the measurements",
+ "title": "Elevation reference",
+ "type": "string",
+ "example": "MSL"
+ },
+ "Wellbores": {
+ "pattern": ".*U1A1NjA3MDM2Mzk5MzUy:.*",
+ "description": "The Well ID reference.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "Wellbore",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "array"
+ }
+ },
+ "$id": "definitions/wellboreData"
+ }
+ },
+ "properties": {
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs.",
+ "title": "Ancestry",
+ "$ref": "#/definitions/linkList"
+ },
+ "data": {
+ "description": "Wellbore data container",
+ "title": "Wellbore Data",
+ "$ref": "#/definitions/wellboreData"
+ },
+ "kind": {
+ "description": "OSDU demo wellbore kind specification",
+ "title": "Wellbore Kind",
+ "type": "string"
+ },
+ "meta": {
+ "description": "The meta data section linking the 'unitKey', 'crsKey' to self-contained definitions (persistableReference)",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/metaItem"
+ }
+ },
+ "legal": {
+ "description": "The geological interpretation's legal tags",
+ "title": "Legal Tags",
+ "$ref": "#/definitions/legal"
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "#/definitions/tagDictionary"
+ },
+ "id": {
+ "description": "The unique identifier of the wellbore",
+ "title": "Wellbore ID",
+ "type": "string"
+ },
+ "type": {
+ "description": "The reference entity type as declared in common:metadata:entity:*.",
+ "title": "Entity Type",
+ "type": "string"
+ },
+ "version": {
+ "format": "int64",
+ "description": "The version number of this wellbore; set by the framework.",
+ "title": "Entity Version Number",
+ "type": "number",
+ "example": "1040815391631285"
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/index-hints/nested-type-schema.json b/indexer-core/src/test/resources/converter/index-hints/nested-type-schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..d729e0d350647cec7051e0a0d82a541bdfc82bcc
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/index-hints/nested-type-schema.json
@@ -0,0 +1,275 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "GenericDocumentArtefact",
+ "type": "object",
+ "definitions": {
+ "documentData": {
+ "description": "The domain specific data container for a document.",
+ "title": "Document Data",
+ "type": "object",
+ "properties": {
+ "Description": {
+ "type": "string",
+ "description": "Description. Summary of the work product component. Not the same as Remark which captures thoughts of creator about the wpc."
+ },
+ "keywords": {
+ "title": "title",
+ "type": "string",
+ "description": "Represents some user defined keywords"
+ },
+ "originalLocation": {
+ "title": "title",
+ "type": "string",
+ "description": "Represents location of the file in tenant cloud storage"
+ },
+ "wellboreId": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "wellbore",
+ "GroupType": "master-data"
+ }
+ ],
+ "title": "wellboe id",
+ "type": "string"
+ },
+ "classification": {
+ "type": "object",
+ "title": "File Properties",
+ "properties": {
+ "summary": {
+ "title": "summary",
+ "type": "string",
+ "description": "Represents document summary"
+ },
+ "petro": {
+ "description": "Represents petro",
+ "title": "petro",
+ "type": "Object",
+ "properties": {
+ "inferred": {
+ "description": "Represents value of inferred",
+ "title": "inferred",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "accepted": {
+ "description": "Represents value of accepted",
+ "title": "accepted",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "geopoliticalContext": {
+ "description": "Represents value of geopoliticalContext",
+ "title": "geopoliticalContext",
+ "type": "array",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "properties": {
+ "country": {
+ "title": "country",
+ "type": "string",
+ "description": "Represents country"
+ },
+ "taxNode": {
+ "title": "taxNode",
+ "type": "string",
+ "description": "Represents taxNode"
+ },
+ "country_region": {
+ "title": "country_region",
+ "type": "string",
+ "description": "Represents country_region"
+ },
+ "region": {
+ "title": "region",
+ "type": "string",
+ "description": "Represents region"
+ }
+ }
+ }
+ },
+ "otherTerms": {
+ "description": "Represents value of otherTerms",
+ "title": "otherTerms",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ambiguousTerms": {
+ "type": "object",
+ "properties": {}
+ },
+ "context": {
+ "type": "object",
+ "properties": {}
+ },
+ "taxNodes": {
+ "description": "Represents concept-tag collection",
+ "title": "concept-tags",
+ "type": "array",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "properties": {
+ "score": {
+ "title": "score",
+ "type": "number",
+ "description": "Represents score"
+ },
+ "taxNode": {
+ "title": "taxNode",
+ "type": "string",
+ "description": "Represents taxNode"
+ },
+ "taxonomy": {
+ "title": "taxonomy",
+ "type": "string",
+ "description": "Represents taxonomy"
+ },
+ "explanation": {
+ "title": "explanation",
+ "type": "string",
+ "description": "Represents explanation"
+ },
+ "cf-score": {
+ "title": "cf-score",
+ "type": "integer",
+ "description": "Represents cf-score"
+ }
+ }
+ }
+ },
+ "concept-tags": {
+ "description": "Represents concept-tag collection",
+ "title": "concept-tags",
+ "type": "array",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "properties": {
+ "score": {
+ "title": "score",
+ "type": "number",
+ "description": "Represents score"
+ },
+ "concept-tag": {
+ "title": "concept-tag",
+ "type": "string",
+ "description": "Represents concept-tag"
+ },
+ "ttype": {
+ "title": "tag type",
+ "type": "string",
+ "description": "Represents tag type"
+ }
+ }
+ }
+ },
+ "title": {
+ "title": "title",
+ "type": "string",
+ "description": "Represents document title"
+ }
+ }
+ },
+ "Source": {
+ "title": "Data Source",
+ "type": "string",
+ "description": "The entity that produced the record, or from which it is received; could be an organization, agency, system, internal team, or individual. For informational purposes only, the list of sources is not governed."
+ },
+ "Name": {
+ "type": "string",
+ "description": "Name"
+ },
+ "size": {
+ "title": "title",
+ "type": "string",
+ "description": "Represents size of the original file"
+ },
+ "extractedTextFilePath": {
+ "description": "Represents Cloud Storage path of the file containing extracted text from the document",
+ "title": "extractedTextFilePath",
+ "type": "string"
+ },
+ "thumbnailPath": {
+ "description": "Represents Cloud Storage path of the thumbnail size image of the document",
+ "title": "thumbnailPath",
+ "type": "string"
+ },
+ "SubTitle": {
+ "type": "string",
+ "description": "The sub-title of the document."
+ }
+ },
+ "$id": "definitions/documentData"
+ }
+ },
+ "properties": {
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs.",
+ "title": "Ancestry",
+ "$ref": "#/definitions/linkList"
+ },
+ "data": {
+ "description": "Document data container",
+ "title": "Document Data",
+ "$ref": "#/definitions/documentData"
+ },
+ "kind": {
+ "default": "osdu:wks:GenericDocumentArtefact:1.0.0",
+ "description": "OSDU demo document kind specification",
+ "title": "Document Kind",
+ "type": "string"
+ },
+ "meta": {
+ "description": "The meta data section linking the 'unitKey', 'crsKey' to self-contained definitions (persistableReference)",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/metaItem"
+ }
+ },
+ "legal": {
+ "description": "The geological interpretation's legal tags",
+ "title": "Legal Tags",
+ "$ref": "#/definitions/legal"
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "#/definitions/tagDictionary"
+ },
+ "id": {
+ "description": "The unique identifier of the document",
+ "title": "Document ID",
+ "type": "string"
+ },
+ "type": {
+ "description": "The reference entity type as declared in common:metadata:entity:*.",
+ "title": "Entity Type",
+ "type": "string"
+ },
+ "version": {
+ "format": "int64",
+ "description": "The version number of this document; set by the framework.",
+ "title": "Entity Version Number",
+ "type": "number",
+ "example": "1040815391631285"
+ }
+ },
+ "$id": "osdu:wks:DataSet--GenericDocumentArtefact:1.0.0"
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/index-hints/nested-type-schema.json.res b/indexer-core/src/test/resources/converter/index-hints/nested-type-schema.json.res
new file mode 100644
index 0000000000000000000000000000000000000000..9e8ce84cdf6f89655532576087f68e5d04849a39
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/index-hints/nested-type-schema.json.res
@@ -0,0 +1,132 @@
+{
+ "kind": "osdu:osdu:Wellbore:1.0.0",
+ "schema": [
+ {
+ "path": "Description",
+ "kind": "string"
+ },
+ {
+ "path": "keywords",
+ "kind": "string"
+ },
+ {
+ "path": "originalLocation",
+ "kind": "string"
+ },
+ {
+ "path": "wellboreId",
+ "kind": "string"
+ },
+ {
+ "path": "classification.summary",
+ "kind": "string"
+ },
+ {
+ "path": "classification.petro.inferred",
+ "kind": "[]string"
+ },
+ {
+ "path": "classification.petro.accepted",
+ "kind": "[]string"
+ },
+ {
+ "path": "classification.petro.geopoliticalContext",
+ "kind": "nested",
+ "properties": [
+ {
+ "path": "country",
+ "kind": "string"
+ },
+ {
+ "path": "taxNode",
+ "kind": "string"
+ },
+ {
+ "path": "country_region",
+ "kind": "string"
+ },
+ {
+ "path": "region",
+ "kind": "string"
+ }
+ ]
+ },
+ {
+ "path": "classification.petro.otherTerms",
+ "kind": "[]string"
+ },
+ {
+ "path": "classification.taxNodes",
+ "kind": "nested",
+ "properties": [
+ {
+ "path": "score",
+ "kind": "double"
+ },
+ {
+ "path": "taxNode",
+ "kind": "string"
+ },
+ {
+ "path": "taxonomy",
+ "kind": "string"
+ },
+ {
+ "path": "explanation",
+ "kind": "string"
+ },
+ {
+ "path": "cf-score",
+ "kind": "int"
+ }
+ ]
+ },
+ {
+ "path": "classification.concept-tags",
+ "kind": "nested",
+ "properties": [
+ {
+ "path": "score",
+ "kind": "double"
+ },
+ {
+ "path": "concept-tag",
+ "kind": "string"
+ },
+ {
+ "path": "ttype",
+ "kind": "string"
+ }
+ ]
+ },
+ {
+ "path": "classification.title",
+ "kind": "string"
+ },
+ {
+ "path": "Source",
+ "kind": "string"
+ },
+ {
+ "path": "Name",
+ "kind": "string"
+ },
+ {
+ "path": "size",
+ "kind": "string"
+ },
+ {
+ "path": "extractedTextFilePath",
+ "kind": "string"
+ },
+ {
+ "path": "thumbnailPath",
+ "kind": "string"
+ },
+ {
+ "path": "SubTitle",
+ "kind": "string"
+ }
+ ]
+}
+
diff --git a/pom.xml b/pom.xml
index dc99cd5e0ac97e2ef22e9029dca6b90316497150..a59ee5d608b24071b98192b249c169d14a3aa4fa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<java.version>1.8</java.version>
<springfox-version>2.7.0</springfox-version>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.10.0</os-core-common.version>
+ <os-core-common.version>0.11.0-rc4</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.4</jackson.version>
@@ -174,5 +174,45 @@
</profile>
</profiles>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>build-info</id>
+ <goals>
+ <goal>build-info</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>pl.project13.maven</groupId>
+ <artifactId>git-commit-id-plugin</artifactId>
+ <version>4.0.5</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>revision</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <verbose>true</verbose>
+ <dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
+ <generateGitPropertiesFile>true</generateGitPropertiesFile>
+ <generateGitPropertiesFilename>
+ ${project.build.outputDirectory}/git.properties
+ </generateGitPropertiesFilename>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantInfoService.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantInfoService.java
new file mode 100644
index 0000000000000000000000000000000000000000..16f82e4f309ce82d56a675351517240d37ded27e
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantInfoService.java
@@ -0,0 +1,33 @@
+package org.opengroup.osdu.indexer.aws.di;
+
+import java.util.ArrayList;
+import java.util.List;
+import javax.inject.Inject;
+
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@RequestScope
+@Component
+public class TenantInfoService implements ITenantInfoService {
+
+ @Inject
+ private ITenantFactory tenantFactory;
+
+ @Inject
+ private DpsHeaders headers;
+
+ @Override
+ public TenantInfo getTenantInfo() {
+ return tenantFactory.getTenantInfo(headers.getPartitionId());
+ }
+
+ @Override
+ public List<TenantInfo> getAllTenantInfos() {
+ return new ArrayList<>(tenantFactory.listTenantInfo());
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md
index fc52a8e9698754035054a79e0bd1656d2a02575d..8f12c581017ae8e3ae0d9d264107fab0eb8f97d1 100644
--- a/provider/indexer-azure/README.md
+++ b/provider/indexer-azure/README.md
@@ -10,7 +10,7 @@ In order to run this service locally, you will need the following:
- [Maven 3.6.0+](https://maven.apache.org/download.cgi)
- [AdoptOpenJDK8](https://adoptopenjdk.net/)
-- Infrastructure dependencies, deployable through the relevant [infrastructure template](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/infrastructure-templates?path=%2Finfra&version=GBmaster&_a=contents)
+- Infrastructure dependencies, deployable through the relevant [infrastructure template](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/blob/master/docs/service-automation.md)
- While not a strict dependency, example commands in this document use [bash](https://www.gnu.org/software/bash/)
### General Tips
diff --git a/provider/indexer-azure/maven/settings.xml b/provider/indexer-azure/maven/settings.xml
deleted file mode 100644
index d1af8b94a57f11b702675b23a2e5c8555bd73aed..0000000000000000000000000000000000000000
--- a/provider/indexer-azure/maven/settings.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Copyright © Microsoft Corporation
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
- <servers>
- <server>
- <id>os-core</id>
- <username>slb-des-ext-collaboration</username>
- <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
- <!-- The generated token expires on or before 11/14/2019 -->
- <password>${VSTS_FEED_TOKEN}</password>
- </server>
- <server>
- <id>azure-auth</id>
- <configuration>
- <tenant>${AZURE_DEPLOY_TENANT}</tenant>
- <client>${AZURE_DEPLOY_CLIENT_ID}</client>
- <key>${AZURE_DEPLOY_CLIENT_SECRET}</key>
- <environment>AZURE</environment>
- </configuration>
- </server>
- </servers>
-</settings>
\ No newline at end of file
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index a91cc0902f87daddd54c4e417dd99504c5809e8d..e72f50d6d77e850e0b6375742d32de1bd01cb4b1 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -184,6 +184,12 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>${osdu.corelibazure.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoService.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoService.java
index b54253284aa6cfb11e69f0eb776bd8c2897c4e93..53bcd253133745122a4770b868bf92bfd91ff10c 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoService.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoService.java
@@ -14,6 +14,8 @@
package org.opengroup.osdu.indexer.azure.di;
+import java.util.ArrayList;
+import java.util.List;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
@@ -36,4 +38,9 @@ public class TenantInfoService implements ITenantInfoService {
public TenantInfo getTenantInfo() {
return tenantFactory.getTenantInfo(headers.getPartitionId());
}
+
+ @Override
+ public List<TenantInfo> getAllTenantInfos() {
+ return new ArrayList<>(tenantFactory.listTenantInfo());
+ }
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
index ff1bd512a2b333947365187500254b98f3209087..ce59f2e5c65b6aa3aea5361b7bb1a11c2cd1f690 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
@@ -44,6 +44,7 @@ public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
"/swagger-resources/**",
"/configuration/security",
"/swagger",
+ "/info",
"/swagger-ui.html",
"/webjars/**").permitAll()
.anyRequest().authenticated()
diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md
index 8fee43caf512cb0347543246e7724bb53ad82aaa..842139cfd3e3ec77b6ecf15559f63beb6f7c290c 100644
--- a/provider/indexer-gcp/README.md
+++ b/provider/indexer-gcp/README.md
@@ -20,14 +20,13 @@ In order to run the service locally or remotely, you will need to have the follo
| `LOG_PREFIX` | `service` | Logging prefix | no | - |
| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - |
| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
-| `ENTITLEMENTS_HOST` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment |
-| `INDEXER_HOST` | ex `os-indexer-dot-opendes.appspot.com` | Indexer Host | no | output of infrastructure deployment |
| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment |
| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances |
| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment |
| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials |
| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
@@ -36,6 +35,8 @@ In order to run the service locally or remotely, you will need to have the follo
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
+| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment |
+| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment |
### Run Locally
Check that maven is installed:
@@ -131,14 +132,12 @@ You will need to have the following environment variables defined.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
-| `ENTITLEMENTS_HOST` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment |
| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-| `DATA_GROUP` | `opendes` | The service account to this group and substitute | no | - |
| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
| `INTEGRATION_TEST_AUDIENCE` | `********` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
@@ -146,11 +145,8 @@ You will need to have the following environment variables defined.
| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
-| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint | Storage Host | no | output of infrastructure deployment |
+| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
-| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
-
**Entitlements configuration for integration accounts**
@@ -201,9 +197,9 @@ Create king ring and key in the ***master project***
--purpose encryption
```
-Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the ***master project*** through IAM - Role tab
+Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab
-Add **Cloud KMS Encrypt/Decrypt** role to the **App Engine default service account** of ***master project***
+Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab
#### Memory Store (Redis Instance) Setup
diff --git a/provider/indexer-gcp/maven/settings.xml b/provider/indexer-gcp/maven/settings.xml
deleted file mode 100644
index 2a633d9a30c1997609060dcc31667d3dc1b6774a..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/maven/settings.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
- <servers>
- <server>
- <id>os-core</id>
- <username>slb-des-ext-collaboration</username>
- <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
- <!-- The generated token expires on or before 11/14/2019 -->
- <password>${VSTS_FEED_TOKEN}</password>
- </server>
- </servers>
-</settings>
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java
new file mode 100644
index 0000000000000000000000000000000000000000..84b3ca3597ebb4a046ce39c2d9588f56f6bc4e7c
--- /dev/null
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java
@@ -0,0 +1,33 @@
+package org.opengroup.osdu.indexer.di;
+
+import java.util.ArrayList;
+import java.util.List;
+import javax.inject.Inject;
+
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@RequestScope
+@Component
+public class TenantInfoService implements ITenantInfoService {
+
+ @Inject
+ private ITenantFactory tenantFactory;
+
+ @Inject
+ private DpsHeaders headers;
+
+ @Override
+ public TenantInfo getTenantInfo() {
+ return tenantFactory.getTenantInfo(headers.getPartitionId());
+ }
+
+ @Override
+ public List<TenantInfo> getAllTenantInfos() {
+ return new ArrayList<>(tenantFactory.listTenantInfo());
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
similarity index 81%
rename from provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
rename to provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
index 144edee0eb4c1bd99054bd1a1859588adb9be3ae..9a6dc9ef7e3920668aa2155c1c39dc8531ed3529 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
@@ -26,23 +26,18 @@ import java.util.HashMap;
import java.util.Map;
import javax.inject.Inject;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.threeten.bp.Duration;
-@Component
+@Component("indexerDatastoreFactory")
public class DatastoreFactory {
@Inject
private DatastoreCredentialCache cache;
- @Autowired
- public DatastoreFactory(DatastoreCredentialCache cache) {
- this.cache = cache;
- }
-
- private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>();
+ private static Map<String, Datastore> datastoreClients = new HashMap<>();
private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
.setMaxAttempts(6)
@@ -60,16 +55,16 @@ public class DatastoreFactory {
.build();
public Datastore getDatastoreInstance(TenantInfo tenantInfo) {
- if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) {
+ if (datastoreClients.get(tenantInfo.getName()) == null) {
Datastore googleDatastore = DatastoreOptions.newBuilder()
- .setCredentials(new DatastoreCredential(tenantInfo, this.cache))
+ .setCredentials(new DatastoreCredential(tenantInfo))
.setRetrySettings(RETRY_SETTINGS)
.setTransportOptions(TRANSPORT_OPTIONS)
.setNamespace(tenantInfo.getName())
.setProjectId(tenantInfo.getProjectId())
.build().getService();
- DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore);
+ datastoreClients.put(tenantInfo.getName(), googleDatastore);
}
- return DATASTORE_CLIENTS.get(tenantInfo.getName());
+ return datastoreClients.get(tenantInfo.getName());
}
}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java
index 7e773c20b9c7f12bec2878c7c3ef4d3d349f8ab1..a8e3373f44926527a271d4d4390b5f4d5e647177 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java
@@ -26,7 +26,6 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.core.gcp.multitenancy.DatastoreFactory;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.springframework.stereotype.Component;
@@ -52,7 +51,7 @@ public class ElasticRepositoryDatastore implements IElasticRepository {
@Override
public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
- Datastore googleDatastore = this.datastoreFactory.getDatastore(tenantInfo);
+ Datastore googleDatastore = this.datastoreFactory.getDatastoreInstance(tenantInfo);
Key key = googleDatastore.newKeyFactory().setKind(properties.getElasticDatastoreKind()).newKey(properties.getElasticDatastoreId());
Entity datastoreEntity = googleDatastore.get(key);
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
index 8be38e24bb075a6042ecf6aaafe312d97bb177b1..0f70a40bc444ae9fd7e1553c319684df1286273b 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
@@ -40,6 +40,7 @@ public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter {
@Override
public void configure(WebSecurity web) throws Exception {
web.ignoring().antMatchers("/api-docs")
+ .antMatchers("/info")
.antMatchers("/swagger");
}
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index 3ffb6c735078a39f5b1799e540b9a7e0abd3a146..6b70e9a7122fc83890ca55124f79aba69a197f22 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -31,7 +31,7 @@
<packaging>jar</packaging>
<properties>
- <os-core-lib-ibm.version>0.7.1</os-core-lib-ibm.version>
+ <os-core-lib-ibm.version>0.9.0</os-core-lib-ibm.version>
</properties>
<profiles>
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoService.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoService.java
index 9bb0fe9675a7ec8c1b9f1da1610cab9733edda21..041d9a6d315bfbe67618436ffb0fccf2745c5b25 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoService.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoService.java
@@ -3,6 +3,8 @@
package org.opengroup.osdu.indexer.ibm.di;
+import java.util.ArrayList;
+import java.util.List;
import javax.inject.Inject;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
@@ -25,4 +27,9 @@ public class TenantInfoService implements ITenantInfoService {
public TenantInfo getTenantInfo() {
return tenantFactory.getTenantInfo(headers.getPartitionId());
}
+
+ @Override
+ public List<TenantInfo> getAllTenantInfos() {
+ return new ArrayList<>(tenantFactory.listTenantInfo());
+ }
}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java
index 475ba059435176cc018af71a868fde9e7eaf9397..d0737cfecfcb767083f8ade4939234f6bfc8da5c 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java
@@ -26,6 +26,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
"/swagger-resources/**",
"/configuration/security",
"/swagger",
+ "/info",
"/swagger-ui.html",
"/webjars/**").permitAll()
.anyRequest().anonymous();
diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties
index 38a394adc7fc3c8230acc4a554212832c3e8861f..1e3ac9496d6694cfe536d328681b201dea528004 100644
--- a/provider/indexer-ibm/src/main/resources/application.properties
+++ b/provider/indexer-ibm/src/main/resources/application.properties
@@ -66,6 +66,4 @@ ELASTIC_DATASTORE_ID=indexer-service
ELASTIC_HOST=elasticsearch.com
ELASTIC_PORT=443
-ELASTIC_USER_PASSWORD=REPLACE_ME:REPLACE_ME
-
-
+ELASTIC_USER_PASSWORD=REPLACE_ME:REPLACE_ME
\ No newline at end of file
diff --git a/provider/indexer-reference/README.md b/provider/indexer-reference/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..9fa7b2bb954f311e570265d8734c2fb0e989fe01
--- /dev/null
+++ b/provider/indexer-reference/README.md
@@ -0,0 +1,217 @@
+# Indexer Service
+os-indexer-reference is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU hybrid cloud searches against Elasticsearch.
+
+## Getting Started
+These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
+
+### Prerequisites
+Pre-requisites
+
+* GCloud SDK with java (latest version)
+* JDK 8
+* Lombok 1.16 or later
+* Maven
+
+### Installation
+In order to run the service locally or remotely, you will need to have the following environment variables defined.
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - |
+| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
+| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment |
+| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment |
+| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment |
+| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials |
+| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
+| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment |
+| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment |
+| `MONGO_DB_URL` | ex `mongodb://localhost:27017` | Mongo DB Url| yes | output of infrastructure deployment |
+| `MONGO_DB_USER` | ex `mongouser` | Mongo DB userName| yes | output of infrastructure deployment |
+| `MONGO_DB_PASSWORD` | ex `mongopassword` | Mongo DB userPassword| yes | output of infrastructure deployment |
+| `MONGO_DB_NAME` | ex `mongoDBName` | Mongo DB DbName| yes | output of infrastructure deployment |
+| `MB_RABBITMQ_URI` | ex `amqp://guest:guest@127.0.0.1:5672` | MessageBroker RabbitMQ URI | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+
+### Run Locally
+Check that maven is installed:
+
+```bash
+$ mvn --version
+Apache Maven 3.6.0
+Maven home: /usr/share/maven
+Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre
+...
+```
+
+You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.mvn/community-maven.settings.xml`:
+
+```bash
+$ cat ~/.m2/settings.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+ <servers>
+ <server>
+ <id>community-maven-via-private-token</id>
+ <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
+ <!-- The generated token expires on or before 11/14/2019 -->
+ <configuration>
+ <httpHeaders>
+ <property>
+ <name>Private-Token</name>
+ <value>${env.COMMUNITY_MAVEN_TOKEN}</value>
+ </property>
+ </httpHeaders>
+ </configuration>
+ </server>
+ </servers>
+</settings>
+```
+
+* Update the Google cloud SDK to the latest version:
+
+```bash
+gcloud components update
+```
+* Set Google Project Id:
+
+```bash
+gcloud config set project <YOUR-PROJECT-ID>
+```
+
+* Perform a basic authentication in the selected project:
+
+```bash
+gcloud auth application-default login
+```
+
+* Navigate to indexer service's root folder and run:
+
+```bash
+mvn jetty:run
+## Testing
+* Navigate to indexer service's root folder and run:
+
+```bash
+mvn clean install
+```
+
+* If you wish to see the coverage report then go to testing/target/site/jacoco-aggregate and open index.html
+
+* If you wish to build the project without running tests
+
+```bash
+mvn clean install -DskipTests
+```
+
+After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
+
+```bash
+cd provider/indexer-reference/ && mvn spring-boot:run
+```
+
+## Testing
+Navigate to indexer service's root folder and run all the tests:
+
+```bash
+# build + install integration test core
+$ (cd testing/indexer-test-core/ && mvn clean install)
+```
+
+### Running E2E Tests
+This section describes how to run cloud OSDU E2E tests (testing/integration-tests/indexer-test-gcp).
+
+You will need to have the following environment variables defined.
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
+| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment |
+| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
+| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
+| `INTEGRATION_TEST_AUDIENCE` | `********` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
+| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
+| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
+| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+
+**Entitlements configuration for integration accounts**
+
+| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
+| --- | --- |
+| users<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
+
+Execute following command to build code and run all the integration tests:
+
+```bash
+# Note: this assumes that the environment variables for integration tests as outlined
+# above are already exported in your environment.
+$ (cd testing/indexer-test-gcp/ && mvn clean test)
+```
+
+## Deployment
+GKE Google Documentation: https://cloud.google.com/build/docs/deploying-builds/deploy-gke
+Anthos Google Documentation: https://cloud.google.com/anthos/multicluster-management/gateway/tutorials/cloud-build-integration
+
+#### Cloud KMS Setup
+
+Enable cloud KMS on master project
+
+Create king ring and key in the ***master project***
+
+```bash
+ gcloud services enable cloudkms.googleapis.com
+ export KEYRING_NAME="csqp"
+ export CRYPTOKEY_NAME="searchService"
+ gcloud kms keyrings create $KEYRING_NAME --location global
+ gcloud kms keys create $CRYPTOKEY_NAME --location global \
+ --keyring $KEYRING_NAME \
+ --purpose encryption
+```
+
+Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab
+
+Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab
+
+#### Memory Store (Redis Instance) Setup
+
+Create a new Standard tier Redis instance on the ***service project***
+
+The Redis instance must be created under the same region with the App Engine application which needs to access it.
+
+```bash
+ gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD
+```
+
+## Licence
+Copyright © Google LLC
+Copyright © EPAM Systems
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
\ No newline at end of file
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
index 3aa33b3733a74817ef0ea27c5cd52ca8e11ea288..76da4d50f8b5df385f6862ac06a943c3b85b6619 100644
--- a/provider/indexer-reference/pom.xml
+++ b/provider/indexer-reference/pom.xml
@@ -42,12 +42,13 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.1.17</version>
+ <version>0.10.0</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
+ <version>0.11.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -84,17 +85,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
</dependency>
<!-- Test Dependencies -->
@@ -165,7 +163,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-actuator-autoconfigure</artifactId>
</dependency>
-
</dependencies>
<build>
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
index d2c3e68f06d2da43a8988aa470b58d2002f36888..be11d6bbb0f311e1b9bef0e1e20adf96761d3595 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
@@ -17,7 +17,6 @@
package org.opengroup.osdu.indexer;
-import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -25,13 +24,11 @@ import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class,
ManagementWebSecurityAutoConfiguration.class})
@Configuration
-@ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = {
- @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)})
+@ComponentScan(value = {"org.opengroup.osdu"})
public class IndexerAnthosApplication {
public static void main(String[] args) {
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
index 6e832b44671a8ac95b65556076bf15122c6990a3..6a3b330aad0dcc5a7ba09687a82e0c5e1f53548c 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
@@ -1,19 +1,16 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
package org.opengroup.osdu.indexer.cache;
@@ -26,12 +23,8 @@ import org.springframework.stereotype.Component;
@Component
public class DatastoreCredentialCache extends RedisCache<String, AccessToken> {
- @Autowired
- public DatastoreCredentialCache(IndexerConfigurationProperties indexerConfigurationProperties) {
- super(indexerConfigurationProperties.getRedisSearchHost(),
- Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
- 58 * 60,
- String.class,
- AccessToken.class);
- }
+ @Autowired
+ public DatastoreCredentialCache(final IndexerConfigurationProperties configurationProperties) {
+ super(configurationProperties.getRedisSearchHost(), Integer.parseInt(configurationProperties.getRedisSearchPort()), 58 * 60, String.class, AccessToken.class);
+ }
}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..6f4d9b7505504f259ceca3d3b8e1834dc53f91ba
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java
@@ -0,0 +1,39 @@
+/*
+ Copyright 2020 Google LLC
+ Copyright 2020 EPAM Systems, Inc
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.stereotype.Component;
+
+@Component
+public class DatastoreCredentialsCacheFactory extends
+ AbstractFactoryBean<ICache<String, DatastoreCredential>> {
+
+ @Override
+ public Class<?> getObjectType() {
+ return ICache.class;
+ }
+
+ @Override
+ protected ICache<String, DatastoreCredential> createInstance() throws Exception {
+ return new VmCache<>(5 * 60, 20);
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java
deleted file mode 100644
index b60b6e590e5ec671110e74fa49949391c219ce23..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-
-import com.google.gson.Gson;
-import com.mongodb.client.FindIterable;
-import com.mongodb.client.MongoCollection;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import org.bson.Document;
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.indexer.persistence.MongoDdmsClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-
-@Primary
-@Component
-public class TenantFactoryImpl implements ITenantFactory {
-
- private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class);
-
- public static final String MAIN_DATABASE = "main";
- public static final String TENANT_INFO = "tenantinfo";
-
-
- private final MongoDdmsClient mongoClient;
-
- @Autowired
- public TenantFactoryImpl(MongoDdmsClient mongoClient) {
- this.mongoClient = mongoClient;
- }
-
- private Map<String, TenantInfo> tenants;
-
- public boolean exists(String tenantName) {
- if (this.tenants == null) {
- initTenants();
- }
- return this.tenants.containsKey(tenantName);
- }
-
- public TenantInfo getTenantInfo(String tenantName) {
- if (this.tenants == null) {
- initTenants();
- }
- return this.tenants.get(tenantName);
- }
-
- public Collection<TenantInfo> listTenantInfo() {
- if (this.tenants == null) {
- initTenants();
- }
- return this.tenants.values();
- }
-
- public <V> ICache<String, V> createCache(String tenantName, String host, int port,
- int expireTimeSeconds, Class<V> classOfV) {
- return null;
- }
-
- public void flushCache() {
- }
-
- private void initTenants() {
- this.tenants = new HashMap<>();
- MongoCollection<Document> mongoCollection = mongoClient
- .getMongoCollection(MAIN_DATABASE, TENANT_INFO);
- FindIterable<Document> results = mongoCollection.find();
- if (Objects.isNull(results) && Objects.isNull(results.first())) {
- LOG.error(String.format("Collection \'%s\' is empty.", results));
- }
- for (Document document : results) {
- TenantInfo tenantInfo = new Gson().fromJson(document.toJson(), TenantInfo.class);
- this.tenants.put(tenantInfo.getName(), tenantInfo);
- }
- }
-
-}
-
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java
new file mode 100644
index 0000000000000000000000000000000000000000..6ed88c4f209ec55fcc1043d019c67d498c5d36c9
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Log
+@Component
+@RequestScope
+public class TenantFactoryService extends AbstractFactoryBean<ITenantFactory> {
+
+ @Override
+ protected ITenantFactory createInstance() throws Exception {
+ return new TenantFactory();
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return ITenantFactory.class;
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java
new file mode 100644
index 0000000000000000000000000000000000000000..095c346f0f30ce28d1a4a603babc318fd61a792c
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java
@@ -0,0 +1,33 @@
+package org.opengroup.osdu.indexer.di;
+
+import java.util.ArrayList;
+import java.util.List;
+import javax.inject.Inject;
+
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@RequestScope
+@Component
+public class TenantInfoService implements ITenantInfoService {
+
+ @Inject
+ private ITenantFactory tenantFactory;
+
+ @Inject
+ private DpsHeaders headers;
+
+ @Override
+ public TenantInfo getTenantInfo() {
+ return tenantFactory.getTenantInfo(headers.getPartitionId());
+ }
+
+ @Override
+ public List<TenantInfo> getAllTenantInfos() {
+ return new ArrayList<>(tenantFactory.listTenantInfo());
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java
deleted file mode 100644
index 9b7f8b1f5edd751f35dcd850ec57073c594f838f..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.persistence;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.iam.v1.Iam;
-import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt;
-import com.google.api.services.iam.v1.IamScopes;
-import com.google.api.services.iam.v1.model.SignJwtRequest;
-import com.google.api.services.iam.v1.model.SignJwtResponse;
-import com.google.auth.oauth2.AccessToken;
-import com.google.auth.oauth2.GoogleCredentials;
-import com.google.gson.JsonObject;
-import java.util.Collections;
-import java.util.Date;
-import org.apache.commons.lang3.time.DateUtils;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.util.Crc32c;
-import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
-
-public class DatastoreCredential extends GoogleCredentials {
-
- private static final long serialVersionUID = 8344377091688956815L;
- private static final JsonFactory JSON_FACTORY = new JacksonFactory();
- private Iam iam;
-
- private final TenantInfo tenant;
- private final DatastoreCredentialCache cache;
-
- protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) {
- this.tenant = tenant;
- this.cache = cache;
- }
-
- @Override
- public AccessToken refreshAccessToken() {
-
- String cacheKey = this.getCacheKey();
-
- AccessToken accessToken = this.cache.get(cacheKey);
-
- if (accessToken != null) {
- return accessToken;
- }
-
- try {
- SignJwtRequest signJwtRequest = new SignJwtRequest();
- signJwtRequest.setPayload(this.getPayload());
-
- String serviceAccountName = String
- .format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount());
-
- SignJwt signJwt = this.getIam().projects().serviceAccounts()
- .signJwt(serviceAccountName, signJwtRequest);
-
- SignJwtResponse signJwtResponse = signJwt.execute();
- String signedJwt = signJwtResponse.getSignedJwt();
-
- accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600));
-
- this.cache.put(cacheKey, accessToken);
-
- return accessToken;
- } catch (Exception e) {
- throw new RuntimeException("Error creating datastore credential", e);
- }
- }
-
- private String getPayload() {
- JsonObject payload = new JsonObject();
- payload.addProperty("iss", this.tenant.getServiceAccount());
- payload.addProperty("sub", this.tenant.getServiceAccount());
- payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore");
- payload.addProperty("iat", System.currentTimeMillis() / 1000);
-
- return payload.toString();
- }
-
- protected void setIam(Iam iam) {
- this.iam = iam;
- }
-
- private Iam getIam() throws Exception {
- if (this.iam == null) {
-
- GoogleCredential credential = GoogleCredential.getApplicationDefault();
- if (credential.createScopedRequired()) {
- credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM));
- }
-
- Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(),
- JSON_FACTORY,
- credential).setApplicationName("Search Service");
-
- this.iam = builder.build();
- }
- return this.iam;
- }
-
-
- private String getCacheKey() {
- return Crc32c
- .hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName()));
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java
similarity index 94%
rename from provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
rename to provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java
index 0db2f9e557efcc69fbd0419df57fc3a821e024de..9096cbd6eddbbbee6cd7e63c12506bcf06f40dca 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java
@@ -27,7 +27,7 @@ import org.springframework.security.config.annotation.web.configuration.WebSecur
@Configuration
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
-public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter {
+public class SecurityConfig extends WebSecurityConfigurerAdapter {
@Override
protected void configure(HttpSecurity http) throws Exception {
@@ -38,6 +38,7 @@ public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter {
@Override
public void configure(WebSecurity web) throws Exception {
web.ignoring().antMatchers("/api-docs")
+ .antMatchers("/info")
.antMatchers("/swagger");
}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 1be2ba81868c6f3ed56cd4b56a25edfdbd576f99..8f72ac9ec6878d86d9e21bb7e7c789aef72cc142 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -64,7 +64,6 @@ import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
@Service
@@ -142,6 +141,10 @@ public class IndexerServiceImpl implements IndexerService {
retryRecordIds.addAll(deleteFailureRecordIds);
}
+ auditLogger.indexStarted(recordInfos.stream()
+ .map(RecordInfo::getKind)
+ .collect(Collectors.toList()));
+
// process schema change messages
Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
@@ -245,8 +248,13 @@ public class IndexerServiceImpl implements IndexerService {
for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
String kind = entry.getKey();
- IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false);
- if (schemaObj.isDataSchemaMissing()) {
+ List<String> errors = new ArrayList<>();
+ IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, errors);
+ if (!errors.isEmpty()) {
+ this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN,
+ HttpStatus.SC_BAD_REQUEST, String.join("|", errors),
+ String.format("error | kind: %s", kind));
+ } else if (schemaObj.isDataSchemaMissing()) {
this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN,
HttpStatus.SC_NOT_FOUND, "schema not found",
String.format("schema not found | kind: %s", kind));
@@ -360,6 +368,9 @@ public class IndexerServiceImpl implements IndexerService {
document.setVersion(storageRecord.getVersion());
document.setAcl(storageRecord.getAcl());
document.setLegal(storageRecord.getLegal());
+ if (storageRecord.getTags() != null) {
+ document.setTags(storageRecord.getTags());
+ }
RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
if (recordStatus.getIndexProgress().getStatusCode() == 0) {
recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
@@ -439,11 +450,11 @@ public class IndexerServiceImpl implements IndexerService {
String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
if (operation == OperationType.create) {
- IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId())
+ IndexRequest indexRequest = new IndexRequest(index).id(record.getId())
.source(this.gson.toJson(sourceMap), XContentType.JSON);
bulkRequest.add(indexRequest);
} else if (operation == OperationType.update) {
- UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId())
+ UpdateRequest updateRequest = new UpdateRequest(index, record.getId())
.upsert(this.gson.toJson(sourceMap), XContentType.JSON);
bulkRequest.add(updateRequest);
}
@@ -465,7 +476,7 @@ public class IndexerServiceImpl implements IndexerService {
String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
for (String id : record.getValue()) {
- DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
+ DeleteRequest deleteRequest = new DeleteRequest(index, id);
bulkRequest.add(deleteRequest);
}
}
@@ -547,6 +558,7 @@ public class IndexerServiceImpl implements IndexerService {
indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+ indexerPayload.put(RecordMetaAttribute.TAGS.getValue(), record.getTags());
indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
index 86fb43449ea75340340b0ee43de9e85304c79697..226f74229fce34469f84068c3e468f4d59e3b58a 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
@@ -24,7 +24,7 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.gcp.model.AppEngineHeaders;
+import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
import org.opengroup.osdu.core.gcp.util.TraceIdExtractor;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@@ -43,10 +43,10 @@ public class DpsHeaderFactoryGcp extends DpsHeaders {
.stream()
.collect(Collectors.toMap(h -> h, request::getHeader));
- String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT);
+ String traceContext = headers.get(CloudTaskHeaders.CLOUD_TRACE_CONTEXT);
if (!Strings.isNullOrEmpty(traceContext)) {
- headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext));
+ headers.put(CloudTaskHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext));
}
this.addFromMap(headers);
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
index 1e3c3f244cb37470a91c6a72185a2007c2578698..b86b91ce440748e52c559da0152b5d3c4060d0a4 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
@@ -1,62 +1,73 @@
/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ Copyright 2021 Google LLC
+ Copyright 2021 EPAM Systems, Inc
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
*/
package org.opengroup.osdu.indexer.util;
-import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
-
+import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken;
+import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.common.base.Strings;
-import java.util.Map;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.logging.Level;
import lombok.extern.java.Log;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.opengroup.osdu.core.gcp.model.AppEngineHeaders;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
+import javax.inject.Inject;
+import java.util.Map;
+
+import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
+
+
@Log
@Component
@RequestScope
public class RequestInfoImpl implements IRequestInfo {
- private final IndexerConfigurationProperties indexerConfigurationProperties;
- private final TenantInfo tenantInfo;
- private final IServiceAccountJwtClient serviceAccountJwtClient;
- private final DpsHeaders dpsHeaders;
-
- @Autowired
- public RequestInfoImpl(IndexerConfigurationProperties indexerConfigurationProperties,
- TenantInfo tenantInfo, IServiceAccountJwtClient serviceAccountJwtClient,
- DpsHeaders dpsHeaders) {
- this.indexerConfigurationProperties = indexerConfigurationProperties;
- this.tenantInfo = tenantInfo;
- this.serviceAccountJwtClient = serviceAccountJwtClient;
- this.dpsHeaders = dpsHeaders;
- }
+ @Inject
+ private DpsHeaders dpsHeaders;
+
+ @Inject
+ private IServiceAccountJwtClient serviceAccountJwtClient;
+
+ @Inject
+ private TenantInfo tenantInfo;
- private static final String expectedCronHeaderValue = "true";
+ @Inject
+ private IndexerConfigurationProperties properties;
+
+ @Value("${indexer.que.service.mail}")
+ private String indexerQueServiceMail;
+
+ private static final String EXPECTED_CRON_HEADER_VALUE = "true";
@Override
public DpsHeaders getHeaders() {
@@ -88,33 +99,65 @@ public class RequestInfoImpl implements IRequestInfo {
@Override
public boolean isCronRequest() {
- String appEngineCronHeader = this.dpsHeaders.getHeaders()
- .getOrDefault(AppEngineHeaders.CRON_SERVICE, null);
- return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader);
+ String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(CloudTaskHeaders.CLOUD_CRON_SERVICE, null);
+ return EXPECTED_CRON_HEADER_VALUE.equalsIgnoreCase(appEngineCronHeader);
}
@Override
public boolean isTaskQueueRequest() {
- if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) {
+ if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){
+ log.log(Level.INFO,"Request acknowledged as Cloud task, proceeding token validation");
+ return isCloudTaskRequest();
+ }
+ if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){
+ log.log(Level.INFO,"Request acknowledged as AppEngine task, proceeding headers validation");
+ return isAppEngineTaskRequest();
+ }
+ return false;
+ }
+
+ private boolean isCloudTaskRequest() {
+ log.log(Level.INFO,dpsHeaders.getHeaders().toString());
+ try {
+ GoogleIdTokenVerifier verifier = new GoogleIdTokenVerifier.Builder(
+ GoogleNetHttpTransport.newTrustedTransport(), JacksonFactory.getDefaultInstance())
+ .setIssuers(Arrays.asList(
+ "accounts.google.com", "https://accounts.google.com",
+ "googleapis.com", "https://www.googleapis.com/auth/userinfo.profile"
+ )
+ ).build();
+ String authorization = dpsHeaders.getAuthorization().replace("Bearer ", "");
+
+ GoogleIdToken googleIdToken = verifier.verify(authorization);
+ if(googleIdToken.getPayload().getEmail().equals(indexerQueServiceMail)){
+ return true;
+ }
+ log.log(Level.WARNING,"Token email doesn't match with variable \"indexer.que.service.mail\"");
+ return false;
+
+ } catch (GeneralSecurityException | IOException e) {
+ log.log(Level.WARNING,"Not valid or expired cloud task token provided");
return false;
}
+ }
- String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME);
+ private boolean isAppEngineTaskRequest(){
+ if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
+ return false;
+ }
+ String queueId = this.dpsHeaders.getHeaders().get(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME);
return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
}
public String checkOrGetAuthorizationHeader() {
- if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment())
- == DeploymentEnvironment.LOCAL) {
+ if (properties.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) {
String authHeader = this.dpsHeaders.getAuthorization();
if (Strings.isNullOrEmpty(authHeader)) {
- throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header",
- "Authorization token cannot be empty");
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty");
}
String user = this.dpsHeaders.getUserEmail();
if (Strings.isNullOrEmpty(user)) {
- throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header",
- "User header cannot be empty");
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", "User header cannot be empty");
}
return authHeader;
} else {
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
index 98605a1509c597b4b5f7332560374ff072248d44..673565941dda7a398bba598aab3945be8ac6126b 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
@@ -18,33 +18,14 @@
package org.opengroup.osdu.indexer.util;
import com.auth0.jwt.JWT;
-import com.auth0.jwt.exceptions.JWTDecodeException;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.iam.v1.Iam;
-import com.google.api.services.iam.v1.IamScopes;
-import com.google.api.services.iam.v1.model.SignJwtRequest;
-import com.google.api.services.iam.v1.model.SignJwtResponse;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.http.HttpHeaders;
+import com.google.cloud.iam.credentials.v1.GenerateIdTokenResponse;
+import com.google.cloud.iam.credentials.v1.IamCredentialsClient;
+import com.google.cloud.iam.credentials.v1.ServiceAccountName;
+import java.util.Collections;
import org.apache.http.HttpStatus;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.ContentType;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.util.EntityUtils;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
@@ -53,6 +34,7 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.core.gcp.multitenancy.credentials.IamCredentialsProvider;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -64,6 +46,7 @@ public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient
private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token";
private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s";
+ private final IamCredentialsProvider iamCredentialsProvider = new IamCredentialsProvider();
private final JsonFactory JSON_FACTORY = new JacksonFactory();
private final IndexerConfigurationProperties indexerConfigurationProperties;
@@ -104,99 +87,24 @@ public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient
return cachedToken.getTokenValue();
}
- // Getting signed JWT
- Map<String, Object> signJwtPayload = this.getJWTCreationPayload(tenant);
-
- SignJwtRequest signJwtRequest = new SignJwtRequest();
- signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload));
-
- String serviceAccountName = String
- .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount());
-
- Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts()
- .signJwt(serviceAccountName, signJwtRequest);
- SignJwtResponse signJwtResponse = signJwt.execute();
- String signedJwt = signJwtResponse.getSignedJwt();
-
- // Getting id token
- List<NameValuePair> postParameters = new ArrayList<>();
- postParameters
- .add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"));
- postParameters.add(new BasicNameValuePair("assertion", signedJwt));
-
- HttpPost post = new HttpPost(JWT_AUDIENCE);
- post.setHeader(HttpHeaders.CONTENT_TYPE,
- ContentType.APPLICATION_FORM_URLENCODED.getMimeType());
- post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8"));
-
- try (CloseableHttpClient httpclient = HttpClients.createDefault();
- CloseableHttpResponse httpResponse = httpclient.execute(post)) {
- JsonObject jsonContent = new JsonParser()
- .parse(EntityUtils.toString(httpResponse.getEntity()))
- .getAsJsonObject();
-
- if (!jsonContent.has("id_token")) {
- log.error(String.format("Google IAM response: %s", jsonContent.toString()));
- throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied",
- "The user is not authorized to perform this action");
- }
-
- String token = jsonContent.get("id_token").getAsString();
+ try (IamCredentialsClient iamCredentialsClient = iamCredentialsProvider
+ .getIamCredentialsClient()) {
+ ServiceAccountName serviceAccountName = ServiceAccountName
+ .parse(String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getServiceAccount()));
+ GenerateIdTokenResponse idTokenResponse = iamCredentialsClient
+ .generateIdToken(serviceAccountName, Collections
+ .emptyList(), indexerConfigurationProperties.getGoogleAudiences(), true);
+ String token = idTokenResponse.getToken();
IdToken idToken = IdToken.builder().tokenValue(token)
.expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build();
-
this.cacheService.put(tenant.getServiceAccount(), idToken);
-
return token;
}
- } catch (JWTDecodeException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error",
- "Invalid token, error decoding", e);
} catch (AppException e) {
throw e;
} catch (Exception e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error",
- "Error generating token",
- e);
- }
- }
-
- public Iam getIam() throws Exception {
-
- if (this.iam == null) {
- HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
-
- // Authenticate using Google Application Default Credentials.
- GoogleCredential credential = GoogleCredential.getApplicationDefault();
- if (credential.createScopedRequired()) {
- List<String> scopes = new ArrayList<>();
- // Enable full Cloud Platform scope.
- scopes.add(IamScopes.CLOUD_PLATFORM);
- credential = credential.createScoped(scopes);
- }
-
- // Create IAM API object associated with the authenticated transport.
- this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential)
- .setApplicationName(indexerConfigurationProperties.getIndexerHost())
- .build();
+ "Error generating token", e);
}
-
- return this.iam;
- }
-
- private Map<String, Object> getJWTCreationPayload(TenantInfo tenantInfo) {
-
- Map<String, Object> payload = new HashMap<>();
- String googleAudience = indexerConfigurationProperties.getGoogleAudiences();
- if (googleAudience.contains(",")) {
- googleAudience = googleAudience.split(",")[0];
- }
- payload.put("target_audience", googleAudience);
- payload.put("exp", System.currentTimeMillis() / 1000 + 3600);
- payload.put("iat", System.currentTimeMillis() / 1000);
- payload.put("iss", tenantInfo.getServiceAccount());
- payload.put("aud", JWT_AUDIENCE);
-
- return payload;
}
}
diff --git a/provider/indexer-reference/src/main/resources/application.properties b/provider/indexer-reference/src/main/resources/application.properties
index 8690c57a906c0aae8cf6ecc5b0c282be392bbafb..542569ff56509144da5ba45d9d2208b8345b60a9 100644
--- a/provider/indexer-reference/src/main/resources/application.properties
+++ b/provider/indexer-reference/src/main/resources/application.properties
@@ -67,10 +67,8 @@ crs-api=example.com
## use below values for gcp: opendes
redis-group-host=127.0.0.1
redis-search-host=127.0.0.1
+redis-search-port=6379
google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
-mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672
-
-
-
+mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672
\ No newline at end of file
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java
deleted file mode 100644
index ccf35e02cffb041fdf0582ae87e6891028947543..0000000000000000000000000000000000000000
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.step_definitions.index.cleanup;
-
-import cucumber.api.CucumberOptions;
-import cucumber.api.junit.Cucumber;
-import org.junit.runner.RunWith;
-
-@RunWith(Cucumber.class)
-@CucumberOptions(
- features = "classpath:features/indexcleanup/IndexCleanup.feature",
- glue = {"classpath:org.opengroup.osdu.step_definitions/index/cleanup"},
- plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexcleanup.xml"})
-public class RunTest {
-}
\ No newline at end of file
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java
deleted file mode 100644
index b69fd683d35214a73fd9c268e6dcb879d95f97c8..0000000000000000000000000000000000000000
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java
+++ /dev/null
@@ -1,113 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-
-package org.opengroup.osdu.step_definitions.index.cleanup;
-
-import cucumber.api.DataTable;
-import cucumber.api.Scenario;
-import cucumber.api.java.Before;
-import cucumber.api.java.en.Given;
-import cucumber.api.java.en.Then;
-import cucumber.api.java.en.When;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import lombok.extern.java.Log;
-import org.opengroup.osdu.common.CleanupIndiciesSteps;
-import org.opengroup.osdu.core.common.model.legal.Legal;
-import org.opengroup.osdu.util.AWSHTTPClient;
-import org.opengroup.osdu.util.LegalTagUtilsAws;
-import org.opengroup.osdu.util.ElasticUtilsAws;
-
-import static org.opengroup.osdu.util.Config.getOtherRelevantDataCountries;
-
-@Log
-public class Steps extends CleanupIndiciesSteps {
-
- protected LegalTagUtilsAws legalTagUtils;
- private String legalTagName;
-
- public Steps() {
- super(new AWSHTTPClient(), new ElasticUtilsAws());
- }
-
- @Before
- public void before(Scenario scenario) throws Exception {
- this.scenario = scenario;
- this.httpClient = new AWSHTTPClient();
- legalTagUtils = new LegalTagUtilsAws(this.httpClient);
- this.legalTagName = this.legalTagUtils.createRandomName();
- this.legalTagUtils.create(this.legalTagName);
- }
-
- @Override
- public void tearDown() {
- super.tearDown();
- this.legalTagUtils.delete(this.legalTagName);
- }
-
- @Override
- protected Legal generateLegalTag() {
- Legal legal = new Legal();
- Set<String> legalTags = new HashSet<>();
- legalTags.add(this.legalTagName);
- legal.setLegaltags(legalTags);
- Set<String> otherRelevantCountries = new HashSet<>();
- otherRelevantCountries.add(getOtherRelevantDataCountries());
- legal.setOtherRelevantDataCountries(otherRelevantCountries);
- return legal;
- }
-
- @Given("^the schema is created with the following kind$")
- public void theSchemaIsCreatedWithTheFollowingKind(DataTable dataTable) {
- super.theSchemaIsCreatedWithTheFollowingKind(dataTable);
- }
-
- @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
- public void iIngestRecordsWithTheforAGiven(String record, String dataGroup, String kind) {
- super.iIngestRecordsWithTheforAGiven(record, dataGroup, kind);
- }
-
- @Then("^I check that the index for \"(.*?)\" has been created$")
- public void iCheckThatTheIndexForHasBeenCreated(String kind) throws IOException, InterruptedException {
- super.iCheckThatTheIndexForHasBeenCreated(kind);
- }
-
- @Then("^I should delete the records I created earlier$")
- public void iShouldDeleteTheRecordsForICreatedEarlier() {
- super.iShouldDeleteTheRecordsForICreatedEarlier();
- }
-
- @Then("^I should delete the schema for \"(.*?)\" I created earlier$")
- public void iShouldDeleteTheSchemaForICreatedEarlier(String kind) {
- super.iShouldDeleteTheSchemaForICreatedEarlier(kind);
- }
-
- @Then("^I should check that the index for \"(.*?)\" has not been deleted$")
- public void iShouldCheckThetTheIndexforHasNotBeenDeleted(String kind) throws IOException, InterruptedException {
- super.iShouldCheckThetTheIndexforHasNotBeenDeleted(kind);
- }
-
- @Then("^I should to run cleanup of indexes for \"(.*?)\" and \"(.*?)\"$")
- public void iShouldToRunCleanupOfIndexesForAnd(String kind, String message) {
- super.iShouldToRunCleanupOfIndexesForAnd(kind, message);
- }
-
- @Then("^I should check that the index for \"(.*?)\" has been deleted$")
- public void iShouldCheckThatTheIndexForHasBeenDeleted(String kind) throws IOException, InterruptedException {
- super.iShouldCheckThatTheIndexForHasBeenDeleted(kind);
- }
-}
\ No newline at end of file
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index c565d38835769e8e5d5100583fd61202fbc26cd9..01691b4f8b53f1d0db827844231553a969da8529 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -86,8 +86,13 @@ public class Steps extends SchemaServiceRecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
- super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..cba0839f389e3f230834c60088862021b86bc271
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
@@ -0,0 +1,12 @@
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/info/Info.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/info"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-info.xml"})
+public class RunTest {}
\ No newline at end of file
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..ba386451b3bb3ca9f32ce3591e6b3e9fd16e3d46
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -0,0 +1,46 @@
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import org.opengroup.osdu.common.info.InfoSteps;
+import org.opengroup.osdu.util.AWSHTTPClient;
+import org.opengroup.osdu.util.Config;
+
+public class Steps extends InfoSteps {
+
+ public Steps() {
+ super(new AWSHTTPClient());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new AWSHTTPClient();
+ }
+
+ /******************Inputs being set**************/
+
+ @When("^I send get request to version info endpoint")
+ public void i_send_get_request_to_version_info_endpoint() {
+ super.i_send_get_request_to_version_info_endpoint();
+ }
+
+ /******************Assert final response**************/
+
+ @Then("^I should get version info in response$")
+ public void i_should_get_version_info_in_response() {
+ super.i_should_get_version_info_in_response();
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return "GET";
+ }
+
+ @Override
+ protected String getApi() {
+ return Config.getIndexerBaseURL() + "info";
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 9f897aa2140a4d8e6ca82f9d882567fefd9f0014..744346e091543648ed971b079e30c2e80aaf2a83 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -34,9 +34,9 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
- <azure.keyvault.version>4.1.0</azure.keyvault.version>
<junit.jupiter.version>5.6.0</junit.jupiter.version>
<elasticsearch.version>7.8.1</elasticsearch.version>
+ <os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
</properties>
<dependencies>
@@ -46,20 +46,41 @@
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
<version>0.11.0-SNAPSHOT</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- Azure dependencies -->
<!--Added dependency for utilizing azure service principle class-->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
- <version>0.0.33</version>
+ <version>0.11.0-rc4</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -72,11 +93,10 @@
<artifactId>netty-transport</artifactId>
<version>4.1.15.Final</version>
</dependency>
-
<dependency>
<groupId>com.azure</groupId>
- <artifactId>azure-security-keyvault-secrets</artifactId>
- <version>${azure.keyvault.version}</version>
+ <artifactId>azure-core-http-netty</artifactId>
+ <version>1.2.0</version>
</dependency>
<!-- Testing -->
@@ -168,6 +188,11 @@
</dependency>
<!--Logging-->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.30</version>
+ </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
@@ -178,6 +203,16 @@
<artifactId>log4j-api</artifactId>
<version>2.13.0</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.12.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.12.2</version>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index b1dc2343728b339278710726c0d37d4a1735dd0f..5c0f44c22eff61886aba89ecb1693b7a73856999 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -54,8 +54,13 @@ public class Steps extends SchemaServiceRecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
- super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..cba0839f389e3f230834c60088862021b86bc271
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
@@ -0,0 +1,12 @@
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/info/Info.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/info"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-info.xml"})
+public class RunTest {}
\ No newline at end of file
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..a7871ae9130150ddf1c07688489136ea58667cbb
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -0,0 +1,46 @@
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import org.opengroup.osdu.common.info.InfoSteps;
+import org.opengroup.osdu.util.AzureHTTPClient;
+import org.opengroup.osdu.util.Config;
+
+public class Steps extends InfoSteps {
+
+ public Steps() {
+ super(new AzureHTTPClient());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new AzureHTTPClient();
+ }
+
+ /******************Inputs being set**************/
+
+ @When("^I send get request to version info endpoint")
+ public void i_send_get_request_to_version_info_endpoint() {
+ super.i_send_get_request_to_version_info_endpoint();
+ }
+
+ /******************Assert final response**************/
+
+ @Then("^I should get version info in response$")
+ public void i_should_get_version_info_in_response() {
+ super.i_should_get_version_info_in_response();
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return "GET";
+ }
+
+ @Override
+ protected String getApi() {
+ return Config.getIndexerBaseURL() + "info";
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index 7e7a0e821ecb21d523a4ba630d1c233c32261672..8fbb5bb85063e6c4a6acbfc4f3c7a8a65644ed57 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -17,7 +17,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
- <os-core-common.version>0.3.19</os-core-common.version>
+ <os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
</properties>
<dependencies>
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
deleted file mode 100644
index d348fd9d2a48fb4daaad6099f4609fea20882329..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.common;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.opengroup.osdu.util.Config.getEntitlementsDomain;
-import static org.opengroup.osdu.util.Config.getIndexerBaseURL;
-import static org.opengroup.osdu.util.Config.getStorageBaseURL;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import com.sun.jersey.api.client.ClientResponse;
-import cucumber.api.DataTable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import javax.ws.rs.HttpMethod;
-import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.models.Setup;
-import org.opengroup.osdu.models.TestIndex;
-import org.opengroup.osdu.util.FileHandler;
-import org.opengroup.osdu.util.HTTPClient;
-import org.opengroup.osdu.util.ElasticUtils;
-
-@Log
-public class CleanupIndiciesSteps extends TestsBase {
- private final String timeStamp = String.valueOf(System.currentTimeMillis());
- private Map<String, TestIndex> inputIndexMap = new HashMap<>();
- private List<Map<String, Object>> records;
- private boolean shutDownHookAdded = false;
- private final Map<String, String> headers = httpClient.getCommonHeader();
-
- public CleanupIndiciesSteps(HTTPClient httpClient) {
- super(httpClient);
- }
-
- public CleanupIndiciesSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
- super(httpClient, elasticUtils);
- }
-
-
- public void theSchemaIsCreatedWithTheFollowingKind(DataTable dataTable) {
- List<Setup> inputList = dataTable.asList(Setup.class);
- for (Setup input : inputList) {
- TestIndex testIndex = getTextIndex();
- testIndex.setHttpClient(httpClient);
- testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
- testIndex.setKind(generateActualName(input.getKind(), timeStamp));
- testIndex.setSchemaFile(input.getSchemaFile());
- inputIndexMap.put(testIndex.getKind(), testIndex);
- }
-
- if (!shutDownHookAdded) {
- shutDownHookAdded = true;
- for (Map.Entry<String, TestIndex> kind : inputIndexMap.entrySet()) {
- kind.getValue().setupSchema();
- }
- }
- }
-
- public void iIngestRecordsWithTheforAGiven(String record, String dataGroup, String kind) {
- String actualKind = generateActualName(kind, timeStamp);
- try {
- String fileContent = FileHandler.readFile(String.format("%s.%s", record, "json"));
- records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
-
- for (Map<String, Object> testRecord : records) {
- testRecord.put("kind", actualKind);
- testRecord.put("id", generateActualId(testRecord.get("id").toString(), timeStamp, actualKind));
- testRecord.put("legal", generateLegalTag());
- String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
- Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
- testRecord.put("acl", acl);
- }
- String payLoad = new Gson().toJson(records);
- ClientResponse clientResponse = httpClient.send(HttpMethod.PUT, getStorageBaseURL() + "records", payLoad, headers, httpClient.getAccessToken());
- assertEquals(201, clientResponse.getStatus());
- } catch (Exception ex) {
- throw new AssertionError(ex.getMessage());
- }
- }
-
- public void iCheckThatTheIndexForHasBeenCreated(String kind) throws IOException, InterruptedException {
- assertTrue(isNewIndexCreated(generateActualName(kind, timeStamp)));
- }
-
- public void iShouldDeleteTheRecordsForICreatedEarlier() {
- List<Map<String, Object>> deletedRecords = new ArrayList<>();
- if (records != null && !records.isEmpty()) {
- for (Map<String, Object> testRecord : records) {
- String id = testRecord.get("id").toString();
- ClientResponse clientResponse = httpClient.send(HttpMethod.DELETE, getStorageBaseURL()
- + "records/" + id, null, headers, httpClient.getAccessToken());
- if (clientResponse.getStatus() == 204) {
- deletedRecords.add(testRecord);
- log.info("Deleted the records with id " + id);
- }
- }
- assertEquals(records.size(), deletedRecords.size());
- }
- }
-
- public void iShouldDeleteTheSchemaForICreatedEarlier(String kind) {
- ClientResponse response = httpClient.send(HttpMethod.DELETE,
- String.format("%sschemas%s", getStorageBaseURL(), "/" + generateActualName(kind, timeStamp)),null,
- headers, httpClient.getAccessToken());
- assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatus());
- }
-
- public void iShouldCheckThetTheIndexforHasNotBeenDeleted(String kind) throws IOException, InterruptedException {
- assertTrue(isNewIndexExist(generateActualName(kind, timeStamp)));
- }
-
- public void iShouldToRunCleanupOfIndexesForAnd(String kind, String message) {
-
- String url = getIndexerBaseURL() + "index-cleanup";
- log.info("URL: " + url);
- ClientResponse response = httpClient.send(HttpMethod.POST, url,
- convertMessageIntoJson(kind, message), headers, httpClient.getAccessToken());
- assertEquals(HttpStatus.SC_OK, response.getStatus());
- }
-
- public void iShouldCheckThatTheIndexForHasBeenDeleted(String kind) throws IOException, InterruptedException {
- assertFalse(isNewIndexExist(generateActualName(kind, timeStamp)));
- }
-
- private String convertMessageIntoJson(String kind, String message) {
- String actualKind = generateActualName(kind, timeStamp);
- RecordChangedMessages recordChangedMessages = (new Gson()).fromJson(String.format(message,
- actualKind, actualKind, timeStamp), RecordChangedMessages.class);
- return new Gson().toJson(recordChangedMessages);
- }
-
- private boolean isNewIndexExist(String index) throws IOException {
- return elasticUtils.isIndexExist(index.replace(":", "-"));
- }
-
- private boolean isNewIndexCreated(String index) throws IOException, InterruptedException {
- int iterator;
- boolean indexExist = false;
-
- // index.refresh_interval is set to default 30s, wait for 40s initially
- Thread.sleep(40000);
-
- for (iterator = 0; iterator < 20; iterator++) {
- indexExist = elasticUtils.isIndexExist(index.replace(":", "-"));
- if (indexExist) {
- break;
- } else {
- Thread.sleep(5000);
- }
- if ((iterator + 1) % 5 == 0) {
- elasticUtils.refreshIndex(index.replace(":", "-"));
- }
- }
- if (iterator >= 20) {
- fail(String.format("index not created after waiting for %s seconds", ((40000 + iterator * 5000) / 1000)));
- }
- return indexExist;
- }
-
- @Override
- protected String getApi() {
- return null;
- }
-
- @Override
- protected String getHttpMethod() {
- return null;
- }
-}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/InfoBase.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/InfoBase.java
new file mode 100644
index 0000000000000000000000000000000000000000..509b23c9b06946e7f5aab9e7323a2c512fa3a2da
--- /dev/null
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/InfoBase.java
@@ -0,0 +1,68 @@
+package org.opengroup.osdu.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.HashMap;
+import java.util.Map;
+import joptsimple.internal.Strings;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.response.InfoResponseMock;
+import org.opengroup.osdu.util.Config;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.HTTPClient;
+
+@Slf4j
+public class InfoBase extends TestsBase {
+
+ protected Map<String, String> headers = new HashMap<>();
+ private InfoResponseMock response;
+
+ public InfoBase(HTTPClient httpClient) {
+ super(httpClient);
+ }
+
+ public InfoBase(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ super(httpClient, elasticUtils);
+ }
+
+ @Override
+ protected String getApi() {
+ return Config.getIndexerBaseURL() + "info";
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return "GET";
+ }
+
+ public void i_send_get_request_to_version_info_endpoint() {
+ if (Config.getIndexerBaseURL().isEmpty()) {
+ log.warn("Env variable INDEXER_HOST is empty. Version info endpoint test is skipped");
+ return;
+ }
+
+ response =
+ executeQuery(
+ this.getApi(),
+ Strings.EMPTY,
+ headers,
+ httpClient.getAccessToken(),
+ InfoResponseMock.class);
+ }
+
+ public void i_should_get_version_info_in_response() {
+ if (response != null) {
+ assertEquals(200, response.getResponseCode());
+ assertNotNull(response.getGroupId());
+ assertNotNull(response.getArtifactId());
+ assertNotNull(response.getVersion());
+ assertNotNull(response.getBuildTime());
+ assertNotNull(response.getBranch());
+ assertNotNull(response.getCommitId());
+ assertNotNull(response.getCommitMessage());
+ } else {
+ log.warn("Version info endpoint provided null response");
+ }
+ }
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index cd7f2eb02ee34ec824ca51b88a47c57460ef211e..d78985d0d2992c41bb3707cf4ee92414045d58bd 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -48,7 +48,7 @@ public class RecordSteps extends TestsBase {
public RecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
super(httpClient, elasticUtils);
}
-
+
/******************One time cleanup for whole feature**************/
public void tearDown() {
for (String kind : inputIndexMap.keySet()) {
@@ -98,14 +98,22 @@ public class RecordSteps extends TestsBase {
try {
String fileContent = FileHandler.readFile(String.format("%s.%s", record, "json"));
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
+ String createTime = java.time.Instant.now().toString();
for (Map<String, Object> testRecord : records) {
testRecord.put("kind", actualKind);
- testRecord.put("id", generateRecordId(testRecord));
+ testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
testRecord.put("acl", acl);
+ String[] kindParts = kind.split(":");
+ String authority = tenantMap.get(kindParts[0]);
+ String source = kindParts[1];
+ testRecord.put("authority", authority);
+ testRecord.put("source", source);
+ testRecord.put("createUser", "TestUser");
+ testRecord.put("createTime", createTime);
}
String payLoad = new Gson().toJson(records);
log.log(Level.INFO, "Start ingesting records={0}", payLoad);
@@ -126,17 +134,36 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedCount, numOfIndexedDocuments);
}
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
index = generateActualName(index, timeStamp);
Map<String, MappingMetadata> elasticMapping = elasticUtils.getMapping(index);
assertNotNull(elasticMapping);
+ String[] kindParts = kind.split(":");
+ String authority = tenantMap.get(kindParts[0]);
+ String source = kindParts[1];
+ expectedMapping = expectedMapping.replaceAll("<authority-id>", authority).replaceAll("<source-id>", source);
MappingMetadata typeMapping = elasticMapping.get(index);
Map<String, Object> mapping = typeMapping.sourceAsMap();
assertNotNull(mapping);
assertTrue(areJsonEqual(expectedMapping, mapping.toString()));
}
+ public void i_can_validate_indexed_attributes(String index, String kind) throws Throwable {
+ String authority = tenantMap.get(kind.substring(0, kind.indexOf(":")));
+ index = generateActualName(index, timeStamp);
+ List<Map<String, Object>> hits = elasticUtils.fetchRecordsByAttribute(index, "authority", authority);
+
+ assertTrue(hits.size() > 0);
+ for (Map<String, Object> result : hits) {
+ assertTrue(result.containsKey("authority"));
+ assertEquals(authority, result.get("authority"));
+ assertTrue(result.containsKey("source"));
+ assertTrue(result.containsKey("createUser"));
+ assertTrue(result.containsKey("createTime"));
+ }
+ }
+
public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
index = generateActualName(index, timeStamp);
long numOfIndexedDocuments = createIndex(index);
@@ -228,7 +255,7 @@ public class RecordSteps extends TestsBase {
log.info(String.format("difference: %s", result.entriesDiffering()));
return false;
}
-
+
@Override
protected String getApi() {
return null;
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
index e4d6322f715941c1bc18d5fc82ee9b33ce9e2c97..2b8d960eccc3cc1a934e480e9e2d6b31c725be47 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
@@ -18,6 +18,7 @@ import javax.ws.rs.core.MultivaluedMap;
import java.util.*;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.opengroup.osdu.util.Config.*;
@Log
@@ -109,7 +110,7 @@ public abstract class TestsBase {
private <T extends ResponseBase> T getResponse(ClientResponse clientResponse, Class<T> typeParameterClass) {
log.info(String.format("Response status: %s, type: %s", clientResponse.getStatus(), clientResponse.getType().toString()));
- assertEquals(MediaType.APPLICATION_JSON, clientResponse.getType().toString());
+ assertTrue(clientResponse.getType().toString().contains(MediaType.APPLICATION_JSON));
String responseEntity = clientResponse.getEntity(String.class);
T response = new Gson().fromJson(responseEntity, typeParameterClass);
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/info/InfoSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/info/InfoSteps.java
new file mode 100644
index 0000000000000000000000000000000000000000..02924b4d055aa4a444ab79174ec2c0f978ae3818
--- /dev/null
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/info/InfoSteps.java
@@ -0,0 +1,16 @@
+package org.opengroup.osdu.common.info;
+
+import org.opengroup.osdu.common.InfoBase;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.HTTPClient;
+
+public class InfoSteps extends InfoBase {
+
+ public InfoSteps(HTTPClient httpClient) {
+ super(httpClient);
+ }
+
+ public InfoSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ super(httpClient, elasticUtils);
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/response/InfoResponseMock.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/response/InfoResponseMock.java
new file mode 100644
index 0000000000000000000000000000000000000000..682226c3d88f4b1a09195cddba665711c581c639
--- /dev/null
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/response/InfoResponseMock.java
@@ -0,0 +1,16 @@
+package org.opengroup.osdu.response;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+public class InfoResponseMock extends ResponseBase {
+ private String groupId;
+ private String artifactId;
+ private String version;
+ private String buildTime;
+ private String branch;
+ private String commitId;
+ private String commitMessage;
+}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 3d029aa6ecbd34fb9ec08c2e9909cf933e33843a..23419f3fd798d59eec1a771c366a5acff8b577b4 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -52,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.GetMappingsRequest;
import org.elasticsearch.client.indices.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.locationtech.jts.geom.Coordinate;
import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
@@ -241,6 +242,24 @@ public class ElasticUtils {
}
}
+ public List<Map<String, Object>> fetchRecordsByAttribute(String index, String attributeKey, String attributeValue) throws IOException {
+ List<Map<String, Object>> out = new ArrayList<>();
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.query(boolQuery().must(termsQuery(attributeKey, attributeValue)));
+ request.source(sourceBuilder);
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+ for(SearchHit searchHit : searchResponse.getHits()) out.add(searchHit.getSourceAsMap());
+ return out;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return out;
+ }
+ }
+
public long fetchRecordsByExistQuery(String index, String attributeName) throws Exception {
try {
TimeUnit.SECONDS.sleep(40);
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
deleted file mode 100644
index a06fa2efcbcb9107b184934c0304bf105afbb85e..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
+++ /dev/null
@@ -1,28 +0,0 @@
-Feature: Indexing of the documents
- This feature deals with validation of the documents in Elastic Search ingested with different kinds and attributes.
-
- Background:
- Given the schema is created with the following kind
- | kind | index | schemaFile |
- | tenant1:testindex<timestamp>:well:1.0.0 | tenant1-testindex<timestamp>-well-1.0.0 | index_records_1 |
- | tenant1:testindex<timestamp>:well:2.0.0 | tenant1-testindex<timestamp>-well-2.0.0 | index_records_2 |
- | tenant1:testindex<timestamp>:well:3.0.0 | tenant1-testindex<timestamp>-well-3.0.0 | index_records_3 |
-
- Scenario Outline: Ingest the record and Index in the Elastic Search
- When I ingest records with the <recordFile> with <acl> for a given <kind>
- Then I should get the <number> documents for the <index> in the Elastic Search
- Then I should get the elastic <mapping> for the <type> and <index> in the Elastic Search
-
- Examples:
- | kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
-
- Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
- When I ingest records with the <recordFile> with <acl> for a given <kind>
- Then I should get the <number> documents for the <index> in the Elastic Search with out <skippedAttribute>
-
- Examples:
- | kind | recordFile | number | index | skippedAttribute | acl |
- | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.default.viewers@tenant1" |
- | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_3" | 7 | "tenant1-testindex<timestamp>-well-3.0.0" | "data.GeoShape" | "data.default.viewers@tenant1" |
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 05c25961282d3ebec5af580932fd7d73f5917f7b..0888f9528b60901a73d69862039479726a0648a7 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -4,21 +4,22 @@ Feature: Indexing of the documents
Background:
Given the schema is created with the following kind
| kind | index | schemaFile |
- | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1 |
- | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 |
- | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 |
+ | tenant1:indexer:test-data--Integration:1.0.1 | tenant1-indexer-test-data--integration-1.0.1 | index_records_1 |
+ | tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
+ | tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
Then I should get the <number> documents for the <index> in the Elastic Search
- Then I should get the elastic <mapping> for the <type> and <index> in the Elastic Search
+ Then I should get the elastic <mapping> for the <kind> and <index> in the Elastic Search
+ Then I can validate indexed meta attributes for the <index> and given <kind>
Examples:
- | kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:indexer:test-data--Integration:1.0.0" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.0" | "test-data--Integration" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:indexer:test-data--Integration:3.0.0" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.0" | "test-data--Integration" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | kind | recordFile | number | index | acl | mapping |
+ | "tenant1:indexer:test-data--Integration:1.0.1" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:indexer:test-data--Integration:3.0.1" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -26,8 +27,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | skippedAttribute | acl |
- | "tenant1:indexer:test-data--Integration:2.0.0" | "index_records_2" | 4 | "tenant1-indexer-test-data--integration-2.0.0" | "data.Location" | "data.default.viewers@tenant1" |
- | "tenant1:indexer:test-data--Integration:2.0.0" | "index_records_2" | 1 | "tenant1-indexer-test-data--integration-2.0.0" | "data.InvalidInteger" | "data.default.viewers@tenant1" |
+ | "tenant1:indexer:test-data--Integration:2.0.1" | "index_records_2" | 4 | "tenant1-indexer-test-data--integration-2.0.1" | "data.Location" | "data.default.viewers@tenant1" |
+ | "tenant1:indexer:test-data--Integration:2.0.1" | "index_records_2" | 1 | "tenant1-indexer-test-data--integration-2.0.1" | "data.InvalidInteger" | "data.default.viewers@tenant1" |
Scenario Outline: Ingest the record and Index in the Elastic Search with tags
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -35,7 +36,7 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | index | acl | tagKey | tagValue | number |
- | "tenant1:indexer:test-data--Integration:1.0.0" | "index_records_schema_1" | "tenant1-indexer-test-data--integration-1.0.0" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
+ | "tenant1:indexer:test-data--Integration:1.0.1" | "index_records_schema_1" | "tenant1-indexer-test-data--integration-1.0.1" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
Scenario Outline: Ingest the r3-record with geo-shape and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/features/info/Info.feature b/testing/indexer-test-core/src/main/resources/features/info/Info.feature
new file mode 100644
index 0000000000000000000000000000000000000000..6344baeab9b9e20810990e438381b869e8ac79c0
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/features/info/Info.feature
@@ -0,0 +1,5 @@
+Feature: Fetch info about maven build and git repository.
+
+ Scenario: Verify version info endpoint content
+ When I send get request to version info endpoint
+ Then I should get version info in response
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
index b7ecdbc656441b9db8a04ec27484493d0b40a73a..b2a02ea146ec5896262020ddf8c7a014bfdb9056 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
@@ -6,7 +6,7 @@
"entityType": "test-data--Integration",
"schemaVersionMajor": "1",
"schemaVersionMinor": "0",
- "schemaVersionPatch": "0"
+ "schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_2.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_records_2.schema.json
index e67e19502e0283bdf977b03ff5a59dd27883cfa0..e5ee1cb04eedeafed586fa3baa9664ba1d57d2cb 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_2.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_2.schema.json
@@ -6,7 +6,7 @@
"entityType": "test-data--Integration",
"schemaVersionMajor": "2",
"schemaVersionMinor": "0",
- "schemaVersionPatch": "0"
+ "schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema.json
index 87bc4b647523c6ac0ab810853a94c3f0dba54976..cb41544608a800f73b61de0ea449cd5a1b12e537 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema.json
@@ -6,7 +6,7 @@
"entityType": "test-data--Integration",
"schemaVersionMajor": "3",
"schemaVersionMinor": "0",
- "schemaVersionPatch": "0"
+ "schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_1.json b/testing/indexer-test-core/src/main/resources/testData/records_1.json
deleted file mode 100644
index 33c85b80d024481be7ae416ba9bf6865fef2aa4c..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/testData/records_1.json
+++ /dev/null
@@ -1,113 +0,0 @@
-[
- {
- "id": "tenant1:well:1.0.0:1",
- "kind": "<Kind>",
- "namespace": "tenant1:testdatasource",
- "type": "well",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 1",
- "Location": "18.5204,73.8567",
- "Basin": "India",
- "State": "Maharashtra",
- "Country": "India",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE1",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 1,
- "Score": 10,
- "Established": "1990-03-27T23:38:48Z"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- },
- {
- "id": "tenant1:well:1.0.0:2",
- "kind": "<Kind>",
- "namespace": "tenant1:testdatasource",
- "type": "well",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 3",
- "Location": "29.749655,-95.473476",
- "Basin": "KATY",
- "County": "Waller",
- "State": "TEXAS",
- "Country": "United States",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE3",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 2,
- "Score": 12,
- "Established": "2010-03-27T20:38:48Z"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- },
- {
- "id": "tenant1:well:1.0.0:3",
- "kind": "<Kind>",
- "namespace": "tenant1:testdatasource",
- "type": "well",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 4",
- "Location": "29.6197,-95.6349",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE4",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 20,
- "Established": "2018-03-27T23:38:48Z"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- }
-]
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_1.mapping b/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
deleted file mode 100644
index 311cb573db3d78dcb9d3e1a67721e7fd14b2a6e7..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
+++ /dev/null
@@ -1,92 +0,0 @@
-{
-"dynamic":false,
- "properties": {
- "id": {
- "type": "keyword"
- },
- "kind": {
- "type": "keyword"
- },
- "namespace": {
- "type": "keyword"
- },
- "type": {
- "type": "keyword"
- },
- "version": {
- "type": "keyword"
- },
- "x-acl": {
- "type": "keyword"
- },
- "acl": {
- "properties": {
- "owners": {
- "type": "keyword"
- },
- "viewers": {
- "type": "keyword"
- }
- }
- },
- "legal": {
- "properties": {
- "legaltags": {
- "type": "keyword"
- },
- "otherRelevantDataCountries": {
- "type": "keyword"
- },
- "status": {
- "type": "keyword"
- }
- }
- },
- "data": {
- "properties": {
- "Field": {
- "type": "text"
- },
- "Location": {
- "type": "geo_point"
- },
- "Basin": {
- "type": "text"
- },
- "County": {
- "type": "text"
- },
- "State": {
- "type": "text"
- },
- "Country": {
- "type": "text"
- },
- "WellStatus": {
- "type": "text"
- },
- "OriginalOperator": {
- "type": "text"
- },
- "WellName": {
- "type": "text"
- },
- "WellType": {
- "type": "text"
- },
- "EmptyAttribute": {
- "type": "text"
- },
- "Rank": {
- "type": "integer"
- },
- "Score": {
- "type": "integer"
- },
- "Established": {
- "type": "date"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_2.json b/testing/indexer-test-core/src/main/resources/testData/records_2.json
deleted file mode 100644
index 9227d617e772aeb1d1fd0196b3540d04e9ca62ae..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/testData/records_2.json
+++ /dev/null
@@ -1,109 +0,0 @@
-[
- {
- "id": "tenant1:well:2.0.0:1",
- "kind": "<kind>",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 2",
- "Location": "29.7604,-95.3698",
- "Basin": "Houston",
- "County": "Harris",
- "State": "TX",
- "Country": "USA",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE2",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 1,
- "Score": 10,
- "Established": "2000-03-27T23:38:48Z"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- },
- {
- "id": "tenant1:well:2.0.0:2",
- "kind": "<kind>",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 5",
- "Location": "37.3861,-122.0839",
- "Basin": "Mountain View",
- "County": "Orange",
- "State": "CA",
- "Country": "USA",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE5",
- "WellName": "Data Platform Services",
- "WellType": "Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 2,
- "Score": 5,
- "Established": "1980-03-27T20:38:48Z"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- },
- {
- "id": "tenant1:well:2.0.0:3",
- "kind": "<kind>",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 6",
- "Location": "43.6108, 3.8767",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE6",
- "Country": "France",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 20,
- "Established": "2005-03-27T23:38:48Z"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- }
-]
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_2.mapping b/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
deleted file mode 100644
index 362ad772264ebe956532edee74a283c2a5e112b8..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
+++ /dev/null
@@ -1,89 +0,0 @@
-{
-"dynamic":false,
- "properties": {
- "id": {
- "type": "keyword"
- },
- "kind": {
- "type": "keyword"
- },
- "namespace": {
- "type": "keyword"
- },
- "type": {
- "type": "keyword"
- },
- "version": {
- "type": "keyword"
- },
- "x-acl": {
- "type": "keyword"
- },
- "acl": {
- "properties": {
- "owners": {
- "type": "keyword"
- },
- "viewers": {
- "type": "keyword"
- }
- }
- },
- "legal": {
- "properties": {
- "legaltags": {
- "type": "keyword"
- },
- "otherRelevantDataCountries": {
- "type": "keyword"
- },
- "status": {
- "type": "keyword"
- }
- }
- },
- "data": {
- "properties": {
- "Field": {
- "type": "text"
- },
- "Location": {
- "type": "geo_point"
- },
- "Basin": {
- "type": "text"
- },
- "County": {
- "type": "text"
- },
- "State": {
- "type": "text"
- },
- "Country": {
- "type": "text"
- },
- "WellStatus": {
- "type": "text"
- },
- "OriginalOperator": {
- "type": "text"
- },
- "WellName": {
- "type": "text"
- },
- "WellType": {
- "type": "text"
- },
- "EmptyAttribute": {
- "type": "text"
- },
- "Rank": {
- "type": "integer"
- },
- "Established": {
- "type": "date"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_3.json b/testing/indexer-test-core/src/main/resources/testData/records_3.json
deleted file mode 100644
index 697644cc7d20c7a768d3cc55183a9d4a788e99cb..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/testData/records_3.json
+++ /dev/null
@@ -1,116 +0,0 @@
-[
- {
- "id": "tenant1:well:1.0.0:1",
- "kind": "<Kind>",
- "namespace": "tenant1:testdatasource",
- "type": "well",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 1",
- "Location": "18.5204,73.8567",
- "Basin": "India",
- "State": "Maharashtra",
- "Country": "India",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE1",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 1,
- "Score": 10,
- "Established": "1990-03-27T23:38:48Z",
- "Center": "OFFICE -1"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- },
- {
- "id": "tenant1:well:1.0.0:2",
- "kind": "<Kind>",
- "namespace": "tenant1:testdatasource",
- "type": "well",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 3",
- "Location": "29.749655,-95.473476",
- "Basin": "KATY",
- "County": "Waller",
- "State": "TEXAS",
- "Country": "United States",
- "WellStatus": "Under development",
- "OriginalOperator": "KDFC",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 2,
- "Score": 12,
- "Established": "2010-03-27T20:38:48Z",
- "Center": "KATY"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- },
- {
- "id": "tenant1:well:1.0.0:3",
- "kind": "<Kind>",
- "namespace": "tenant1:testdatasource",
- "type": "well",
- "acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
- "x-acl": [
- "<Data-Group>"
- ],
- "legal": {
- "legaltags": [
- "<Legal-Tag>"
- ],
- "otherRelevantDataCountries": [
- "<Country>"
- ],
- "status": "compliant"
- },
- "data": {
- "Field": "OSDU OFFICE - 4",
- "Location": "29.6197,-95.6349",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE4",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 20,
- "Established": "2018-03-27T23:38:48Z",
- "Center": "SUGARLAND"
- },
- "index": {
- "statusCode": 200,
- "lastUpdateTime": "2018-10-04T10:11:22.303000"
- }
- }
-]
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_3.mapping b/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
deleted file mode 100644
index 3823c22a974043897938553841de212c1d2b50a1..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
+++ /dev/null
@@ -1,100 +0,0 @@
-{
-"dynamic":false,
- "properties": {
- "id": {
- "type": "keyword"
- },
- "kind": {
- "type": "keyword"
- },
- "namespace": {
- "type": "keyword"
- },
- "type": {
- "type": "keyword"
- },
- "version": {
- "type": "keyword"
- },
- "x-acl": {
- "type": "keyword"
- },
- "acl": {
- "properties": {
- "owners": {
- "type": "keyword"
- },
- "viewers": {
- "type": "keyword"
- }
- }
- },
- "legal": {
- "properties": {
- "legaltags": {
- "type": "keyword"
- },
- "otherRelevantDataCountries": {
- "type": "keyword"
- },
- "status": {
- "type": "keyword"
- }
- }
- },
- "data": {
- "properties": {
- "Field": {
- "type": "text"
- },
- "Location": {
- "type": "geo_point"
- },
- "Basin": {
- "type": "text"
- },
- "County": {
- "type": "text"
- },
- "State": {
- "type": "text"
- },
- "Country": {
- "type": "text"
- },
- "WellStatus": {
- "type": "text"
- },
- "OriginalOperator": {
- "type": "text"
- },
- "WellName": {
- "type": "text"
- },
- "WellType": {
- "type": "text"
- },
- "EmptyAttribute": {
- "type": "text"
- },
- "Rank": {
- "type": "integer"
- },
- "Score": {
- "type": "integer"
- },
- "Established": {
- "type": "date"
- },
- "Center": {
- "type": "text",
- "fields": {
- "raw": {
- "type": "keyword"
- }
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index ab965b04ed67538ee2e97d80537afd2d87b81839..0409e9a21c866f5e3e81aad5af72ae01e56b88c0 100644
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -42,9 +42,13 @@ public class Steps extends SchemaServiceRecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index)
- throws Throwable {
- super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..cba0839f389e3f230834c60088862021b86bc271
--- /dev/null
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
@@ -0,0 +1,12 @@
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/info/Info.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/info"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-info.xml"})
+public class RunTest {}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..e9b432e6ca920063543e10e5fe2b1935e85c4a87
--- /dev/null
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -0,0 +1,46 @@
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import org.opengroup.osdu.common.info.InfoSteps;
+import org.opengroup.osdu.util.Config;
+import org.opengroup.osdu.util.GCPHTTPClient;
+
+public class Steps extends InfoSteps {
+
+ public Steps() {
+ super(new GCPHTTPClient());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new GCPHTTPClient();
+ }
+
+ /******************Inputs being set**************/
+
+ @When("^I send get request to version info endpoint")
+ public void i_send_get_request_to_version_info_endpoint() {
+ super.i_send_get_request_to_version_info_endpoint();
+ }
+
+ /******************Assert final response**************/
+
+ @Then("^I should get version info in response$")
+ public void i_should_get_version_info_in_response() {
+ super.i_should_get_version_info_in_response();
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return "GET";
+ }
+
+ @Override
+ protected String getApi() {
+ return Config.getIndexerBaseURL() + "info";
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 8caf85cacd30b7f146c13d0cb58cd156cbf9fc9a..3ec2f0033a5cabb1eba90983c1fc3c4692094e53 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -59,8 +59,13 @@ public class Steps extends SchemaServiceRecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
- super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..cba0839f389e3f230834c60088862021b86bc271
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
@@ -0,0 +1,12 @@
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/info/Info.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/info"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-info.xml"})
+public class RunTest {}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..7524db63d9b6f35e2757972a5fa7ac245b2f7e96
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -0,0 +1,46 @@
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import org.opengroup.osdu.common.info.InfoSteps;
+import org.opengroup.osdu.util.Config;
+import org.opengroup.osdu.util.IBMHTTPClient;
+
+public class Steps extends InfoSteps {
+
+ public Steps() {
+ super(new IBMHTTPClient());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new IBMHTTPClient();
+ }
+
+ /******************Inputs being set**************/
+
+ @When("^I send get request to version info endpoint")
+ public void i_send_get_request_to_version_info_endpoint() {
+ super.i_send_get_request_to_version_info_endpoint();
+ }
+
+ /******************Assert final response**************/
+
+ @Then("^I should get version info in response$")
+ public void i_should_get_version_info_in_response() {
+ super.i_should_get_version_info_in_response();
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return "GET";
+ }
+
+ @Override
+ protected String getApi() {
+ return Config.getIndexerBaseURL() + "info";
+ }
+}
\ No newline at end of file
diff --git a/testing/maven/settings.xml b/testing/maven/settings.xml
deleted file mode 100644
index 2a633d9a30c1997609060dcc31667d3dc1b6774a..0000000000000000000000000000000000000000
--- a/testing/maven/settings.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
- <servers>
- <server>
- <id>os-core</id>
- <username>slb-des-ext-collaboration</username>
- <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
- <!-- The generated token expires on or before 11/14/2019 -->
- <password>${VSTS_FEED_TOKEN}</password>
- </server>
- </servers>
-</settings>
\ No newline at end of file