diff --git a/NOTICE b/NOTICE index 2d0d130ef1c2fc8dda6e40c436cd0baec0f2b0f1..318af935077e87aac709ca934981383e9c28d550 100644 --- a/NOTICE +++ b/NOTICE @@ -203,6 +203,7 @@ The following software have components provided under the terms of this license: - Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client) - Asynchronous Http Client Netty Utils (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client-netty-utils) - AutoValue Annotations (from https://github.com/google/auto/tree/master/value) +- AutoValue Annotations (from https://github.com/google/auto/tree/master/value) - AutoValue Processor (from https://github.com/google/auto/tree/master/value) - Azure Spring Boot Starter for Azure AD Spring Security Integration (from https://github.com/Azure/azure-sdk-for-java) - BSON (from https://bsonspec.org) @@ -234,10 +235,10 @@ The following software have components provided under the terms of this license: - Doxia Sitetools :: Decoration Model (from http://maven.apache.org/doxia/doxia-sitetools/doxia-decoration-model/) - Doxia Sitetools :: Site Renderer Component (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/) - Elastic JNA Distribution (from https://github.com/java-native-access/jna) -- Elastic JNA Distribution (from https://github.com/java-native-access/jna) - Expression Language 3.0 (from http://uel.java.net) - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) - GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson) +- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson) - Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) - Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) - Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine) @@ -266,7 +267,6 @@ The following software have components provided under the terms of this license: - Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) - Guava: Google Core Libraries for Java (from https://repo1.maven.org/maven2/com/google/guava/guava) - HPPC Collections (from https://repo1.maven.org/maven2/com/carrotsearch/hppc) -- HPPC Collections (from https://repo1.maven.org/maven2/com/carrotsearch/hppc) - HTTP functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty) - Hibernate Validator Engine (from https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator) - HttpCore (NIO extensions module) (from http://hc.apache.org/httpcomponents-core/) @@ -332,34 +332,19 @@ The following software have components provided under the terms of this license: - Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core) - Logback Contrib :: Jackson (from ) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Grouping (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping) -- Lucene Grouping (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping) -- Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter) - Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter) - Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join) -- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join) - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs) - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory) -- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory) -- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs) -- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc) - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc) - Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries) -- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries) -- Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser) - Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser) - Lucene Sandbox (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox) -- Lucene Sandbox (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox) -- Lucene Spatial (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial) - Lucene Spatial 3D (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial3d) -- Lucene Spatial 3D (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial3d) -- Lucene Spatial Extras (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial-extras) - Lucene Spatial Extras (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial-extras) - Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest) -- Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest) - MapStruct Core (from https://repo1.maven.org/maven2/org/mapstruct/mapstruct) - Maven Artifact (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact) - Maven Artifact Manager (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact-manager) @@ -507,7 +492,6 @@ The following software have components provided under the terms of this license: - Zipkin Reporter: Core (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter) - aalto-xml (from ) - aggs-matrix-stats (from https://github.com/elastic/elasticsearch) -- aggs-matrix-stats (from https://github.com/elastic/elasticsearch) - aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client) - com.google.api.grpc:proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis) - com.google.api.grpc:proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis) @@ -515,16 +499,11 @@ The following software have components provided under the terms of this license: - com.google.api.grpc:proto-google-common-protos (from https://github.com/googleapis/googleapis) - com.google.api.grpc:proto-google-common-protos (from https://github.com/googleapis/googleapis) - compiler (from http://github.com/spullara/mustache.java) -- compiler (from http://github.com/spullara/mustache.java) - datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client) - elasticsearch-cli (from https://github.com/elastic/elasticsearch) -- elasticsearch-cli (from https://github.com/elastic/elasticsearch) -- elasticsearch-core (from https://github.com/elastic/elasticsearch) - elasticsearch-core (from https://github.com/elastic/elasticsearch) - elasticsearch-geo (from https://github.com/elastic/elasticsearch) - elasticsearch-secure-sm (from https://github.com/elastic/elasticsearch) -- elasticsearch-secure-sm (from https://github.com/elastic/elasticsearch) -- elasticsearch-x-content (from https://github.com/elastic/elasticsearch) - elasticsearch-x-content (from https://github.com/elastic/elasticsearch) - error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) - error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) @@ -534,11 +513,13 @@ The following software have components provided under the terms of this license: - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) - io.grpc:grpc-api (from https://github.com/grpc/grpc-java) +- io.grpc:grpc-api (from https://github.com/grpc/grpc-java) - io.grpc:grpc-auth (from https://github.com/grpc/grpc-java) - io.grpc:grpc-auth (from https://github.com/grpc/grpc-java) - io.grpc:grpc-context (from https://github.com/grpc/grpc-java) - io.grpc:grpc-context (from https://github.com/grpc/grpc-java) - io.grpc:grpc-context (from https://github.com/grpc/grpc-java) +- io.grpc:grpc-context (from https://github.com/grpc/grpc-java) - io.grpc:grpc-core (from https://github.com/grpc/grpc-java) - io.grpc:grpc-core (from https://github.com/grpc/grpc-java) - io.grpc:grpc-core (from https://github.com/grpc/grpc-java) @@ -578,7 +559,6 @@ The following software have components provided under the terms of this license: - jose4j (from https://bitbucket.org/b_c/jose4j/) - json-path (from http://code.google.com/p/json-path/) - lang-mustache (from https://github.com/elastic/elasticsearch) -- lang-mustache (from https://github.com/elastic/elasticsearch) - lettuce (from http://github.com/mp911de/lettuce/wiki) - logging-interceptor (from https://github.com/square/okhttp) - mapper-extras (from https://github.com/elastic/elasticsearch) @@ -596,7 +576,6 @@ The following software have components provided under the terms of this license: - org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/) - org.xmlunit:xmlunit-core (from https://www.xmlunit.org/) - parent-join (from https://github.com/elastic/elasticsearch) -- parent-join (from https://github.com/elastic/elasticsearch) - perfmark:perfmark-api (from https://github.com/perfmark/perfmark) - powermock-module-junit4-common (from https://repo1.maven.org/maven2/org/powermock/powermock-module-junit4-common) - proto-google-cloud-iamcredentials-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-iamcredentials-v1) @@ -606,16 +585,12 @@ The following software have components provided under the terms of this license: - proto-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-pubsub-v1) - proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1) - rank-eval (from https://github.com/elastic/elasticsearch) -- rank-eval (from https://github.com/elastic/elasticsearch) - resilience4j (from https://github.com/resilience4j/resilience4j) - resilience4j (from https://github.com/resilience4j/resilience4j) - rest (from https://github.com/elastic/elasticsearch) -- rest (from https://github.com/elastic/elasticsearch) -- rest-high-level (from https://github.com/elastic/elasticsearch) - rest-high-level (from https://github.com/elastic/elasticsearch) - rxjava (from https://github.com/ReactiveX/RxJava) - server (from https://github.com/elastic/elasticsearch) -- server (from https://github.com/elastic/elasticsearch) - spring-boot-starter-amqp (from https://spring.io/projects/spring-boot) - spring-boot-starter-jersey (from https://spring.io/projects/spring-boot) - spring-boot-starter-json (from https://spring.io/projects/spring-boot) @@ -664,14 +639,11 @@ The following software have components provided under the terms of this license: - GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) - GAX (Google Api eXtensions) for Java (from https://github.com/googleapis/gax-java) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) - Reflections (from http://github.com/ronmamo/reflections) - Stax2 API (from http://github.com/FasterXML/stax2-api) - ThreeTen backport (from https://www.threeten.org/threetenbp) -- ThreeTen backport (from https://www.threeten.org/threetenbp) - jersey-ext-bean-validation (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-bean-validation) - jersey-spring4 (from https://repo1.maven.org/maven2/org/glassfish/jersey/ext/jersey-spring4) @@ -700,14 +672,10 @@ The following software have components provided under the terms of this license: - Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials) - Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials) - Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http) -- Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http) - JavaBeans Activation Framework API jar (from ) - Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common) -- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest) -- Lucene Suggest (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -724,7 +692,6 @@ The following software have components provided under the terms of this license: - SnakeYAML (from http://www.snakeyaml.org) - Spring Core (from https://github.com/spring-projects/spring-framework) - ThreeTen backport (from https://www.threeten.org/threetenbp) -- ThreeTen backport (from https://www.threeten.org/threetenbp) - asm-all-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged) - classworlds (from http://classworlds.codehaus.org/) - jakarta.xml.bind-api (from ) @@ -825,7 +792,6 @@ The following software have components provided under the terms of this license: - JDOM (from http://www.jdom.org) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) -- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Woodstox (from https://github.com/FasterXML/woodstox) ======================================================================== @@ -962,7 +928,6 @@ The following software have components provided under the terms of this license: - Cobertura (from http://cobertura.sourceforge.net) - Commons Lang (from http://commons.apache.org/lang/) - Elastic JNA Distribution (from https://github.com/java-native-access/jna) -- Elastic JNA Distribution (from https://github.com/java-native-access/jna) - Java Native Access (from https://github.com/java-native-access/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) - Javassist (from http://www.javassist.org/) @@ -1017,8 +982,6 @@ The following software have components provided under the terms of this license: - JUL to SLF4J bridge (from http://www.slf4j.org) - Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) -- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) -- Lucene Sandbox (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -1056,7 +1019,6 @@ The following software have components provided under the terms of this license: - mockito-core (from https://github.com/mockito/mockito) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java) -- server (from https://github.com/elastic/elasticsearch) - spring-security-core (from https://spring.io/projects/spring-security) ======================================================================== @@ -1126,7 +1088,6 @@ SunPro ======================================================================== The following software have components provided under the terms of this license: -- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) ======================================================================== @@ -1159,7 +1120,6 @@ X11 ======================================================================== The following software have components provided under the terms of this license: -- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) - Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core) ======================================================================== diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md index 8fee43caf512cb0347543246e7724bb53ad82aaa..842139cfd3e3ec77b6ecf15559f63beb6f7c290c 100644 --- a/provider/indexer-gcp/README.md +++ b/provider/indexer-gcp/README.md @@ -20,14 +20,13 @@ In order to run the service locally or remotely, you will need to have the follo | `LOG_PREFIX` | `service` | Logging prefix | no | - | | `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - | | `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | -| `ENTITLEMENTS_HOST` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | | `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment | -| `INDEXER_HOST` | ex `os-indexer-dot-opendes.appspot.com` | Indexer Host | no | output of infrastructure deployment | | `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment | | `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances | | `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment | | `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | | `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | | `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | | `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | @@ -36,6 +35,8 @@ In order to run the service locally or remotely, you will need to have the follo | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | | `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | | `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - | +| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment | +| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment | ### Run Locally Check that maven is installed: @@ -131,14 +132,12 @@ You will need to have the following environment variables defined. | name | value | description | sensitive? | source | | --- | --- | --- | --- | --- | -| `ENTITLEMENTS_HOST` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | | `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment | | `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment | | `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment | | `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment | | `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment | | `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment | -| `DATA_GROUP` | `opendes` | The service account to this group and substitute | no | - | | `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - | | `INTEGRATION_TEST_AUDIENCE` | `********` | client application ID | yes | https://console.cloud.google.com/apis/credentials | | `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - | @@ -146,11 +145,8 @@ You will need to have the following environment variables defined. | `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - | | `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | | `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - | -| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint | Storage Host | no | output of infrastructure deployment | +| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment | | `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | -| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | -| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - | - **Entitlements configuration for integration accounts** @@ -201,9 +197,9 @@ Create king ring and key in the ***master project*** --purpose encryption ``` -Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the ***master project*** through IAM - Role tab +Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab -Add **Cloud KMS Encrypt/Decrypt** role to the **App Engine default service account** of ***master project*** +Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab #### Memory Store (Redis Instance) Setup diff --git a/provider/indexer-reference/README.md b/provider/indexer-reference/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9fa7b2bb954f311e570265d8734c2fb0e989fe01 --- /dev/null +++ b/provider/indexer-reference/README.md @@ -0,0 +1,217 @@ +# Indexer Service +os-indexer-reference is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU hybrid cloud searches against Elasticsearch. + +## Getting Started +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. + +### Prerequisites +Pre-requisites + +* GCloud SDK with java (latest version) +* JDK 8 +* Lombok 1.16 or later +* Maven + +### Installation +In order to run the service locally or remotely, you will need to have the following environment variables defined. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `LOG_PREFIX` | `service` | Logging prefix | no | - | +| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - | +| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | +| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment | +| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment | +| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances | +| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment | +| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | +| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | +| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | +| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - | +| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment | +| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment | +| `MONGO_DB_URL` | ex `mongodb://localhost:27017` | Mongo DB Url| yes | output of infrastructure deployment | +| `MONGO_DB_USER` | ex `mongouser` | Mongo DB userName| yes | output of infrastructure deployment | +| `MONGO_DB_PASSWORD` | ex `mongopassword` | Mongo DB userPassword| yes | output of infrastructure deployment | +| `MONGO_DB_NAME` | ex `mongoDBName` | Mongo DB DbName| yes | output of infrastructure deployment | +| `MB_RABBITMQ_URI` | ex `amqp://guest:guest@127.0.0.1:5672` | MessageBroker RabbitMQ URI | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | + +### Run Locally +Check that maven is installed: + +```bash +$ mvn --version +Apache Maven 3.6.0 +Maven home: /usr/share/maven +Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre +... +``` + +You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.mvn/community-maven.settings.xml`: + +```bash +$ cat ~/.m2/settings.xml +<?xml version="1.0" encoding="UTF-8"?> +<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> + <servers> + <server> + <id>community-maven-via-private-token</id> + <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. --> + <!-- The generated token expires on or before 11/14/2019 --> + <configuration> + <httpHeaders> + <property> + <name>Private-Token</name> + <value>${env.COMMUNITY_MAVEN_TOKEN}</value> + </property> + </httpHeaders> + </configuration> + </server> + </servers> +</settings> +``` + +* Update the Google cloud SDK to the latest version: + +```bash +gcloud components update +``` +* Set Google Project Id: + +```bash +gcloud config set project <YOUR-PROJECT-ID> +``` + +* Perform a basic authentication in the selected project: + +```bash +gcloud auth application-default login +``` + +* Navigate to indexer service's root folder and run: + +```bash +mvn jetty:run +## Testing +* Navigate to indexer service's root folder and run: + +```bash +mvn clean install +``` + +* If you wish to see the coverage report then go to testing/target/site/jacoco-aggregate and open index.html + +* If you wish to build the project without running tests + +```bash +mvn clean install -DskipTests +``` + +After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.* + +```bash +cd provider/indexer-reference/ && mvn spring-boot:run +``` + +## Testing +Navigate to indexer service's root folder and run all the tests: + +```bash +# build + install integration test core +$ (cd testing/indexer-test-core/ && mvn clean install) +``` + +### Running E2E Tests +This section describes how to run cloud OSDU E2E tests (testing/integration-tests/indexer-test-gcp). + +You will need to have the following environment variables defined. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment | +| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment | +| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment | +| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment | +| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment | +| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment | +| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - | +| `INTEGRATION_TEST_AUDIENCE` | `********` | client application ID | yes | https://console.cloud.google.com/apis/credentials | +| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - | +| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - | +| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - | +| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - | +| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment | +| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | + +**Entitlements configuration for integration accounts** + +| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER | +| --- | --- | +| users<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com | + +Execute following command to build code and run all the integration tests: + +```bash +# Note: this assumes that the environment variables for integration tests as outlined +# above are already exported in your environment. +$ (cd testing/indexer-test-gcp/ && mvn clean test) +``` + +## Deployment +GKE Google Documentation: https://cloud.google.com/build/docs/deploying-builds/deploy-gke +Anthos Google Documentation: https://cloud.google.com/anthos/multicluster-management/gateway/tutorials/cloud-build-integration + +#### Cloud KMS Setup + +Enable cloud KMS on master project + +Create king ring and key in the ***master project*** + +```bash + gcloud services enable cloudkms.googleapis.com + export KEYRING_NAME="csqp" + export CRYPTOKEY_NAME="searchService" + gcloud kms keyrings create $KEYRING_NAME --location global + gcloud kms keys create $CRYPTOKEY_NAME --location global \ + --keyring $KEYRING_NAME \ + --purpose encryption +``` + +Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab + +Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab + +#### Memory Store (Redis Instance) Setup + +Create a new Standard tier Redis instance on the ***service project*** + +The Redis instance must be created under the same region with the App Engine application which needs to access it. + +```bash + gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD +``` + +## Licence +Copyright © Google LLC +Copyright © EPAM Systems + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml index 3aa33b3733a74817ef0ea27c5cd52ca8e11ea288..76da4d50f8b5df385f6862ac06a943c3b85b6619 100644 --- a/provider/indexer-reference/pom.xml +++ b/provider/indexer-reference/pom.xml @@ -42,12 +42,13 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-gcp</artifactId> - <version>0.1.17</version> + <version>0.10.0</version> </dependency> <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>os-core-common</artifactId> + <version>0.11.0-SNAPSHOT</version> </dependency> <dependency> @@ -84,17 +85,14 @@ <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> - <version>6.6.2</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-client</artifactId> - <version>6.6.2</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> - <version>6.6.2</version> </dependency> <!-- Test Dependencies --> @@ -165,7 +163,6 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-actuator-autoconfigure</artifactId> </dependency> - </dependencies> <build> diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java index d2c3e68f06d2da43a8988aa470b58d2002f36888..be11d6bbb0f311e1b9bef0e1e20adf96761d3595 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java @@ -17,7 +17,6 @@ package org.opengroup.osdu.indexer; -import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -25,13 +24,11 @@ import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; @SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) @Configuration -@ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)}) +@ComponentScan(value = {"org.opengroup.osdu"}) public class IndexerAnthosApplication { public static void main(String[] args) { diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java index 6e832b44671a8ac95b65556076bf15122c6990a3..6a3b330aad0dcc5a7ba09687a82e0c5e1f53548c 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java @@ -1,19 +1,16 @@ -/* - * Copyright 2021 Google LLC - * Copyright 2021 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package org.opengroup.osdu.indexer.cache; @@ -26,12 +23,8 @@ import org.springframework.stereotype.Component; @Component public class DatastoreCredentialCache extends RedisCache<String, AccessToken> { - @Autowired - public DatastoreCredentialCache(IndexerConfigurationProperties indexerConfigurationProperties) { - super(indexerConfigurationProperties.getRedisSearchHost(), - Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), - 58 * 60, - String.class, - AccessToken.class); - } + @Autowired + public DatastoreCredentialCache(final IndexerConfigurationProperties configurationProperties) { + super(configurationProperties.getRedisSearchHost(), Integer.parseInt(configurationProperties.getRedisSearchPort()), 58 * 60, String.class, AccessToken.class); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..6f4d9b7505504f259ceca3d3b8e1834dc53f91ba --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java @@ -0,0 +1,39 @@ +/* + Copyright 2020 Google LLC + Copyright 2020 EPAM Systems, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.cache.VmCache; +import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential; +import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.stereotype.Component; + +@Component +public class DatastoreCredentialsCacheFactory extends + AbstractFactoryBean<ICache<String, DatastoreCredential>> { + + @Override + public Class<?> getObjectType() { + return ICache.class; + } + + @Override + protected ICache<String, DatastoreCredential> createInstance() throws Exception { + return new VmCache<>(5 * 60, 20); + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java deleted file mode 100644 index b60b6e590e5ec671110e74fa49949391c219ce23..0000000000000000000000000000000000000000 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 Google LLC - * Copyright 2021 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.di; - - -import com.google.gson.Gson; -import com.mongodb.client.FindIterable; -import com.mongodb.client.MongoCollection; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import org.bson.Document; -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.indexer.persistence.MongoDdmsClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; - -@Primary -@Component -public class TenantFactoryImpl implements ITenantFactory { - - private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); - - public static final String MAIN_DATABASE = "main"; - public static final String TENANT_INFO = "tenantinfo"; - - - private final MongoDdmsClient mongoClient; - - @Autowired - public TenantFactoryImpl(MongoDdmsClient mongoClient) { - this.mongoClient = mongoClient; - } - - private Map<String, TenantInfo> tenants; - - public boolean exists(String tenantName) { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.containsKey(tenantName); - } - - public TenantInfo getTenantInfo(String tenantName) { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.get(tenantName); - } - - public Collection<TenantInfo> listTenantInfo() { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.values(); - } - - public <V> ICache<String, V> createCache(String tenantName, String host, int port, - int expireTimeSeconds, Class<V> classOfV) { - return null; - } - - public void flushCache() { - } - - private void initTenants() { - this.tenants = new HashMap<>(); - MongoCollection<Document> mongoCollection = mongoClient - .getMongoCollection(MAIN_DATABASE, TENANT_INFO); - FindIterable<Document> results = mongoCollection.find(); - if (Objects.isNull(results) && Objects.isNull(results.first())) { - LOG.error(String.format("Collection \'%s\' is empty.", results)); - } - for (Document document : results) { - TenantInfo tenantInfo = new Gson().fromJson(document.toJson(), TenantInfo.class); - this.tenants.put(tenantInfo.getName(), tenantInfo); - } - } - -} - diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java new file mode 100644 index 0000000000000000000000000000000000000000..6ed88c4f209ec55fcc1043d019c67d498c5d36c9 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + +import lombok.extern.java.Log; +import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; +import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory; +import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Log +@Component +@RequestScope +public class TenantFactoryService extends AbstractFactoryBean<ITenantFactory> { + + @Override + protected ITenantFactory createInstance() throws Exception { + return new TenantFactory(); + } + + @Override + public Class<?> getObjectType() { + return ITenantFactory.class; + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java deleted file mode 100644 index 9b7f8b1f5edd751f35dcd850ec57073c594f838f..0000000000000000000000000000000000000000 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2021 Google LLC - * Copyright 2021 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.persistence; - -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.iam.v1.Iam; -import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt; -import com.google.api.services.iam.v1.IamScopes; -import com.google.api.services.iam.v1.model.SignJwtRequest; -import com.google.api.services.iam.v1.model.SignJwtResponse; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.gson.JsonObject; -import java.util.Collections; -import java.util.Date; -import org.apache.commons.lang3.time.DateUtils; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.util.Crc32c; -import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; - -public class DatastoreCredential extends GoogleCredentials { - - private static final long serialVersionUID = 8344377091688956815L; - private static final JsonFactory JSON_FACTORY = new JacksonFactory(); - private Iam iam; - - private final TenantInfo tenant; - private final DatastoreCredentialCache cache; - - protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) { - this.tenant = tenant; - this.cache = cache; - } - - @Override - public AccessToken refreshAccessToken() { - - String cacheKey = this.getCacheKey(); - - AccessToken accessToken = this.cache.get(cacheKey); - - if (accessToken != null) { - return accessToken; - } - - try { - SignJwtRequest signJwtRequest = new SignJwtRequest(); - signJwtRequest.setPayload(this.getPayload()); - - String serviceAccountName = String - .format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount()); - - SignJwt signJwt = this.getIam().projects().serviceAccounts() - .signJwt(serviceAccountName, signJwtRequest); - - SignJwtResponse signJwtResponse = signJwt.execute(); - String signedJwt = signJwtResponse.getSignedJwt(); - - accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600)); - - this.cache.put(cacheKey, accessToken); - - return accessToken; - } catch (Exception e) { - throw new RuntimeException("Error creating datastore credential", e); - } - } - - private String getPayload() { - JsonObject payload = new JsonObject(); - payload.addProperty("iss", this.tenant.getServiceAccount()); - payload.addProperty("sub", this.tenant.getServiceAccount()); - payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore"); - payload.addProperty("iat", System.currentTimeMillis() / 1000); - - return payload.toString(); - } - - protected void setIam(Iam iam) { - this.iam = iam; - } - - private Iam getIam() throws Exception { - if (this.iam == null) { - - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - if (credential.createScopedRequired()) { - credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM)); - } - - Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), - JSON_FACTORY, - credential).setApplicationName("Search Service"); - - this.iam = builder.build(); - } - return this.iam; - } - - - private String getCacheKey() { - return Crc32c - .hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName())); - } -} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java deleted file mode 100644 index 144edee0eb4c1bd99054bd1a1859588adb9be3ae..0000000000000000000000000000000000000000 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2021 Google LLC - * Copyright 2021 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.persistence; - -import com.google.api.gax.retrying.RetrySettings; -import com.google.cloud.TransportOptions; -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.http.HttpTransportOptions; -import java.util.HashMap; -import java.util.Map; -import javax.inject.Inject; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.threeten.bp.Duration; - -@Component -public class DatastoreFactory { - - @Inject - private DatastoreCredentialCache cache; - - @Autowired - public DatastoreFactory(DatastoreCredentialCache cache) { - this.cache = cache; - } - - private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>(); - - private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() - .setMaxAttempts(6) - .setInitialRetryDelay(Duration.ofSeconds(10)) - .setMaxRetryDelay(Duration.ofSeconds(32)) - .setRetryDelayMultiplier(2.0) - .setTotalTimeout(Duration.ofSeconds(50)) - .setInitialRpcTimeout(Duration.ofSeconds(50)) - .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofSeconds(50)) - .build(); - - private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder() - .setReadTimeout(30000) - .build(); - - public Datastore getDatastoreInstance(TenantInfo tenantInfo) { - if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) { - Datastore googleDatastore = DatastoreOptions.newBuilder() - .setCredentials(new DatastoreCredential(tenantInfo, this.cache)) - .setRetrySettings(RETRY_SETTINGS) - .setTransportOptions(TRANSPORT_OPTIONS) - .setNamespace(tenantInfo.getName()) - .setProjectId(tenantInfo.getProjectId()) - .build().getService(); - DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore); - } - return DATASTORE_CLIENTS.get(tenantInfo.getName()); - } -} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java similarity index 95% rename from provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java rename to provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java index 6fd8a720a7fd9cf48729aeb85520f792b7ecae3e..9096cbd6eddbbbee6cd7e63c12506bcf06f40dca 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java @@ -27,7 +27,7 @@ import org.springframework.security.config.annotation.web.configuration.WebSecur @Configuration @EnableWebSecurity @EnableGlobalMethodSecurity(prePostEnabled = true) -public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter { +public class SecurityConfig extends WebSecurityConfigurerAdapter { @Override protected void configure(HttpSecurity http) throws Exception { diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java index 1be2ba81868c6f3ed56cd4b56a25edfdbd576f99..8f72ac9ec6878d86d9e21bb7e7c789aef72cc142 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java @@ -64,7 +64,6 @@ import org.opengroup.osdu.indexer.logging.AuditLogger; import org.opengroup.osdu.indexer.provider.interfaces.IPublisher; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; -import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; @Service @@ -142,6 +141,10 @@ public class IndexerServiceImpl implements IndexerService { retryRecordIds.addAll(deleteFailureRecordIds); } + auditLogger.indexStarted(recordInfos.stream() + .map(RecordInfo::getKind) + .collect(Collectors.toList())); + // process schema change messages Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); if (schemaMsgs != null && !schemaMsgs.isEmpty()) { @@ -245,8 +248,13 @@ public class IndexerServiceImpl implements IndexerService { for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) { String kind = entry.getKey(); - IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false); - if (schemaObj.isDataSchemaMissing()) { + List<String> errors = new ArrayList<>(); + IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, errors); + if (!errors.isEmpty()) { + this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, + HttpStatus.SC_BAD_REQUEST, String.join("|", errors), + String.format("error | kind: %s", kind)); + } else if (schemaObj.isDataSchemaMissing()) { this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind)); @@ -360,6 +368,9 @@ public class IndexerServiceImpl implements IndexerService { document.setVersion(storageRecord.getVersion()); document.setAcl(storageRecord.getAcl()); document.setLegal(storageRecord.getLegal()); + if (storageRecord.getTags() != null) { + document.setTags(storageRecord.getTags()); + } RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId()); if (recordStatus.getIndexProgress().getStatusCode() == 0) { recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK); @@ -439,11 +450,11 @@ public class IndexerServiceImpl implements IndexerService { String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind()); if (operation == OperationType.create) { - IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()) + IndexRequest indexRequest = new IndexRequest(index).id(record.getId()) .source(this.gson.toJson(sourceMap), XContentType.JSON); bulkRequest.add(indexRequest); } else if (operation == OperationType.update) { - UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()) + UpdateRequest updateRequest = new UpdateRequest(index, record.getId()) .upsert(this.gson.toJson(sourceMap), XContentType.JSON); bulkRequest.add(updateRequest); } @@ -465,7 +476,7 @@ public class IndexerServiceImpl implements IndexerService { String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey()); for (String id : record.getValue()) { - DeleteRequest deleteRequest = new DeleteRequest(index, type, id); + DeleteRequest deleteRequest = new DeleteRequest(index, id); bulkRequest.add(deleteRequest); } } @@ -547,6 +558,7 @@ public class IndexerServiceImpl implements IndexerService { indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType()); indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion()); indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl()); + indexerPayload.put(RecordMetaAttribute.TAGS.getValue(), record.getTags()); indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl())); indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal()); indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress()); diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java index 86fb43449ea75340340b0ee43de9e85304c79697..226f74229fce34469f84068c3e468f4d59e3b58a 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java @@ -24,7 +24,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; +import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders; import org.opengroup.osdu.core.gcp.util.TraceIdExtractor; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; @@ -43,10 +43,10 @@ public class DpsHeaderFactoryGcp extends DpsHeaders { .stream() .collect(Collectors.toMap(h -> h, request::getHeader)); - String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT); + String traceContext = headers.get(CloudTaskHeaders.CLOUD_TRACE_CONTEXT); if (!Strings.isNullOrEmpty(traceContext)) { - headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); + headers.put(CloudTaskHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); } this.addFromMap(headers); diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java index 1e3c3f244cb37470a91c6a72185a2007c2578698..b86b91ce440748e52c559da0152b5d3c4060d0a4 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java @@ -1,62 +1,73 @@ /* - * Copyright 2021 Google LLC - * Copyright 2021 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + Copyright 2021 Google LLC + Copyright 2021 EPAM Systems, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ package org.opengroup.osdu.indexer.util; -import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION; - +import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken; +import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; import com.google.common.base.Strings; -import java.util.Map; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.logging.Level; import lombok.extern.java.Log; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.Constants; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; -import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.springframework.beans.factory.annotation.Autowired; +import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; +import javax.inject.Inject; +import java.util.Map; + +import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION; + + @Log @Component @RequestScope public class RequestInfoImpl implements IRequestInfo { - private final IndexerConfigurationProperties indexerConfigurationProperties; - private final TenantInfo tenantInfo; - private final IServiceAccountJwtClient serviceAccountJwtClient; - private final DpsHeaders dpsHeaders; - - @Autowired - public RequestInfoImpl(IndexerConfigurationProperties indexerConfigurationProperties, - TenantInfo tenantInfo, IServiceAccountJwtClient serviceAccountJwtClient, - DpsHeaders dpsHeaders) { - this.indexerConfigurationProperties = indexerConfigurationProperties; - this.tenantInfo = tenantInfo; - this.serviceAccountJwtClient = serviceAccountJwtClient; - this.dpsHeaders = dpsHeaders; - } + @Inject + private DpsHeaders dpsHeaders; + + @Inject + private IServiceAccountJwtClient serviceAccountJwtClient; + + @Inject + private TenantInfo tenantInfo; - private static final String expectedCronHeaderValue = "true"; + @Inject + private IndexerConfigurationProperties properties; + + @Value("${indexer.que.service.mail}") + private String indexerQueServiceMail; + + private static final String EXPECTED_CRON_HEADER_VALUE = "true"; @Override public DpsHeaders getHeaders() { @@ -88,33 +99,65 @@ public class RequestInfoImpl implements IRequestInfo { @Override public boolean isCronRequest() { - String appEngineCronHeader = this.dpsHeaders.getHeaders() - .getOrDefault(AppEngineHeaders.CRON_SERVICE, null); - return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader); + String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(CloudTaskHeaders.CLOUD_CRON_SERVICE, null); + return EXPECTED_CRON_HEADER_VALUE.equalsIgnoreCase(appEngineCronHeader); } @Override public boolean isTaskQueueRequest() { - if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) { + if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){ + log.log(Level.INFO,"Request acknowledged as Cloud task, proceeding token validation"); + return isCloudTaskRequest(); + } + if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){ + log.log(Level.INFO,"Request acknowledged as AppEngine task, proceeding headers validation"); + return isAppEngineTaskRequest(); + } + return false; + } + + private boolean isCloudTaskRequest() { + log.log(Level.INFO,dpsHeaders.getHeaders().toString()); + try { + GoogleIdTokenVerifier verifier = new GoogleIdTokenVerifier.Builder( + GoogleNetHttpTransport.newTrustedTransport(), JacksonFactory.getDefaultInstance()) + .setIssuers(Arrays.asList( + "accounts.google.com", "https://accounts.google.com", + "googleapis.com", "https://www.googleapis.com/auth/userinfo.profile" + ) + ).build(); + String authorization = dpsHeaders.getAuthorization().replace("Bearer ", ""); + + GoogleIdToken googleIdToken = verifier.verify(authorization); + if(googleIdToken.getPayload().getEmail().equals(indexerQueServiceMail)){ + return true; + } + log.log(Level.WARNING,"Token email doesn't match with variable \"indexer.que.service.mail\""); + return false; + + } catch (GeneralSecurityException | IOException e) { + log.log(Level.WARNING,"Not valid or expired cloud task token provided"); return false; } + } - String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME); + private boolean isAppEngineTaskRequest(){ + if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) { + return false; + } + String queueId = this.dpsHeaders.getHeaders().get(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME); return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER); } public String checkOrGetAuthorizationHeader() { - if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment()) - == DeploymentEnvironment.LOCAL) { + if (properties.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) { String authHeader = this.dpsHeaders.getAuthorization(); if (Strings.isNullOrEmpty(authHeader)) { - throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", - "Authorization token cannot be empty"); + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty"); } String user = this.dpsHeaders.getUserEmail(); if (Strings.isNullOrEmpty(user)) { - throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", - "User header cannot be empty"); + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", "User header cannot be empty"); } return authHeader; } else { diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java index 98605a1509c597b4b5f7332560374ff072248d44..673565941dda7a398bba598aab3945be8ac6126b 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java @@ -18,33 +18,14 @@ package org.opengroup.osdu.indexer.util; import com.auth0.jwt.JWT; -import com.auth0.jwt.exceptions.JWTDecodeException; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.iam.v1.Iam; -import com.google.api.services.iam.v1.IamScopes; -import com.google.api.services.iam.v1.model.SignJwtRequest; -import com.google.api.services.iam.v1.model.SignJwtResponse; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.http.HttpHeaders; +import com.google.cloud.iam.credentials.v1.GenerateIdTokenResponse; +import com.google.cloud.iam.credentials.v1.IamCredentialsClient; +import com.google.cloud.iam.credentials.v1.ServiceAccountName; +import java.util.Collections; import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; @@ -53,6 +34,7 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; +import org.opengroup.osdu.core.gcp.multitenancy.credentials.IamCredentialsProvider; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -64,6 +46,7 @@ public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token"; private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s"; + private final IamCredentialsProvider iamCredentialsProvider = new IamCredentialsProvider(); private final JsonFactory JSON_FACTORY = new JacksonFactory(); private final IndexerConfigurationProperties indexerConfigurationProperties; @@ -104,99 +87,24 @@ public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient return cachedToken.getTokenValue(); } - // Getting signed JWT - Map<String, Object> signJwtPayload = this.getJWTCreationPayload(tenant); - - SignJwtRequest signJwtRequest = new SignJwtRequest(); - signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload)); - - String serviceAccountName = String - .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount()); - - Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts() - .signJwt(serviceAccountName, signJwtRequest); - SignJwtResponse signJwtResponse = signJwt.execute(); - String signedJwt = signJwtResponse.getSignedJwt(); - - // Getting id token - List<NameValuePair> postParameters = new ArrayList<>(); - postParameters - .add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); - postParameters.add(new BasicNameValuePair("assertion", signedJwt)); - - HttpPost post = new HttpPost(JWT_AUDIENCE); - post.setHeader(HttpHeaders.CONTENT_TYPE, - ContentType.APPLICATION_FORM_URLENCODED.getMimeType()); - post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); - - try (CloseableHttpClient httpclient = HttpClients.createDefault(); - CloseableHttpResponse httpResponse = httpclient.execute(post)) { - JsonObject jsonContent = new JsonParser() - .parse(EntityUtils.toString(httpResponse.getEntity())) - .getAsJsonObject(); - - if (!jsonContent.has("id_token")) { - log.error(String.format("Google IAM response: %s", jsonContent.toString())); - throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", - "The user is not authorized to perform this action"); - } - - String token = jsonContent.get("id_token").getAsString(); + try (IamCredentialsClient iamCredentialsClient = iamCredentialsProvider + .getIamCredentialsClient()) { + ServiceAccountName serviceAccountName = ServiceAccountName + .parse(String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getServiceAccount())); + GenerateIdTokenResponse idTokenResponse = iamCredentialsClient + .generateIdToken(serviceAccountName, Collections + .emptyList(), indexerConfigurationProperties.getGoogleAudiences(), true); + String token = idTokenResponse.getToken(); IdToken idToken = IdToken.builder().tokenValue(token) .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build(); - this.cacheService.put(tenant.getServiceAccount(), idToken); - return token; } - } catch (JWTDecodeException e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", - "Invalid token, error decoding", e); } catch (AppException e) { throw e; } catch (Exception e) { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", - "Error generating token", - e); - } - } - - public Iam getIam() throws Exception { - - if (this.iam == null) { - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - - // Authenticate using Google Application Default Credentials. - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - if (credential.createScopedRequired()) { - List<String> scopes = new ArrayList<>(); - // Enable full Cloud Platform scope. - scopes.add(IamScopes.CLOUD_PLATFORM); - credential = credential.createScoped(scopes); - } - - // Create IAM API object associated with the authenticated transport. - this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(indexerConfigurationProperties.getIndexerHost()) - .build(); + "Error generating token", e); } - - return this.iam; - } - - private Map<String, Object> getJWTCreationPayload(TenantInfo tenantInfo) { - - Map<String, Object> payload = new HashMap<>(); - String googleAudience = indexerConfigurationProperties.getGoogleAudiences(); - if (googleAudience.contains(",")) { - googleAudience = googleAudience.split(",")[0]; - } - payload.put("target_audience", googleAudience); - payload.put("exp", System.currentTimeMillis() / 1000 + 3600); - payload.put("iat", System.currentTimeMillis() / 1000); - payload.put("iss", tenantInfo.getServiceAccount()); - payload.put("aud", JWT_AUDIENCE); - - return payload; } } diff --git a/provider/indexer-reference/src/main/resources/application.properties b/provider/indexer-reference/src/main/resources/application.properties index 4f83a153bbfe8c8f1969d57bed237c6f162983ca..542569ff56509144da5ba45d9d2208b8345b60a9 100644 --- a/provider/indexer-reference/src/main/resources/application.properties +++ b/provider/indexer-reference/src/main/resources/application.properties @@ -67,6 +67,7 @@ crs-api=example.com ## use below values for gcp: opendes redis-group-host=127.0.0.1 redis-search-host=127.0.0.1 +redis-search-port=6379 google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com