diff --git a/NOTICE b/NOTICE index 495d15c3e70ed0b70fc37ad2ecce1261d4644830..31690502fe7bc60b37552c42ae2d6705b1460c08 100644 --- a/NOTICE +++ b/NOTICE @@ -112,6 +112,7 @@ The following software have components provided under the terms of this license: - Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary) - Jackson dataformat: Smile (from http://github.com/FasterXML/jackson-dataformats-binary) - Jackson datatype: JSR310 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310) +- Jackson datatype: Joda (from https://github.com/FasterXML/jackson-datatype-joda) - Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8) - Jackson extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson) - Jackson module: Afterburner (from https://github.com/FasterXML/jackson-modules-base) @@ -121,7 +122,6 @@ The following software have components provided under the terms of this license: - Jackson-core (from https://github.com/FasterXML/jackson-core) - Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding) - Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson) -- Jackson-datatype-Joda (from http://wiki.fasterxml.com/JacksonModuleJoda) - Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names) - Jakarta Bean Validation API (from https://beanvalidation.org) - Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) @@ -135,7 +135,7 @@ The following software have components provided under the terms of this license: - Javassist (from http://www.javassist.org/) - Javassist (from http://www.javassist.org/) - JetBrains Java Annotations (from https://github.com/JetBrains/java-annotations) -- Joda time (from http://joda-time.sourceforge.net) +- Joda-Time (from https://www.joda.org/joda-time/) - KeePassJava2 :: All (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2) - KeePassJava2 :: DOM (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-dom) - KeePassJava2 :: JAXB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-jaxb) @@ -150,8 +150,8 @@ The following software have components provided under the terms of this license: - Lucene Grouping (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping) - Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter) - Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join) -- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory) - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs) +- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory) - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc) - Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries) - Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser) @@ -202,6 +202,8 @@ The following software have components provided under the terms of this license: - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) - Objenesis (from http://objenesis.org) +- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp) +- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp) - OkHttp Logging Interceptor (from https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) @@ -222,13 +224,24 @@ The following software have components provided under the terms of this license: - SnakeYAML (from http://www.snakeyaml.org) - Spring AOP (from https://github.com/spring-projects/spring-framework) - Spring Beans (from https://github.com/spring-projects/spring-framework) +- Spring Boot (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot) +- Spring Boot AOP Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-aop) - Spring Boot Actuator (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator) - Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure) - Spring Boot Actuator Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-actuator) -- Spring Boot Reactor Netty Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-reactor-netty) +- Spring Boot AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-autoconfigure) +- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json) +- Spring Boot Logging Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-logging) +- Spring Boot Security Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-security) +- Spring Boot Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter) +- Spring Boot Test (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-test) +- Spring Boot Test Auto-Configure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-test-autoconfigure) +- Spring Boot Test Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-test) +- Spring Boot Tomcat Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat) - Spring Boot Validation Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-validation) - Spring Boot Validation Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-validation) -- Spring Boot WebFlux Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-webflux) +- Spring Boot Web Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-web) +- Spring Boot Web Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-web) - Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework) - Spring Context (from https://github.com/spring-projects/spring-framework) - Spring Core (from https://github.com/spring-projects/spring-framework) @@ -306,8 +319,6 @@ The following software have components provided under the terms of this license: - mockito-core (from http://mockito.org) - nio-multipart-parser (from ) - nio-stream-storage (from https://github.com/synchronoss/nio-stream-storage) -- okhttp (from https://square.github.io/okhttp/) -- okhttp (from https://square.github.io/okhttp/) - okhttp-urlconnection (from https://github.com/square/okhttp) - okhttp-urlconnection (from https://github.com/square/okhttp) - okio (from https://github.com/square/okio/) @@ -338,23 +349,12 @@ The following software have components provided under the terms of this license: - rest (from https://github.com/elastic/elasticsearch) - rest-high-level (from https://github.com/elastic/elasticsearch) - rxjava (from https://github.com/ReactiveX/RxJava) -- spring-boot (from https://spring.io/projects/spring-boot) -- spring-boot-autoconfigure (from https://spring.io/projects/spring-boot) - spring-boot-dependencies (from https://spring.io/projects/spring-boot) -- spring-boot-starter (from https://spring.io/projects/spring-boot) -- spring-boot-starter-aop (from https://spring.io/projects/spring-boot) -- spring-boot-starter-json (from https://spring.io/projects/spring-boot) - spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot) -- spring-boot-starter-logging (from https://spring.io/projects/spring-boot) -- spring-boot-starter-security (from https://spring.io/projects/spring-boot) -- spring-boot-starter-test (from https://spring.io/projects/spring-boot) -- spring-boot-starter-tomcat (from https://spring.io/projects/spring-boot) +- spring-boot-starter-reactor-netty (from https://spring.io/projects/spring-boot) - spring-boot-starter-undertow (from https://spring.io/projects/spring-boot) - spring-boot-starter-undertow (from https://spring.io/projects/spring-boot) -- spring-boot-starter-web (from https://spring.io/projects/spring-boot) -- spring-boot-starter-web (from https://spring.io/projects/spring-boot) -- spring-boot-test (from https://spring.io/projects/spring-boot) -- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot) +- spring-boot-starter-webflux (from https://spring.io/projects/spring-boot) - spring-security-config (from http://spring.io/spring-security) - spring-security-config (from http://spring.io/spring-security) - spring-security-core (from http://spring.io/spring-security) @@ -722,7 +722,7 @@ The following software have components provided under the terms of this license: - Javassist (from http://www.javassist.org/) - Javassist (from http://www.javassist.org/) -- okhttp (from https://square.github.io/okhttp/) +- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp) ======================================================================== MS-RL @@ -778,7 +778,7 @@ The following software have components provided under the terms of this license: - Guava: Google Core Libraries for Java (from https://github.com/google/guava) - Guava: Google Core Libraries for Java (from https://github.com/google/guava) - HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/) -- Joda time (from http://joda-time.sourceforge.net) +- Joda-Time (from https://www.joda.org/joda-time/) - LatencyUtils (from http://latencyutils.github.io/LatencyUtils/) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java) diff --git a/pom.xml b/pom.xml index e180807493e1a530dab432b12edad3c254c5bc00..f06e315558e4c86423f415e1f82873410511a079 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ <module>provider/notification-azure</module> <module>provider/notification-ibm</module> <module>provider/notification-aws</module> + <module>provider/notification-reference</module> </modules> <repositories> diff --git a/provider/notification-reference/README.md b/provider/notification-reference/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6cfa317806a48ecb47185d38bcfd40a3fc7e70a3 --- /dev/null +++ b/provider/notification-reference/README.md @@ -0,0 +1,184 @@ +# Notification Service +notification-reference is a [Spring Boot](https://spring.io/projects/spring-boot) service that allow for interested consumers to subscribe to data and metadata changes using a publish/subscriber pattern. +This service could be used for OSDU hybrid cloud. + +## Getting Started +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. + +### Requirements +* Java 8 +* [Maven 3.6.0+](https://maven.apache.org/download.cgi) +* GCloud command line tool +* GCloud access to opendes project + +### General Tips + +**Environment Variable Management** +The following tools make environment variable configuration simpler + - [direnv](https://direnv.net/) - for a shell/terminal environment + - [EnvFile](https://plugins.jetbrains.com/plugin/7861-envfile) - for [Intellij IDEA](https://www.jetbrains.com/idea/) + +**Lombok** +This project uses [Lombok](https://projectlombok.org/) for code generation. You may need to configure your IDE to take advantage of this tool. + - [Intellij configuration](https://projectlombok.org/setup/intellij) + - [VSCode configuration](https://projectlombok.org/setup/vscode) + +### Installation +In order to run the service locally or remotely, you will need to have the following environment variables defined. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `APP_ENTITLEMENTS` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | +| `APP_REGISTER` | ex `https://register.com/api/register/v1` | Storage API endpoint | no | output of infrastructure deployment | +| `APP_PROJECT` | ex `opendes` | Google Cloud Project Id | no | output of infrastructure deployment | +| `APP_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | +| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - | + +**System Environment required to run service** + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `SPRING_PROFILES_ACTIVE` | `local` | spring active profile | no | + +### Run Locally +Check that maven is installed: +```bash +$ mvn --version +Apache Maven 3.6.0 +Maven home: /usr/share/maven +Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre +... +``` + +You will need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.m2/settings.xml`: + +```bash +$ cat ~/.m2/settings.xml +<?xml version="1.0" encoding="UTF-8"?> +<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> + <servers> + <server> + <id>os-core</id> + <username>slb-des-ext-collaboration</username> + <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. --> + <password>${VSTS_FEED_TOKEN}</password> + </server> + </servers> +</settings> +``` + +* Update the Google cloud SDK to the latest version: + +```bash +gcloud components update +``` +* Set Google Project Id: + +```bash +gcloud config set project <YOUR-PROJECT-ID> +``` + +* Perform a basic authentication in the selected project: + +```bash +gcloud auth application-default login +``` + +* Navigate to notification service's root folder and run: + +```bash +mvn jetty:run +## Testing +* Navigate to notification service's root folder and run: + +```bash +mvn clean install +``` + +* If you wish to see the coverage report then go to testing/target/site/jacoco-aggregate and open index.html + +* If you wish to build the project without running tests + +```bash +mvn clean install -DskipTests +``` + +After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.* + +```bash +cd provider/notification-reference/ && mvn spring-boot:run -Dspring-boot.run.profiles=local +``` + +## Testing +Navigate to notification service's root folder and run all the tests: + +```bash +# build + test + install core service code +$ (cd notification-core/ && mvn clean install) +``` + +## Test the application + +After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/api/notification/v1/swagger-ui.html](http://localhost:8080/swagger-ui.html). If the request does not fail, you can then run the integration tests. + +### Dependencies needed to run the integration tests +* Java 8 +* Maven +* Values for the following environment variables in Config.java + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `DE_OPS_TESTER` | `*****` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already, also **Private key id** of this account must be set in Register service variable SUBSCRIBER_PRIVATE_KEY_ID | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `DE_ADMIN_TESTER` | `*****` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `DE_EDITOR_TESTER` | `*****` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `DE_NO_ACCESS_TESTER` | `*****` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `ENVIRONMENT` | `dev` OR `local` OR `dev_gke`| Local for running locally with services url's predefined as http://localhost , Dev & Dev_Gke is configurable environment | no | - | +| `HMAC_SECRET` | ex`7a786376626e` | String in hex , must match pattern ^[a-zA-Z0-9]{8,30}+$ & be in register variable SUBSCRIBER_SECRET | yes | - | +| `REGISTER_BASE_URL` | `http://localhost:8081/api/register/v1` | Register service url | no | - | +| `NOTIFICATION_BASE_URL` | `http://localhost:8080/api/notification/v1/` | Notification service url | no | - | +| `INTEGRATION_TEST_AUDIENCE` | `********` | Client application ID | yes | https://console.cloud.google.com/apis/credentials | +| `CLIENT_TENANT` | ex `opendes` | Client tenant | no | - | +| `OSDU_TENANT` | ex `osdu` | Osdu tenant | no | - | +| `TOPIC_ID` | ex `records-changed` | PubSub topic id | no | https://console.cloud.google.com/cloudpubsub/topic | +| `REGISTER_CUSTOM_PUSH_URL_HMAC` | ex `http://localhost:8081/api/register/v1/test/challenge/hmac-integration-test` | Register testing push url | no | - | + + **Entitlements configuration for integration accounts** + + | DE_OPS_TESTER | DE_ADMIN_TESTER | DE_EDITOR_TESTER | DE_NO_ACCESS_TESTER | + | --- | --- | --- | --- | + |notification.pubsub<br/>service.entitlements.user<br/>users<br/>users.datalake.ops</br>| service.entitlements.user<br/>users<br/>users.datalake.admins</br> | service.entitlements.user<br/>users<br/>users.datalake.editors</br> | service.entitlements.user<br/>users<br/>| + +Above variables should be configured in the release pipeline to run integration tests. You should also replace them with proper values if you wish to run tests locally. + +### Commands to run tests +* Integration tests are refactored into two pieces: Core and Provider. Core contains business logic for tests and is a dependency for executing the tests from provider module. To build the core module, simply navigate to `notification-test-core` directory and run `mvn clean install`. This will build the core module +* Next, to execute the integration tests, navigate to the provider module and execute `mvn test` +```bash +# (cd testing/notification-test-core/ && mvn clean install) +# Note: this assumes that the environment variables for integration tests as outlined +# above are already exported in your environment. +$ (cd testing/notification-test-gcp/ && mvn clean test) +``` + +## Deployment +GKE Google Documentation: https://cloud.google.com/build/docs/deploying-builds/deploy-gke +Anthos Google Documentation: https://cloud.google.com/anthos/multicluster-management/gateway/tutorials/cloud-build-integration + +## License +Copyright © Google LLC + +Copyright © EPAM Systems + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/provider/notification-reference/kubernetes/deployments/deployment-os-notification-service.yml b/provider/notification-reference/kubernetes/deployments/deployment-os-notification-service.yml new file mode 100644 index 0000000000000000000000000000000000000000..ff445d3cdade8484695dd3f9e57ea88d66af7481 --- /dev/null +++ b/provider/notification-reference/kubernetes/deployments/deployment-os-notification-service.yml @@ -0,0 +1,81 @@ +apiVersion: v1 +data: + APP_ENTITLEMENTS: ${APP_ENTITLEMENTS} + APP_REGISTER: ${APP_REGISTER} + APP_PROJECT: ${APP_PROJECT} + APP_AUDIENCES: ${APP_AUDIENCES} + PARTITION_API: ${PARTITION_API} + +kind: ConfigMap +metadata: + labels: + app: notification-reference + name: notification-config + namespace: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + generateName: notification-reference-anthos + labels: + app: notification-reference + name: notification-reference + namespace: default +spec: + selector: + matchLabels: + app: notification-reference + replicas: 1 + template: + metadata: + labels: + app: notification-reference + spec: + containers: + - env: + - name: APP_ENTITLEMENTS + valueFrom: + configMapKeyRef: + key: APP_ENTITLEMENTS + name: notification-config + - name: APP_REGISTER + valueFrom: + configMapKeyRef: + key: APP_REGISTER + name: notification-config + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + key: LOG_LEVEL + name: notification-config + - name: APP_PROJECT + valueFrom: + configMapKeyRef: + key: APP_PROJECT + name: notification-config + - name: APP_AUDIENCES + valueFrom: + configMapKeyRef: + key: APP_AUDIENCES + name: notification-config + - name: PARTITION_API + valueFrom: + configMapKeyRef: + key: PARTITION_API + name: notification-config + image: us.gcr.io/osdu-anthos-02/os-notification/anthos-notification-reference:9966597-dirty + name: notification-reference +--- +apiVersion: v1 +kind: Service +metadata: + name: notification-reference + namespace: default +spec: + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + selector: + app: notification-reference + type: LoadBalancer \ No newline at end of file diff --git a/provider/notification-reference/pom.xml b/provider/notification-reference/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..cd5610ffa7fdd0a6db36e08998ed93bdda91674d --- /dev/null +++ b/provider/notification-reference/pom.xml @@ -0,0 +1,154 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.opengroup.osdu</groupId> + <artifactId>os-notification</artifactId> + <version>0.12.0-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + + <groupId>org.opengroup.osdu</groupId> + <artifactId>notification-reference</artifactId> + <version>0.12.0-SNAPSHOT</version> + <packaging>jar</packaging> + + <properties> + <java.version>8</java.version> + <maven.compiler.target>${java.version}</maven.compiler.target> + <maven.compiler.source>${java.version}</maven.compiler.source> + </properties> + + <dependencies> + <dependency> + <groupId>org.opengroup.osdu</groupId> + <artifactId>core-lib-gcp</artifactId> + <version>0.11.0</version> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-amqp</artifactId> + </dependency> + + <dependency> + <groupId>org.opengroup.osdu</groupId> + <artifactId>notification-core</artifactId> + <version>0.12.0-SNAPSHOT</version> + </dependency> + + <dependency> + <groupId>org.opengroup.osdu</groupId> + <artifactId>os-core-common</artifactId> + </dependency> + + <dependency> + <groupId>ch.qos.logback.contrib</groupId> + <artifactId>logback-json-classic</artifactId> + <version>0.1.5</version> + </dependency> + + <!-- unit test dependencies --> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito2</artifactId> + <version>2.0.2</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>2.0.2</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>2.0.2-beta</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>2.0.2</version> + <scope>test</scope> + </dependency> + + </dependencies> + + + <build> + <plugins> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <configuration> + <profiles> + <profile> + <id>local</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <properties> + <spring.profiles.active>local</spring.profiles.active> + </properties> + </profile> + <profile> + <id>dev</id> + <properties> + <spring.profiles.active>dev</spring.profiles.active> + </properties> + </profile> + </profiles> + </configuration> + <executions> + <execution> + <goals> + <goal>repackage</goal> + </goals> + <configuration> + <classifier>spring-boot</classifier> + <mainClass> + org.opengroup.osdu.notification.provider.reference.Application + </mainClass> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-war-plugin</artifactId> + <configuration> + <failOnMissingWebXml>false</failOnMissingWebXml> + </configuration> + </plugin> + <plugin> + <groupId>org.jacoco</groupId> + <artifactId>jacoco-maven-plugin</artifactId> + <version>0.7.7.201606060606</version> + <executions> + <execution> + <goals> + <goal>prepare-agent</goal> + </goals> + </execution> + <execution> + <id>report</id> + <phase>prepare-package</phase> + <goals> + <goal>report</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/Application.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/Application.java new file mode 100644 index 0000000000000000000000000000000000000000..e7639be09c10c143fd035434c5f62f9f53d2bcda --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/Application.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.ComponentScan.Filter; +import org.springframework.context.annotation.FilterType; +import org.springframework.scheduling.annotation.EnableAsync; + +@SpringBootApplication +@ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = { + @Filter( + type = FilterType.REGEX, + pattern = {"org.opengroup.osdu.core.gcp.multitenancy.StorageFactory"} + ) +}) +@EnableAsync +public class Application { + + public static void main(String[] args) { + SpringApplication.run(new Class[]{Application.class}, args); + } +} \ No newline at end of file diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/config/PropertiesConfiguration.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/config/PropertiesConfiguration.java new file mode 100644 index 0000000000000000000000000000000000000000..c34830e3c279a9bcaca126ba82efae448752618a --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/config/PropertiesConfiguration.java @@ -0,0 +1,38 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Data +public class PropertiesConfiguration { + + private String authorizeAPI; + private String registerAPI; + private String projectId; + private Integer expireTime = 300; + private Integer maxCacheSize = 10; + + private String googleCloudProject; + private String googleCloudProjectRegion; + private String googleAudiences; +} diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/pubsub/PubsubHandshakeHandler.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/pubsub/PubsubHandshakeHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..6e3d7e7dd1f9cfbf2a0f904b68653f089a66cb3c --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/pubsub/PubsubHandshakeHandler.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.pubsub; + +import org.opengroup.osdu.notification.provider.interfaces.IPubsubHandshakeHandler; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Component +@Lazy +public class PubsubHandshakeHandler implements IPubsubHandshakeHandler { + + @Override + public String getHandshakeResponse() { + return null; + } +} \ No newline at end of file diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/pubsub/PubsubRequestBodyExtractor.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/pubsub/PubsubRequestBodyExtractor.java new file mode 100644 index 0000000000000000000000000000000000000000..8928a72498717e86317789d054d95eecb65eca31 --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/pubsub/PubsubRequestBodyExtractor.java @@ -0,0 +1,144 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.pubsub; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.BufferedReader; +import java.io.IOException; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.servlet.http.HttpServletRequest; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.storage.MessageContent; +import org.opengroup.osdu.notification.provider.interfaces.IPubsubRequestBodyExtractor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +public class PubsubRequestBodyExtractor implements IPubsubRequestBodyExtractor { + + private static final String INVALID_PUBSUB_MESSAGE = "Invalid pubsub message"; + private static final Gson GSON = new Gson(); + private MessageContent messageContent; + private JsonObject root = null; + + @Autowired + private HttpServletRequest request; + + @Autowired + private JaxRsDpsLog log; + + public Map<String, String> extractAttributesFromRequestBody() { + if (this.messageContent == null) { + this.messageContent = this.extractPubsubMessageFromRequestBody(); + } + return this.messageContent.getAttributes(); + } + + public String extractDataFromRequestBody() { + if (this.messageContent == null) { + this.messageContent = this.extractPubsubMessageFromRequestBody(); + } + return this.messageContent.getData(); + } + + public String extractNotificationIdFromRequestBody() { + if (this.root == null) { + this.root = this.extractRootJsonElementFromRequestBody(); + } + JsonElement subscription = this.root.get("subscription"); + if (subscription == null) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), INVALID_PUBSUB_MESSAGE, + "subscription object not found"); + } + + String[] fullNotificationId = subscription.getAsString().split("/"); + return fullNotificationId[fullNotificationId.length - 1]; + } + + @Override + public boolean isHandshakeRequest() { + return false; + } + + private MessageContent extractPubsubMessageFromRequestBody() { + if (this.root == null) { + this.root = this.extractRootJsonElementFromRequestBody(); + } + JsonElement message = this.root.get("message"); + if (message == null) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), INVALID_PUBSUB_MESSAGE, + "message object not found"); + } + MessageContent content = GSON.fromJson(message.toString(), MessageContent.class); + + Map<String, String> attributes = content.getAttributes(); + if (attributes == null || attributes.isEmpty()) { + log.error("Incorrect Message: " + message.toString()); + throw new AppException(HttpStatus.BAD_REQUEST.value(), INVALID_PUBSUB_MESSAGE, + "attribute map not found"); + } + String data = content.getData(); + if (Strings.isNullOrEmpty(data)) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), INVALID_PUBSUB_MESSAGE, + "data field not found"); + } + Map<String, String> lowerCase = new HashMap<>(); + attributes.forEach((key, value) -> lowerCase.put(key.toLowerCase(), value)); + if (Strings.isNullOrEmpty(attributes.get("data-partition-id"))) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), INVALID_PUBSUB_MESSAGE, + "No tenant information from pubsub message."); + } + content.setAttributes(lowerCase); + + String decoded = new String(Base64.getDecoder().decode(data)); + content.setData(decoded); + + return content; + } + + private JsonObject extractRootJsonElementFromRequestBody() { + try { + JsonParser jsonParser = new JsonParser(); + BufferedReader reader = request.getReader(); + Stream<String> lines = reader.lines(); + String requestBody = lines.collect(Collectors.joining("\n")); + JsonElement rootElement = jsonParser.parse(requestBody); + if (!(rootElement instanceof JsonObject)) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), "RequestBody is not JsonObject.", + "Request Body should be JsonObject to be processed."); + } + return rootElement.getAsJsonObject(); + } catch (IOException e) { + throw new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), + "Request payload parsing error", + "Unable to parse request payload.", e); + } + } +} diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/security/SecurityConfig.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/security/SecurityConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..a60a619cdebadfddf4c31a9033587d9a93d5a5b8 --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/security/SecurityConfig.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.security; + +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; + +@Configuration +@EnableGlobalMethodSecurity(prePostEnabled = true) +public class SecurityConfig extends WebSecurityConfigurerAdapter { + + @Override + protected void configure(HttpSecurity httpSecurity) throws Exception { + httpSecurity + .httpBasic().disable() + .csrf().disable(); + } +} \ No newline at end of file diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/AppProperties.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/AppProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..baaa243682eea04f7023ade9cc04c18ef393ccb2 --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/AppProperties.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.util; + +import lombok.RequiredArgsConstructor; +import org.opengroup.osdu.notification.provider.interfaces.IAppProperties; +import org.opengroup.osdu.notification.provider.reference.config.PropertiesConfiguration; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class AppProperties implements IAppProperties { + + private final PropertiesConfiguration propertiesConfiguration; + + public String getAuthorizeAPI() { + return propertiesConfiguration.getAuthorizeAPI(); + } + + public String getRegisterAPI() { + return propertiesConfiguration.getRegisterAPI(); + } + + public String getPubSubServiceAccountEmail() { + return String.format("de-notification-service@%s.iam.gserviceaccount.com", + propertiesConfiguration.getProjectId()); + } + + public String getGoogleAudiences() { + return propertiesConfiguration.getGoogleAudiences(); + } +} diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountImpl.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..7f511a67ed894418932f029b1b4037423042b9b2 --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.util; + +import lombok.SneakyThrows; +import org.apache.http.impl.client.CloseableHttpClient; +import org.opengroup.osdu.core.gcp.GoogleIdToken.IGoogleIdTokenFactory; +import org.opengroup.osdu.notification.provider.interfaces.IGoogleServiceAccount; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class GoogleServiceAccountImpl implements IGoogleServiceAccount { + + @Autowired + private IGoogleIdTokenFactory googleIdTokenFactory; + @Autowired + private CloseableHttpClient closeableHttpClient; + + @SneakyThrows + @Override + public String getIdToken(String keyString, String audience) { + return this.googleIdTokenFactory.getGoogleIdToken(keyString, audience, + this.closeableHttpClient); + } +} \ No newline at end of file diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountValidatorGenerator.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountValidatorGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..462f9012c4ceee99a22bd7d5230bb51ee5a2abb6 --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountValidatorGenerator.java @@ -0,0 +1,42 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.util; + +import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; +import java.util.Arrays; +import org.springframework.stereotype.Component; + +@Component +public class GoogleServiceAccountValidatorGenerator { + + public GoogleIdTokenVerifier getVerifier(NetHttpTransport transport, JacksonFactory factory, + String... googleAudiences) { + GoogleIdTokenVerifier verifier; + if (googleAudiences == null || googleAudiences.length == 0) { + verifier = new GoogleIdTokenVerifier.Builder(transport, factory) + .build(); + } else { + verifier = new GoogleIdTokenVerifier.Builder(transport, factory) + .setAudience(Arrays.asList(googleAudiences)) + .build(); + } + return verifier; + } +} diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountValidatorImpl.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountValidatorImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..5841f8a7ea7f25eefacc0fda1bd5041fbb68a57a --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/GoogleServiceAccountValidatorImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.util; + +import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken; +import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.notification.provider.interfaces.IServiceAccountValidator; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class GoogleServiceAccountValidatorImpl implements IServiceAccountValidator { + + private final NetHttpTransport netHttpTransport = new NetHttpTransport(); + private final JacksonFactory jacksonFactory = new JacksonFactory(); + + @Autowired + private JaxRsDpsLog log; + @Autowired + private AppProperties appConfig; + @Autowired + private GoogleServiceAccountValidatorGenerator verifierGenerator; + + @Override + public boolean isValidPublisherServiceAccount(String jwt) { + return isValidServiceAccount(jwt, this.appConfig.getPubSubServiceAccountEmail()); + } + + @Override + public boolean isValidServiceAccount(String jwt, String userIdentity, String... googleAudiences) { + GoogleIdTokenVerifier verifier = this.verifierGenerator.getVerifier(this.netHttpTransport, + this.jacksonFactory, googleAudiences); + try { + GoogleIdToken idToken = verifier.verify(jwt); + if (idToken != null) { + GoogleIdToken.Payload payload = idToken.getPayload(); + + String email = payload.getEmail(); + Boolean emailVerified = payload.getEmailVerified(); + + return (emailVerified && (email.equalsIgnoreCase(userIdentity))); + } else { + return false; + } + } catch (Exception e) { + this.log.error("Error when validating google id token", e); + return false; + } + } +} diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/JwtValidity.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/JwtValidity.java new file mode 100644 index 0000000000000000000000000000000000000000..282ad4c19c4aadae3306bb3c4db0b430aee6debe --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/JwtValidity.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.util; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class JwtValidity { + + String token; + long expiryTime; + + JwtValidity(String jwt, long expiryTime) { + this.token = jwt; + this.expiryTime = expiryTime; + } +} diff --git a/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/ServiceAccountJwtGcpClientImpl.java b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/ServiceAccountJwtGcpClientImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..0bf66a94a46a2a85e3a98a5d799634364a96351f --- /dev/null +++ b/provider/notification-reference/src/main/java/org/opengroup/osdu/notification/provider/reference/util/ServiceAccountJwtGcpClientImpl.java @@ -0,0 +1,223 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.provider.reference.util; + +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.services.iam.v1.Iam; +import com.google.api.services.iam.v1.IamScopes; +import com.google.api.services.iam.v1.model.SignJwtRequest; +import com.google.api.services.iam.v1.model.SignJwtResponse; +import com.google.auth.http.HttpCredentialsAdapter; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.common.base.Strings; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; +import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory; + +public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient { + + private AppProperties config; + private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token"; + private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s"; + private static final JsonFactory JSON_FACTORY = new JacksonFactory(); + static final String INVALID_INPUT = "Invalid inputs provided to getIdToken function"; + static final String INVALID_DATA_PARTITION = "Invalid data partition id"; + + private static ConcurrentHashMap<String, JwtValidity> jwtCache = new ConcurrentHashMap<>(); + private Iam iam; + + public ServiceAccountJwtGcpClientImpl(AppProperties config) { + if (config == null) { + throw new IllegalArgumentException("AppProperties is null when initializing jwt client."); + } else { + this.config = config; + } + } + + public String getIdToken(String dataPartitionId) { + String googleAudience = this.config.getGoogleAudiences(); + String hostName = this.config.getRegisterAPI(); + if (Strings.isNullOrEmpty(dataPartitionId) || Strings.isNullOrEmpty(googleAudience) + || Strings.isNullOrEmpty(hostName)) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, + "data partition id, audiences or hostname are null", INVALID_INPUT); + } + try { + // Check if there is already a valid jwt + String key = dataPartitionId + googleAudience + hostName; + String jwt = checkAndGetJwtIfValid(key); + if (!Strings.isNullOrEmpty(jwt)) { + return jwt; + } + + TenantInfo tenantInfo = new TenantFactory().getTenantInfo(dataPartitionId); + if (tenantInfo == null) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, "data partition id is invalid", + INVALID_DATA_PARTITION); + } + long currentTime = System.currentTimeMillis() / 1000; + long expiryTime = currentTime + 3600; + + // get signed JWT + Map<String, Object> signJwtPayload = this.getJwtCreationPayload(tenantInfo, googleAudience, + currentTime, expiryTime); + + SignJwtRequest signJwtRequest = new SignJwtRequest(); + signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload)); + + String serviceAccountName = String.format(SERVICE_ACCOUNT_NAME_FORMAT, + tenantInfo.getProjectId(), + tenantInfo.getServiceAccount()); + + Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam(hostName).projects() + .serviceAccounts() + .signJwt(serviceAccountName, signJwtRequest); + SignJwtResponse signJwtResponse = signJwt.execute(); + String signedJwt = signJwtResponse.getSignedJwt(); + + // get id token + List<NameValuePair> postParameters = new ArrayList<>(); + postParameters.add( + new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); + postParameters.add(new BasicNameValuePair("assertion", signedJwt)); + + HttpPost post = new HttpPost(JWT_AUDIENCE); + post.setHeader(HttpHeaders.CONTENT_TYPE, + ContentType.APPLICATION_FORM_URLENCODED.getMimeType()); + post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); + + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + CloseableHttpResponse httpResponse = httpClient.execute(post); + + JsonObject jsonContent = new JsonParser().parse( + EntityUtils.toString(httpResponse.getEntity())) + .getAsJsonObject(); + + if (!jsonContent.has("id_token")) { + throw new AppException(HttpStatus.SC_UNAUTHORIZED, + "User is not authorized to perform this operation.", + "Unauthorized to generate token"); + } + + String token = "Bearer " + jsonContent.get("id_token").getAsString(); + jwtCache.put(key, new JwtValidity(token, expiryTime)); + return token; + } + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, + "Error happens when generating sauth token", "Error generating token", e); + } + + } + + Iam getIam(String hostName) throws GeneralSecurityException, IOException { + if (this.iam == null) { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + + GoogleCredentials credential = GoogleCredentials.getApplicationDefault(); + if (credential.createScopedRequired()) { + List<String> scopes = new ArrayList<>(); + scopes.add(IamScopes.CLOUD_PLATFORM); + credential = credential.createScoped(scopes); + } + + this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, + new HttpCredentialsAdapter(credential)) + .setApplicationName(hostName).build(); + } + + return this.iam; + } + + // THIS METHOD IS ONLY TO ENABLE UNIT TESTING + boolean reduceTenantExpiry(String dataPartitionId, String googleAudience, String hostName, + long keepDifference) { + JwtValidity jwtValidity = jwtCache.get(dataPartitionId + googleAudience + hostName); + if (jwtValidity == null) { + return false; + } + + long currentTime = System.currentTimeMillis() / 1000; + jwtValidity.expiryTime = currentTime + keepDifference; + return true; + } + + // THIS METHOD IS ONLY TO ENABLE UNIT TESTING + void clearCache() { + jwtCache.clear(); + } + + private String checkAndGetJwtIfValid(String key) { + JwtValidity jwtValidity = jwtCache.get(key); + if (jwtValidity == null) { + return null; + } + + // get current time + long currentTime = System.currentTimeMillis() / 1000; + + // If exipring in less than 5 minutes then need to renew the token + if (jwtValidity.expiryTime - 300 < currentTime) { + jwtCache.remove(key); + return null; + } + + return jwtValidity.token; + } + + + private Map<String, Object> getJwtCreationPayload(TenantInfo tenantInfo, String googleAudience, + long currentTime, long expiryTime) { + if (googleAudience.contains(",")) { + googleAudience = googleAudience.split(",")[0]; + } + Map<String, Object> payload = new HashMap<>(); + payload.put("target_audience", googleAudience); + payload.put("aud", JWT_AUDIENCE); + payload.put("exp", expiryTime); + payload.put("iat", currentTime); + payload.put("iss", tenantInfo.getServiceAccount()); + return payload; + } +} + + diff --git a/provider/notification-reference/src/main/resources/application.properties b/provider/notification-reference/src/main/resources/application.properties new file mode 100644 index 0000000000000000000000000000000000000000..f16a36215afb5b0e96810040ffcfe01e8eee60ca --- /dev/null +++ b/provider/notification-reference/src/main/resources/application.properties @@ -0,0 +1,33 @@ +# +# Copyright 2017-2020, Schlumberger +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +LOG_PREFIX=notification +logging.level.org.springframework.web=${LOG_LEVEL:DEBUG} +server.servlet.contextPath=/api/notification/v1 +app.expireTime=300 +app.maxCacheSize=10 +server.error.whitelabel.enabled=false + +authorize-api=${APP_ENTITLEMENTS} +register-api=${APP_REGISTER} +project-api=${APP_PROJECT} +expire-time=${APP_EXPIRE_TIME:300} +max-cache-size=${APP_MAX_CACHE_SIZE:10} + + +GOOGLE_AUDIENCES=${APP_AUDIENCES} +google-audiences=${APP_AUDIENCES} +partition-api=${PARTITION_API:http://localhost:8081/api/partition/v1} \ No newline at end of file diff --git a/provider/notification-reference/src/main/resources/logback.xml b/provider/notification-reference/src/main/resources/logback.xml new file mode 100644 index 0000000000000000000000000000000000000000..cbd32dfed70fc29a19a7876f0bb7deacc8daf3cb --- /dev/null +++ b/provider/notification-reference/src/main/resources/logback.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + <include resource="org/springframework/boot/logging/logback/defaults.xml"/> + <property resource="application.properties"/> + <logger name="org.opengroup.osdu" level="${LOG_LEVEL}"/> + <springProfile name="local"> + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) + %highlight(%msg) %n + </pattern> + <charset>utf8</charset> + </encoder> + </appender> + <root level="info"> + <appender-ref ref="CONSOLE"/> + </root> + </springProfile> + + <springProfile name="!local"> + <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder"> + <layout class="ch.qos.logback.contrib.json.classic.JsonLayout"> + <timestampFormat>yyyy-MM-dd HH:mm:ss.SSS</timestampFormat> + <timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId> + <appendLineSeparator>true</appendLineSeparator> + + <jsonFormatter class="org.opengroup.osdu.core.gcp.logging.formatter.GoogleJsonFormatter"> + <prettyPrint>false</prettyPrint> + </jsonFormatter> + </layout> + </encoder> + </appender> + --> + + <root level="info"> + <appender-ref ref="stdout"/> + </root> + </springProfile> + +</configuration> \ No newline at end of file diff --git a/provider/notification-reference/src/main/test/java/org/opengroup/osdu/notification/util/GoogleServiceAccountValidatorGeneratorTest.java b/provider/notification-reference/src/main/test/java/org/opengroup/osdu/notification/util/GoogleServiceAccountValidatorGeneratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..73f2011d0c27363e4e0da89635b9f487e277e98d --- /dev/null +++ b/provider/notification-reference/src/main/test/java/org/opengroup/osdu/notification/util/GoogleServiceAccountValidatorGeneratorTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.util; + +import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; +import java.util.Collection; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.opengroup.osdu.notification.provider.reference.util.GoogleServiceAccountValidatorGenerator; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +public class GoogleServiceAccountValidatorGeneratorTest { + + private final NetHttpTransport netHttpTransport = new NetHttpTransport(); + private final JacksonFactory jacksonFactory = new JacksonFactory(); + private static final String AUDIENCE_1 = "aud1"; + private static final String AUDIENCE_2 = "aud2"; + + @InjectMocks + private GoogleServiceAccountValidatorGenerator sut; + + @Test + public void should_returnVerifierWithoutAudiences_when_noAudiencesProvided() { + GoogleIdTokenVerifier verifier = this.sut.getVerifier(netHttpTransport, jacksonFactory); + Assert.assertNull(verifier.getAudience()); + } + + @Test + public void should_returnVerifierWithAudiences_when_AudiencesProvided() { + GoogleIdTokenVerifier verifier = this.sut.getVerifier(netHttpTransport, jacksonFactory, + AUDIENCE_1, AUDIENCE_2); + Collection<String> audiences = verifier.getAudience(); + Assert.assertTrue(audiences.contains(AUDIENCE_1)); + Assert.assertTrue(audiences.contains(AUDIENCE_2)); + } +} diff --git a/provider/notification-reference/src/main/test/java/org/opengroup/osdu/notification/util/GoogleServiceAccountValidatorImplTests.java b/provider/notification-reference/src/main/test/java/org/opengroup/osdu/notification/util/GoogleServiceAccountValidatorImplTests.java new file mode 100644 index 0000000000000000000000000000000000000000..22f9497bfcfbdcbc5d92b5801d193f661f1d0557 --- /dev/null +++ b/provider/notification-reference/src/main/test/java/org/opengroup/osdu/notification/util/GoogleServiceAccountValidatorImplTests.java @@ -0,0 +1,104 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.notification.util; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken; +import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier; +import java.io.IOException; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.notification.provider.reference.util.GoogleServiceAccountValidatorGenerator; +import org.opengroup.osdu.notification.provider.reference.util.GoogleServiceAccountValidatorImpl; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +public class GoogleServiceAccountValidatorImplTests { + + private static final String TEST_JWT = "testjwt"; + private static final String TEST_USER_IDENTITY = "testidentity"; + + @Mock + private JaxRsDpsLog log; + @Mock + private GoogleServiceAccountValidatorGenerator verifierGenerator; + @Mock + private GoogleIdTokenVerifier verifier; + @Mock + private GoogleIdToken idToken; + @Mock + private GoogleIdToken.Payload payload; + @InjectMocks + private GoogleServiceAccountValidatorImpl sut; + + @Test + public void should_returnTrue_when_tokenValidAndUserIdentityCorrect() throws Exception { + when(this.verifierGenerator.getVerifier(any(), any())).thenReturn(this.verifier); + when(this.verifier.verify(TEST_JWT)).thenReturn(this.idToken); + when(this.idToken.getPayload()).thenReturn(this.payload); + when(this.payload.getEmail()).thenReturn(TEST_USER_IDENTITY); + when(this.payload.getEmailVerified()).thenReturn(Boolean.TRUE); + Assert.assertTrue(this.sut.isValidServiceAccount(TEST_JWT, TEST_USER_IDENTITY)); + } + + @Test + public void should_returnFalse_when_tokenInvalid() throws Exception { + when(this.verifierGenerator.getVerifier(any(), any())).thenReturn(this.verifier); + when(this.verifier.verify(TEST_JWT)).thenReturn(null); + Assert.assertFalse(this.sut.isValidServiceAccount(TEST_JWT, TEST_USER_IDENTITY)); + } + + @Test + public void should_returnFalse_when_tokenValidAndUserIdentityIncorrect() throws Exception { + when(this.verifierGenerator.getVerifier(any(), any())).thenReturn(this.verifier); + when(this.verifier.verify(TEST_JWT)).thenReturn(this.idToken); + when(this.idToken.getPayload()).thenReturn(this.payload); + when(this.payload.getEmail()).thenReturn("wrongIdentity"); + when(this.payload.getEmailVerified()).thenReturn(Boolean.TRUE); + Assert.assertFalse(this.sut.isValidServiceAccount(TEST_JWT, TEST_USER_IDENTITY)); + } + + @Test + public void should_returnFalse_when_tokenValidAndUserIdentityCorrect_butEmailNotVerified() + throws Exception { + when(this.verifierGenerator.getVerifier(any(), any())).thenReturn(this.verifier); + when(this.verifier.verify(TEST_JWT)).thenReturn(this.idToken); + when(this.idToken.getPayload()).thenReturn(this.payload); + when(this.payload.getEmail()).thenReturn(TEST_USER_IDENTITY); + when(this.payload.getEmailVerified()).thenReturn(Boolean.FALSE); + Assert.assertFalse(this.sut.isValidServiceAccount(TEST_JWT, TEST_USER_IDENTITY)); + } + + @Test + public void should_logExceptionAndReturnFalse_when_tokenValidationThrowsException() + throws Exception { + when(this.verifierGenerator.getVerifier(any(), any())).thenReturn(this.verifier); + IOException e = new IOException("invalid token"); + when(this.verifier.verify(TEST_JWT)).thenThrow(e); + Assert.assertFalse(this.sut.isValidServiceAccount(TEST_JWT, TEST_USER_IDENTITY)); + verify(this.log, times(1)).error("Error when validating google id token", e); + } +} diff --git a/testing/notification-test-gcp/src/test/resources/logback-test.xml b/testing/notification-test-gcp/src/test/resources/logback-test.xml new file mode 100644 index 0000000000000000000000000000000000000000..22c6175d45f368b69c4006deb9ebf3ff5084579a --- /dev/null +++ b/testing/notification-test-gcp/src/test/resources/logback-test.xml @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + <include resource="org/springframework/boot/logging/logback/defaults.xml"/> + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n</pattern> + <charset>utf8</charset> + </encoder> + </appender> + <root level="INFO"> + <appender-ref ref="CONSOLE" /> + </root> +</configuration>