From df0d31e2009b6bb273a6ff6832b4e8f45acd4d2a Mon Sep 17 00:00:00 2001
From: Chad Leong <CLeong4@slb.com>
Date: Tue, 19 Mar 2024 17:30:12 +0100
Subject: [PATCH] Documentation update with mkdocs
---
.gitlab-ci.yml | 279 ++++++-----
README.md | 30 +-
docs/{tutorial => docs}/DataNotification.md | 485 +++++++++++---------
docs/mkdocs.yml | 38 ++
publish.yml | 30 ++
5 files changed, 480 insertions(+), 382 deletions(-)
rename docs/{tutorial => docs}/DataNotification.md (73%)
create mode 100644 docs/mkdocs.yml
create mode 100644 publish.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ab8dfe05c..0b07a4c28 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,163 +1,162 @@
variables:
- AWS_BUILD_SUBDIR: provider/notification-aws/build-aws
- AWS_TEST_SUBDIR: testing/notification-test-aws
- AWS_CHART_SUBDIR: devops/aws/chart
- AWS_SERVICE: notification
- AWS_SERVICE_GATEWAY: osdu-gateway
- AWS_ENVIRONMENT: dev
- AWS_DEPLOY_TARGET: TF
- AWS_EKS_DEPLOYMENT_NAME: os-notification
-
- AZURE_SERVICE: notification
- AZURE_BUILD_SUBDIR: provider/notification-azure
- AZURE_TEST_SUBDIR: testing/notification-test-azure
- AZURE_DEPLOYMENTS_SUBDIR: deployments/scripts/azure
- AZURE_COVERAGE_THRESHOLD: 80
-
- IBM_BUILD_SUBDIR: provider/notification-ibm
- IBM_INT_TEST_SUBDIR: testing/notification-test-ibm
- IBM_HELM_CONFIG_PATH: devops/ibm/ibm-notification-config
- IBM_HELM_DEPLOY_PATH: devops/ibm/ibm-notification-deploy
+ AWS_BUILD_SUBDIR: provider/notification-aws/build-aws
+ AWS_TEST_SUBDIR: testing/notification-test-aws
+ AWS_CHART_SUBDIR: devops/aws/chart
+ AWS_SERVICE: notification
+ AWS_SERVICE_GATEWAY: osdu-gateway
+ AWS_ENVIRONMENT: dev
+ AWS_DEPLOY_TARGET: TF
+ AWS_EKS_DEPLOYMENT_NAME: os-notification
+
+ AZURE_SERVICE: notification
+ AZURE_BUILD_SUBDIR: provider/notification-azure
+ AZURE_TEST_SUBDIR: testing/notification-test-azure
+ AZURE_DEPLOYMENTS_SUBDIR: deployments/scripts/azure
+ AZURE_COVERAGE_THRESHOLD: 80
+
+ IBM_BUILD_SUBDIR: provider/notification-ibm
+ IBM_INT_TEST_SUBDIR: testing/notification-test-ibm
+ IBM_HELM_CONFIG_PATH: devops/ibm/ibm-notification-config
+ IBM_HELM_DEPLOY_PATH: devops/ibm/ibm-notification-deploy
include:
- - project: "osdu/platform/ci-cd-pipelines"
- file: "standard-setup.yml"
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "standard-setup.yml"
- - project: "osdu/platform/ci-cd-pipelines"
- file: "build/maven.yml"
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "build/maven.yml"
- - project: "osdu/platform/ci-cd-pipelines"
- file: "scanners/fossa-maven.yml"
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "scanners/fossa-maven.yml"
- - project: "osdu/platform/ci-cd-pipelines"
- file: "scanners/gitlab-ultimate.yml"
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "scanners/gitlab-ultimate.yml"
- - project: "osdu/platform/ci-cd-pipelines"
- file: "cloud-providers/azure.yml"
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "cloud-providers/azure.yml"
- - project: "osdu/platform/ci-cd-pipelines"
- file: "cloud-providers/gc-global.yml"
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "cloud-providers/gc-global.yml"
- - project: "osdu/platform/ci-cd-pipelines"
- file: "cloud-providers/ibm.yml"
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "cloud-providers/ibm.yml"
- # aws
- - local: "devops/aws/pipeline/override-stages.yml"
+ # aws
+ - local: "devops/aws/pipeline/override-stages.yml"
- - local: "devops/gc/pipeline/override-stages.yml"
+ - local: "devops/gc/pipeline/override-stages.yml"
+ - local: "/publish.yml"
.maven:
- image: maven:3.9.3-eclipse-temurin-17
- tags: ['osdu-medium']
- variables:
- MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
- MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
-
- cache:
- paths:
- - $MAVEN_REPO_PATH
-
- artifacts:
- paths:
- - ./**/target/*.jar
- - ./**/maven-*-output.txt
- when: always
- expire_in: 2 days
-
- before_script:
- - | # Check for the presence of a maven wrapper script
- apt-get update && apt-get install -y python3
- if [ -e "$CI_PROJECT_DIR/mvnw" ]; then
- export MAVEN_BINARY="$CI_PROJECT_DIR/mvnw"
- unset MAVEN_CONFIG
- mkdir -pv .mvn/wrapper
- else
- export MAVEN_BINARY="mvn"
- fi
- - export MAVEN="$MAVEN_BINARY $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH"
- - echo $MAVEN_BINARY
- - echo $MAVEN_CLI_OPTS
- - echo $MAVEN_REPO_PATH
- - echo $MAVEN
- - mkdir -pv $MAVEN_REPO_PATH
- - export ALL_MAVEN_BUILD_DIRS_Z=$(mktemp)
- - python3 --version
-
- # This scans for all pom.xml files that aren't listed as a child pom (as
- # determine by the presence of a <parent> tag).
- - |
- (python3 | sort -z > $ALL_MAVEN_BUILD_DIRS_Z) <<EOF
- from xml.dom.minidom import parse
- import os
- import sys
-
- allPomFiles = set()
- submoduleFiles = set()
-
- for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
- if 'pom.xml' in filenames:
- pomFile = os.path.join(root, 'pom.xml')
- allPomFiles.add(pomFile)
-
- dom = parse(pomFile)
- for modules in dom.getElementsByTagName('modules'):
- for module in modules.getElementsByTagName('module'):
- submoduleFiles.add(os.path.join(root, module.firstChild.nodeValue, "pom.xml"))
-
- for pomFile in allPomFiles:
- if pomFile not in submoduleFiles:
- dir = os.path.dirname(pomFile)
- sys.stdout.write(dir + '\0')
- EOF
- - xargs -0rn 1 echo < $ALL_MAVEN_BUILD_DIRS_Z
-
- # This creates a script for running maven, capturing output to a file.
- # - First argument is the directory to build from
- # - Second argument is the name to use for the logfile
- # - The remaining arguments are the maven command to run
- - export MAVEN_BUILD=$(tempfile -m 755 -p build -s .sh)
- - echo "#!/bin/bash" > $MAVEN_BUILD
- - |
- cat >> $MAVEN_BUILD <<EOF
- id=maven_build_\$(echo "\$@" | sha1sum | head -c 7)
- cd "\$1"; shift
- outputFile="\$1"; shift
-
- echo "********************************************************************************"
- echo -e "\e[0Ksection_start:\$(date +%s):\${id}\r\e[0K\e[1m\$(pwd)\e[0m"
- echo -e "\e[32;1m\$MAVEN" "\$@" "\e[0m"
-
- if \$MAVEN "\$@" > "\$outputFile" 2>&1; then
- tail -n 25 "\$outputFile";
- echo -e "\e[36m**** Showing the last 25 lines of the passed build (above). See Job artifacts for full log details *****\e[0m";
- echo -e "\e[0Ksection_end:\$(date +%s):\${id}\r\e[0K"
- else
- tail -n 200 "\$outputFile";
- echo -e "\e[31m**** Showing the last 200 lines of the failed build (above). See Job artifacts for full log details *****\e[0m";
- echo -e "\e[0Ksection_end:\$(date +%s):\${id}\r\e[0K"
- exit 255;
- fi
- EOF
-
- # This creates a script for applying the $MAVEN_BUILD script to all build directories
- - export MAVEN_FOREACH=$(tempfile -m 755 -p fr-ec -s .sh)
- - echo "#!/bin/sh" > $MAVEN_FOREACH
- - echo 'xargs -0rI {} $MAVEN_BUILD {} "$@" < $ALL_MAVEN_BUILD_DIRS_Z' >> $MAVEN_FOREACH
+ image: maven:3.9.3-eclipse-temurin-17
+ tags: ["osdu-medium"]
+ variables:
+ MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
+ MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
+
+ cache:
+ paths:
+ - $MAVEN_REPO_PATH
+
+ artifacts:
+ paths:
+ - ./**/target/*.jar
+ - ./**/maven-*-output.txt
+ when: always
+ expire_in: 2 days
+
+ before_script:
+ - | # Check for the presence of a maven wrapper script
+ apt-get update && apt-get install -y python3
+ if [ -e "$CI_PROJECT_DIR/mvnw" ]; then
+ export MAVEN_BINARY="$CI_PROJECT_DIR/mvnw"
+ unset MAVEN_CONFIG
+ mkdir -pv .mvn/wrapper
+ else
+ export MAVEN_BINARY="mvn"
+ fi
+ - export MAVEN="$MAVEN_BINARY $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH"
+ - echo $MAVEN_BINARY
+ - echo $MAVEN_CLI_OPTS
+ - echo $MAVEN_REPO_PATH
+ - echo $MAVEN
+ - mkdir -pv $MAVEN_REPO_PATH
+ - export ALL_MAVEN_BUILD_DIRS_Z=$(mktemp)
+ - python3 --version
+
+ # This scans for all pom.xml files that aren't listed as a child pom (as
+ # determine by the presence of a <parent> tag).
+ - |
+ (python3 | sort -z > $ALL_MAVEN_BUILD_DIRS_Z) <<EOF
+ from xml.dom.minidom import parse
+ import os
+ import sys
+
+ allPomFiles = set()
+ submoduleFiles = set()
+
+ for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
+ if 'pom.xml' in filenames:
+ pomFile = os.path.join(root, 'pom.xml')
+ allPomFiles.add(pomFile)
+
+ dom = parse(pomFile)
+ for modules in dom.getElementsByTagName('modules'):
+ for module in modules.getElementsByTagName('module'):
+ submoduleFiles.add(os.path.join(root, module.firstChild.nodeValue, "pom.xml"))
+
+ for pomFile in allPomFiles:
+ if pomFile not in submoduleFiles:
+ dir = os.path.dirname(pomFile)
+ sys.stdout.write(dir + '\0')
+ EOF
+ - xargs -0rn 1 echo < $ALL_MAVEN_BUILD_DIRS_Z
+ # This creates a script for running maven, capturing output to a file.
+ # - First argument is the directory to build from
+ # - Second argument is the name to use for the logfile
+ # - The remaining arguments are the maven command to run
+ - export MAVEN_BUILD=$(tempfile -m 755 -p build -s .sh)
+ - echo "#!/bin/bash" > $MAVEN_BUILD
+ - |
+ cat >> $MAVEN_BUILD <<EOF
+ id=maven_build_\$(echo "\$@" | sha1sum | head -c 7)
+ cd "\$1"; shift
+ outputFile="\$1"; shift
+
+ echo "********************************************************************************"
+ echo -e "\e[0Ksection_start:\$(date +%s):\${id}\r\e[0K\e[1m\$(pwd)\e[0m"
+ echo -e "\e[32;1m\$MAVEN" "\$@" "\e[0m"
+
+ if \$MAVEN "\$@" > "\$outputFile" 2>&1; then
+ tail -n 25 "\$outputFile";
+ echo -e "\e[36m**** Showing the last 25 lines of the passed build (above). See Job artifacts for full log details *****\e[0m";
+ echo -e "\e[0Ksection_end:\$(date +%s):\${id}\r\e[0K"
+ else
+ tail -n 200 "\$outputFile";
+ echo -e "\e[31m**** Showing the last 200 lines of the failed build (above). See Job artifacts for full log details *****\e[0m";
+ echo -e "\e[0Ksection_end:\$(date +%s):\${id}\r\e[0K"
+ exit 255;
+ fi
+ EOF
+
+ # This creates a script for applying the $MAVEN_BUILD script to all build directories
+ - export MAVEN_FOREACH=$(tempfile -m 755 -p fr-ec -s .sh)
+ - echo "#!/bin/sh" > $MAVEN_FOREACH
+ - echo 'xargs -0rI {} $MAVEN_BUILD {} "$@" < $ALL_MAVEN_BUILD_DIRS_Z' >> $MAVEN_FOREACH
gemnasium-maven-dependency_scanning:
- variables:
- DS_JAVA_VERSION: 17
+ variables:
+ DS_JAVA_VERSION: 17
azure_containerize:
- variables:
- AZURE_CONTAINERIZE_REPOSITORY: "-b jdk-17 https://gitlab-ci-token:${CI_JOB_TOKEN}@community.opengroup.org/osdu/platform/deployment-and-operations/base-containers-azure/service-base-image"
+ variables:
+ AZURE_CONTAINERIZE_REPOSITORY: "-b jdk-17 https://gitlab-ci-token:${CI_JOB_TOKEN}@community.opengroup.org/osdu/platform/deployment-and-operations/base-containers-azure/service-base-image"
azure_test:
- image: community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/azure-maven17:v0.0.1
-
+ image: community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/azure-maven17:v0.0.1
fossa-analyze:
- image: $CI_REGISTRY/divido/fossa-with-cache:v0.9-jdk17
+ image: $CI_REGISTRY/divido/fossa-with-cache:v0.9-jdk17
fossa-check-notice:
- image: $CI_REGISTRY/divido/fossa-with-cache:v0.9-jdk17
+ image: $CI_REGISTRY/divido/fossa-with-cache:v0.9-jdk17
diff --git a/README.md b/README.md
index d55e3c652..4a3c1cfdd 100644
--- a/README.md
+++ b/README.md
@@ -12,20 +12,24 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-# Notification service
+# Register service
The Notification service, with Register service, allow interested consumers to subscribe to data and metadata changes using a publisher/subscriber pattern
+## Documentation
+
+Official documentation home for [Notification Service](https://osdu.pages.opengroup.org/platform/system/notification/)
+
## Dependencies needed to run the code locally
-* JDK11
-* Maven
-* Azure Devops access to slb-des-ext-collaboration organization. You need to generate a PAT that can access dependencies held in the Azure artifacts
-* Working Register service endpoint
+- JDK11
+- Maven
+- Azure Devops access to slb-des-ext-collaboration organization. You need to generate a PAT that can access dependencies held in the Azure artifacts
+- Working Register service endpoint
## Build and run service locally
-* Build the service and run the unit tests. This step also generates the executable jar file. Run this command from the root folder, where parent pom.xml resides
+- Build the service and run the unit tests. This step also generates the executable jar file. Run this command from the root folder, where parent pom.xml resides
**Note**: At runtime, you must set a Spring profile so the right runtime configuration gets picked up for your app. Since this is a Spring Boot application, runtime configuration is defined in one of the application properties files under `<provider>->src->main->resources`. You can set the right Spring Boot runtime profile with `-Dspring.profiles.active=test`. Please read [this](https://stackoverflow.com/questions/39738901/how-do-i-activate-a-spring-boot-profile-when-running-from-intellij) for more info.
@@ -33,23 +37,23 @@ The Notification service, with Register service, allow interested consumers to s
mvn clean install
```
-* To run the service locally, navigate to `provider/notification-<your_provider>/target` directory and execute the spring-boot jar
+- To run the service locally, navigate to `provider/notification-<your_provider>/target` directory and execute the spring-boot jar
```
java -jar notification-gc-1.0.0-spring-boot.jar
```
-* To debug service locally, create a remote debug configuration first and then use following command to start service in debug mode
+- To debug service locally, create a remote debug configuration first and then use following command to start service in debug mode
```
java -Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=n -jar notification-gc-1.0.0-spring-boot.jar
```
-**Note:** The port number "8000" in the command should match with the remote debug config settings.
+**Note:** The port number "8000" in the command should match with the remote debug config settings.
## Access the service
-The port and path for the service endpoint can be configured in ```application.properties``` as following. If not specified, then the web container (ex. Tomcat) default is used:
+The port and path for the service endpoint can be configured in `application.properties` as following. If not specified, then the web container (ex. Tomcat) default is used:
```bash
server.servlet.contextPath=/
@@ -57,6 +61,7 @@ server.port=8080
```
## Open API 3.0 - Swagger
+
- Swagger UI : https://host/context-path/swagger (will redirect to https://host/context-path/swagger-ui/index.html)
- api-docs (JSON) : https://host/context-path/api-docs
- api-docs (YAML) : https://host/context-path/api-docs.yaml
@@ -64,12 +69,11 @@ server.port=8080
All the Swagger and OpenAPI related common properties are managed here [swagger.properties](./notification-core/src/main/resources/swagger.properties)
#### Server Url(full path vs relative path) configuration
+
- `api.server.fullUrl.enabled=true` It will generate full server url in the OpenAPI swagger
- `api.server.fullUrl.enabled=false` It will generate only the contextPath only
- default value is false (Currently only in Azure it is enabled)
-[Reference]:(https://springdoc.org/faq.html#_how_is_server_url_generated)
-
-# new update
+ [Reference]:(https://springdoc.org/faq.html#_how_is_server_url_generated)
## AWS
diff --git a/docs/tutorial/DataNotification.md b/docs/docs/DataNotification.md
similarity index 73%
rename from docs/tutorial/DataNotification.md
rename to docs/docs/DataNotification.md
index 63247ee61..ea4f8d6f1 100644
--- a/docs/tutorial/DataNotification.md
+++ b/docs/docs/DataNotification.md
@@ -1,42 +1,55 @@
## Data Notification
## Table of Contents <a name="TOC"></a>
-* [Introduction](#introduction)
-* [Workflow steps](#workflow-steps)
- * [Get topics available to subscribe](#list-topics)
- * [Subscribing to a topic](#create-subscription)
- * [Hash-based Message Authentication Code (HMAC) Subscription](#create-hmac-subscription)
- * [Google Service Account (GSA) Subscription](#create-gsa-subscription)
- * [Get a Subscription by ID](#get-subscription)
- * [Delete a Subscription by ID](#delete-subscription)
- * [Handle notifications](#process-messages)
- * [Update secret for a Subscription](#update-subscription)
-* [Version info endpoint](#version-info-endpoint)
-* [Current Limitations](#limitation)
-* [Support for Collaboration Context](#collaboration-support)
+
+1. [Data Notification](#data-notification)
+2. [Table of Contents ](#table-of-contents-)
+3. [Introduction ](#introduction-)
+4. [Steps ](#steps-)
+ 1. [Get topics available to subscribe](#get-topics-available-to-subscribe)
+5. [Subscribing to a topic ](#subscribing-to-a-topic-)
+ 1. [Hash-based Message Authentication Code (HMAC) Subscription using a "secret" string ](#hash-based-message-authentication-code-hmac-subscription-using-a-secret-string-)
+ 2. [Google Service Account (GSA) Subscription using audience \& service account key ](#google-service-account-gsa-subscription-using-audience--service-account-key-)
+6. [Get a Subscription by ID ](#get-a-subscription-by-id-)
+7. [Delete a Subscription by ID ](#delete-a-subscription-by-id-)
+8. [Handling notifications ](#handling-notifications-)
+ 1. [Message Contents](#message-contents)
+ 2. [Notification handler endpoint - HMAC Secret type](#notification-handler-endpoint---hmac-secret-type)
+ 3. [Notification handler Endpoint - GSA Secret type](#notification-handler-endpoint---gsa-secret-type)
+ 4. [Responding to Notifications](#responding-to-notifications)
+9. [Update secret for a Subscription ](#update-secret-for-a-subscription-)
+10. [Version info endpoint](#version-info-endpoint)
+ 1. [Example response:](#example-response)
+11. [Current Limitations ](#current-limitations-)
+12. [Support for Collaboration Context ](#support-for-collaboration-context-)
+ 1. [Example of a message when the x-collaboration header is provided:](#example-of-a-message-when-the-x-collaboration-header-is-provided)
## Introduction <a name="introduction"></a>
+
The OSDU notification system allows for interested consumers to subscribe to data and metadata changes using a publish/subscriber pattern.
A typical workflow using notification is:
-* Consumer finds a "topic" that they want to keep up to date with any changes in OSDU.
-* Consumer creates Push endpoint, that is used to receive notifications on the interested topic.
-* Consumer creates a Subscription in OSDU Notification System, and proves the ownership of the Push endpoint.
-* Consumer starts to receive notifications for that topic and processes the message to synchronize with the OSDU state.
-* Consumer periodically rotates the "secret" used for subscription.
+- Consumer finds a "topic" that they want to keep up to date with any changes in OSDU.
+- Consumer creates Push endpoint, that is used to receive notifications on the interested topic.
+- Consumer creates a Subscription in OSDU Notification System, and proves the ownership of the Push endpoint.
+- Consumer starts to receive notifications for that topic and processes the message to synchronize with the OSDU state.
+- Consumer periodically rotates the "secret" used for subscription.
The topics below describe these steps/APIs in detail that allow consumers to create such integrated workflows using OSDU Notification.
[Back to Table of Contents](#TOC)
## Steps <a name="workflow-steps"></a>
+
### Get topics available to subscribe<a name="list-topics"></a>
+
Consumer uses Data notification "topics" API to view the list of supported notification topics and corresponding sample messages.
-```
+```json
GET api/register/v1/topics
```
+
<details><summary>curl</summary>
```
@@ -46,9 +59,11 @@ curl --request GET \
--header 'Content-Type: application/json' \
--header 'data-partition-id: common' \
```
+
</details>
A sample output is shown below. Please note the "name" of the topic. This is required to create a Subscription for a topic you are interested in.
+
<details><summary>Sample response</summary>
```
@@ -111,7 +126,7 @@ A sample output is shown below. Please note the "name" of the topic. This is req
"recordIdVersion": "1610537924768407",
"stage": "STORAGE_SYNC",
"status": "FAILED",
- "message": "acl is not valid",
+ "message": "acl is not valid",
"errorCode ": 400,
"timestamp ": 1622118996000
}
@@ -137,14 +152,16 @@ A sample output is shown below. Please note the "name" of the topic. This is req
[Back to Table of Contents](#TOC)
## Subscribing to a topic <a name="create-subscription"></a>
+
The consumer uses the data notification Subscription API to create a Subscription for the topic of interest. A subscription id is returned in the response that can be used to get the subscription details again or delete the subscription later.
+
```
POST /api/register/v1/subscription/
```
-To subscribe to a topic, consumers must have a "https" endpoint supporting both "GET" and "POST" methods. "GET" is used as a challenge endpoint when creating (or updating) a subscription to validate that consumer owns this endpoint. "POST" is used for pushing the notifications to consumers.
+To subscribe to a topic, consumers must have a "https" endpoint supporting both "GET" and "POST" methods. "GET" is used as a challenge endpoint when creating (or updating) a subscription to validate that consumer owns this endpoint. "POST" is used for pushing the notifications to consumers.
-The challenge is performed only when creating a subscription or when [Updating secret for a Subscription](#update-subscription)
+The challenge is performed only when creating a subscription or when [Updating secret for a Subscription](#update-subscription)
Below are the details of the two types of Subscriptions and the challenge process:
@@ -172,11 +189,11 @@ Below are the details of the two types of Subscriptions and the challenge proces
</details>
-Before creating an HMAC Subscription, the consumer needs to make sure that "GET" is supported on the endpoint being registered with OSDU Notification and the endpoint accepts query parameters named "crc" & "hmac". OSDU Notification will send a "GET" request on this endpoint with a random crc, and expects a response hash generated using the crc & the secret value (i.e. "testSecret" in the example above).
+Before creating an HMAC Subscription, the consumer needs to make sure that "GET" is supported on the endpoint being registered with OSDU Notification and the endpoint accepts query parameters named "crc" & "hmac". OSDU Notification will send a "GET" request on this endpoint with a random crc, and expects a response hash generated using the crc & the secret value (i.e. "testSecret" in the example above).
In addition, consumers may also want to validate the hmac field, which is the signature that will be used when a message is pushed to this endpoint. The signature verification must be used in the push endpoint implementation before processing the messages, to ensure that the message is coming from OSDU Notification.
-Note: Secret value may not contain any special characters (only alphanumeric characters) and the number of characters must be even.
+Note: Secret value may not contain any special characters (only alphanumeric characters) and the number of characters must be even.
<details><summary>Sample API definition for setting up the challange end point</summary>
@@ -217,7 +234,7 @@ definitions:
- schema
properties:
responseHash:
- type: string
+ type: string
```
@@ -225,7 +242,7 @@ definitions:
<details><summary>Sample Java code to generate the hmac signature, validate it and send a response with hash</summary>
-```
+```java
...
import javax.crypto.Mac;
@@ -235,13 +252,13 @@ import javax.xml.bind.DatatypeConverter;
@GET
@Path(<DomainEndpoint>)
-public Response challenge(@QueryParam("crc") @NotBlank String crc,
+public Response challenge(@QueryParam("crc") @NotBlank String crc,
@QueryParam("hmac") @NotBlank String hmac) {
// Use the secret you send to the subscriber registration create request
- // Hint: Secret string can be stored as configuration for the service
+ // Hint: Secret string can be stored as configuration for the service
String secret = <getSecretString()>
verifyHmacSignature(hmac, secret);
-
+
String response = getResponseHash( secret + crc);
return Response.status(HttpStatus.SC_OK).entity(Collections.singletonMap("responseHash", response)).build();
}
@@ -328,7 +345,6 @@ private byte[] computeHmacSha256(final byte[] data, final byte[] key) throws Exc
return mac.doFinal(data);
}
-
```
</details>
@@ -337,7 +353,7 @@ private byte[] computeHmacSha256(final byte[] data, final byte[] key) throws Exc
<details><summary>curl</summary>
-```
+```curl
curl --request POST \
--url 'https://register-svc.osdu.com/api/register/v1/subscriber \
-header 'Authorization: Bearer <JWT>' \
@@ -357,15 +373,16 @@ private byte[] computeHmacSha256(final byte[] data, final byte[] key) throws Exc
}
}'
```
+
</details>
-Before creating a GSA Subscription, the consumer needs to make sure that "GET" is supported on the endpoint being registered with OSDU Notification and it accepts a query parameter named "crc". OSDU Notification will send a "GET" request on this endpoint with a random crc, and expects a response hash generated using crc & the private_key_id field from the Service account used for subscription.
+Before creating a GSA Subscription, the consumer needs to make sure that "GET" is supported on the endpoint being registered with OSDU Notification and it accepts a query parameter named "crc". OSDU Notification will send a "GET" request on this endpoint with a random crc, and expects a response hash generated using crc & the private_key_id field from the Service account used for subscription.
In addition, consumers may also want to validate the google id token provided as "authorization" header, which will be generated using the audience & key provided. The google id token must be used in the push endpoint implementation before processing the messages, to ensure that the message is coming from OSDU Notification.
<details><summary>Sample API definition for setting up the challange end point</summary>
-```
+```yaml
...
paths:
...
@@ -397,7 +414,7 @@ definitions:
- schema
properties:
responseHash:
- type: string
+ type: string
```
@@ -405,7 +422,7 @@ definitions:
<details><summary>Sample Java code to validate the google id token and send a response with hash</summary>
-```
+```java
...
@@ -432,7 +449,7 @@ private boolean verifyToken() {
GoogleIdTokenVerifier verifier = new GoogleIdTokenVerifier.Builder(new NetHttpTransport(), JacksonFactory.getDefaultInstance())
.setAudience(Collections.singletonList(<getGoogleAudiences()>))
.build();
- GoogleIdToken idToken = verifier.verify(<getAuthorizationHeader()>);
+ GoogleIdToken idToken = verifier.verify(<getAuthorizationHeader()>);
return idToken != null;
} catch (Exception ex) {
return false;
@@ -454,6 +471,7 @@ private Response getResponse(String crc, String secretString) {
.build();
}
```
+
</details>
[Back to Table of Contents](#TOC)
@@ -465,20 +483,23 @@ Consumers use this API to get the subscription details for the Subscription with
```
GET /api/register/v1/subscription/{id}
```
+
<details><summary>curl</summary>
-```
+```curl
curl --request GET \
--url 'https://register-svc.osdu.com/api/register/v1/subscription/{id} \
--header 'Authorization: Bearer <JWT>' \
--header 'data-partition-id: common'
```
+
</details>
[Back to Table of Contents](#TOC)
## Delete a Subscription by ID <a name="#delete-subscription"></a>
+
Consumers use this API to delete a Subscription with the given subscription id.
```
@@ -487,26 +508,28 @@ DELETE /api/register/v1/subscription/{id}
<details><summary>curl</summary>
-```
+```curl
curl --request DELETE\
--url 'https://register-svc.osdu.com/api/register/v1/subscription/<id>' \
--header 'authorization: Bearer <JWT>' \
--header 'content-type: application/json' \
--header 'data-partition-id: common'
```
+
</details>
[Back to Table of Contents](#TOC)
## Handling notifications <a name="process-messages"></a>
-Consumers will start receiving messages for the topics that they have subscribed for.
+Consumers will start receiving messages for the topics that they have subscribed for.
### Message Contents
+
The sample message for record change notification looks like this:
-```
-[
+```json
+[
{"id":"record_id_1","kind":"kind1","op":"create","recordUpdated":"false"},
{"id":"record_id_1","kind":"kind1","op":"create","recordUpdated":"true"},
{"id":"record_id_2","kind":"kind2","op":"delete","deletionType":"soft"},
@@ -515,12 +538,13 @@ The sample message for record change notification looks like this:
]
```
-Please note that each message can contain a maximum of 50 record updates and can have updates for multiple kinds and operations. If there are more than 50 record updates, subscribers will receive multiple messages.
+Please note that each message can contain a maximum of 50 record updates and can have updates for multiple kinds and operations. If there are more than 50 record updates, subscribers will receive multiple messages.
+
+Possible values of operation types (i.e. "op" field in above example) are as follows:
-Possible values of operation types (i.e. "op" field in above example) are as follows:
- - create
- - delete
- - create_schema
+- create
+- delete
+- create_schema
### Notification handler endpoint - HMAC Secret type
@@ -528,35 +552,31 @@ Endpoints with HMAC Subscriptions must accept a parameter named "hmac", which ha
<details><summary>A simple API definition to receive notifications </summary>
-```
-
-...
- '/consumer':
- post:
- summary: Receive notification
- description: "Receives push notification from OSDU"
- consumes:
- - application/json
- produces:
- - application/json
- parameters:
- - in: query
- name: hmac
- type: string
- required: true
- responses:
- '200':
- description: OK
-...
-
+```yaml
+---
+"/consumer":
+ post:
+ summary: Receive notification
+ description: "Receives push notification from OSDU"
+ consumes:
+ - application/json
+ produces:
+ - application/json
+ parameters:
+ - in: query
+ name: hmac
+ type: string
+ required: true
+ responses:
+ "200":
+ description: OK
```
</details>
-
<details><summary>A simple Java implementation of the endpoint</summary>
-```
+```java
@POST
@Path(<ConsumerEndpoint>)
public Response processMessage(@NotBlank(message = "Request body can not be null") String data,
@@ -568,11 +588,11 @@ public Response processMessage(@NotBlank(message = "Request body can not be null
...
// Get message from body
-
+
// Process message
-
+
...
-
+
return response
}
@@ -586,30 +606,26 @@ Endpoints with GSA Subscriptions must accept a google id token as Authorization
<details><summary>A simple API definition to receive notifications </summary>
-```
-
-...
- '/consumer':
- post:
- summary: Receive notification
- description: "Receives push notification from OSDU"
- consumes:
- - application/json
- produces:
- - application/json
- responses:
- '200':
- description: OK
-...
-
+```yaml
+---
+"/consumer":
+ post:
+ summary: Receive notification
+ description: "Receives push notification from OSDU"
+ consumes:
+ - application/json
+ produces:
+ - application/json
+ responses:
+ "200":
+ description: OK
```
</details>
<details><summary>A simple implementation of the the endpoint will look like this</summary>
-
-```
+```java
@POST
@Path(<ConsumerEndpoint>)
public Response processMessage(Object o) {
@@ -619,23 +635,26 @@ public Response processMessage(Object o) {
...
// Get message from signature
-
+
// Process message
-
+
...
-
+
return response;
}
```
+
</details>
### Responding to Notifications
+
The notification service expects a response with the code in the 200-299 range for successfully acknowledged messages. It expects such a response from the consumer endpoint within 30 seconds. If acknowledgement is not received in this time, the notification service will continue to call the endpoint for 5 days. The frequency of the message slows down if it consistently fails to receive a successful acknowledgement.
[Back to Table of Contents](#TOC)
## Update secret for a Subscription <a name="update-subscription"></a>
+
Consumers might want to regularly update their secret for the Subscriptions to avoid security issues.
This can be done using the OSDU Notification update subscription API. Consumers must update the "GET" endpoint first to point to new secret, as the same verification will be performed again with new secret value.
@@ -648,7 +667,7 @@ PUT /api/register/v1/subscription/{id}/secret
<details><summary>curl</summary>
-```
+```curl
curl --request PUT \
--url 'https://register-svc.osdu.com/api/register/v1/subscription/{id}/secret' \
--header 'authorization: Bearer <JWT>' \
@@ -659,58 +678,66 @@ curl --request PUT \
"value": <newValue>
}'
```
+
</details>
[Back to Table of Contents](#TOC)
## Version info endpoint
+
For deployment available public `/info` endpoint, which provides build and git related information.
+
#### Example response:
+
```json
{
- "groupId": "org.opengroup.osdu",
- "artifactId": "storage-gcp",
- "version": "0.10.0-SNAPSHOT",
- "buildTime": "2021-07-09T14:29:51.584Z",
- "branch": "feature/GONRG-2681_Build_info",
- "commitId": "7777",
- "commitMessage": "Added copyright to version info properties file",
- "connectedOuterServices": [
- {
- "name": "elasticSearch",
- "version":"..."
- },
- {
- "name": "postgresSql",
- "version":"..."
- },
- {
- "name": "redis",
- "version":"..."
- }
- ]
+ "groupId": "org.opengroup.osdu",
+ "artifactId": "storage-gcp",
+ "version": "0.10.0-SNAPSHOT",
+ "buildTime": "2021-07-09T14:29:51.584Z",
+ "branch": "feature/GONRG-2681_Build_info",
+ "commitId": "7777",
+ "commitMessage": "Added copyright to version info properties file",
+ "connectedOuterServices": [
+ {
+ "name": "elasticSearch",
+ "version": "..."
+ },
+ {
+ "name": "postgresSql",
+ "version": "..."
+ },
+ {
+ "name": "redis",
+ "version": "..."
+ }
+ ]
}
```
+
This endpoint takes information from files, generated by `spring-boot-maven-plugin`,
`git-commit-id-plugin` plugins. Need to specify paths for generated files to matching
properties:
+
- `version.info.buildPropertiesPath`
- `version.info.gitPropertiesPath`
[Back to table of contents](#TOC)
## Current Limitations <a name="limitation"></a>
-- There is no filtering applied on messages (such as based on the kind etc.) at the moment in OSDU. All the messages will be pushed to consumers.
+
+- There is no filtering applied on messages (such as based on the kind etc.) at the moment in OSDU. All the messages will be pushed to consumers.
[Back to Table of Contents](#TOC)
## Support for Collaboration Context <a name="collaboration-support"></a>
+
Register service and Notification service are collaboration aware. For now, to enable collaboration context support you have to enable collaboration feature flag in the services. Refer to this [Wiki](https://community.opengroup.org/groups/osdu/platform/system/-/wikis/Feature-Flag) for more details on how to do that.
That means when the collaboration context feature flag is enabled the list of topics returned will have a new topic "recordstopic-v2" which will receive notifications when "x-collaboration" header is provided in the request.
<details><summary>curl for a collaboration context header provided request</summary>
-```
+```curl
curl --request GET \
--url 'https://register-svc.osdu.com/api/register/v1/topics' \
--header 'Authorization: Bearer <JWT>' \
@@ -718,136 +745,136 @@ curl --request GET \
--header 'data-partition-id: common' \
--header 'x-collaboration: id=<collaboration id>,application=<application name>' \
```
+
</details>
A sample output is shown below when the collaboration context feature flag is set to true.
<details><summary>Sample response</summary>
-```
+```json
[
- {
- "name": "recordstopic",
- "description": "This notification is sent whenever a new record or record version is created, updated or deleted in storage. 'previousVersionKind' is noted upon 'kind' update. Record deletion is noted as a soft 'deletionType'. Record purge is noted as a hard 'deletionType'.",
- "state": "ACTIVE",
- "example": [
- {
- "id": "osdu:abc:123",
- "kind": "osdu:petrel:regularheightfieldsurface:1.0.0",
- "op": "create"
- },
- {
- "id": "osdu:abc:345",
- "kind": "osdu:petrel:regularheightfieldsurface:1.0.1",
- "op": "update",
- "previousVersionKind": "osdu:petrel:regularheightfieldsurface:1.0.0"
- },
- {
- "id": "osdu:abc:567",
- "kind": "osdu:petrel:regularheightfieldsurface:1.0.0",
- "op": "delete",
- "deletionType": "soft"
- },
- {
- "id": "osdu:abc:789",
- "kind": "osdu:petrel:regularheightfieldsurface:1.0.0",
- "op": "delete",
- "deletionType": "hard"
- }
- ]
- },
- {
- "name": "schemachangedtopic",
- "description": "This notification is sent whenever a new schema is created or updated via schema-service.",
- "state": "ACTIVE",
- "example": [
- {
- "kind": "osdu:wks:wellbore:1.0.0",
- "op": "update"
- },
- {
- "kind": "osdu:wks:wellbore:2.0.0",
- "op": "create"
- }
- ]
- },
- {
- "name": "statuschangedtopic",
- "description": "Every Service/Stage would publish their respective status changed information in this topic.",
- "state": "ACTIVE",
- "example": [
- {
- "kind": "status",
- "properties": {
- "correlationId": "12345",
- "recordId": "osdu:file:3479d828-a47d-4e13-a1f8-9791a19e1a7e",
- "recordIdVersion": "1610537924768407",
- "stage": "STORAGE_SYNC",
- "status": "FAILED",
- "message": "acl is not valid",
- "errorCode ": 400.0,
- "timestamp ": 1.622118996E12
- }
- },
- {
- "kind": "dataSetDetails",
- "properties": {
- "correlationId": "12345",
- "dataSetId": "12345",
- "dataSetIdVersion": "1",
- "dataSetType": "FILE",
- "recordCount": 10.0,
- "timestamp ": 1.622118996E12
- }
- }
- ]
- },
- {
- "name": "recordstopic-v2",
- "description": "This notification is sent whenever a new record or record version is created, updated or deleted in storage for all collaboration and non-collaboration context changes. The collaboration context will be provided as part of the message properties if exist. 'previousVersionKind' is noted upon 'kind' update. Record deletion is noted as a soft 'deletionType'. Record purge is noted as a hard 'deletionType'.",
- "state": "ACTIVE",
- "example": {
- "message": {
- "data": [
- {
- "id": "osdu:abc:123",
- "kind": "osdu:petrel:regularheightfieldsurface:1.0.0",
- "op": "create"
- }
- ],
- "account-id": "opendes",
- "data-partition-id": "opendes",
- "correlation-id": "4f1982a2-cbdf-438a-b5a1-e0c6239d46fc",
- "x-collaboration": "id=1e1c4e74-3b9b-4b17-a0d5-67766558ec65,application=Test App"
- }
- }
+ {
+ "name": "recordstopic",
+ "description": "This notification is sent whenever a new record or record version is created, updated or deleted in storage. 'previousVersionKind' is noted upon 'kind' update. Record deletion is noted as a soft 'deletionType'. Record purge is noted as a hard 'deletionType'.",
+ "state": "ACTIVE",
+ "example": [
+ {
+ "id": "osdu:abc:123",
+ "kind": "osdu:petrel:regularheightfieldsurface:1.0.0",
+ "op": "create"
+ },
+ {
+ "id": "osdu:abc:345",
+ "kind": "osdu:petrel:regularheightfieldsurface:1.0.1",
+ "op": "update",
+ "previousVersionKind": "osdu:petrel:regularheightfieldsurface:1.0.0"
+ },
+ {
+ "id": "osdu:abc:567",
+ "kind": "osdu:petrel:regularheightfieldsurface:1.0.0",
+ "op": "delete",
+ "deletionType": "soft"
+ },
+ {
+ "id": "osdu:abc:789",
+ "kind": "osdu:petrel:regularheightfieldsurface:1.0.0",
+ "op": "delete",
+ "deletionType": "hard"
+ }
+ ]
+ },
+ {
+ "name": "schemachangedtopic",
+ "description": "This notification is sent whenever a new schema is created or updated via schema-service.",
+ "state": "ACTIVE",
+ "example": [
+ {
+ "kind": "osdu:wks:wellbore:1.0.0",
+ "op": "update"
+ },
+ {
+ "kind": "osdu:wks:wellbore:2.0.0",
+ "op": "create"
+ }
+ ]
+ },
+ {
+ "name": "statuschangedtopic",
+ "description": "Every Service/Stage would publish their respective status changed information in this topic.",
+ "state": "ACTIVE",
+ "example": [
+ {
+ "kind": "status",
+ "properties": {
+ "correlationId": "12345",
+ "recordId": "osdu:file:3479d828-a47d-4e13-a1f8-9791a19e1a7e",
+ "recordIdVersion": "1610537924768407",
+ "stage": "STORAGE_SYNC",
+ "status": "FAILED",
+ "message": "acl is not valid",
+ "errorCode ": 400.0,
+ "timestamp ": 1.622118996e12
+ }
+ },
+ {
+ "kind": "dataSetDetails",
+ "properties": {
+ "correlationId": "12345",
+ "dataSetId": "12345",
+ "dataSetIdVersion": "1",
+ "dataSetType": "FILE",
+ "recordCount": 10.0,
+ "timestamp ": 1.622118996e12
}
+ }
+ ]
+ },
+ {
+ "name": "recordstopic-v2",
+ "description": "This notification is sent whenever a new record or record version is created, updated or deleted in storage for all collaboration and non-collaboration context changes. The collaboration context will be provided as part of the message properties if exist. 'previousVersionKind' is noted upon 'kind' update. Record deletion is noted as a soft 'deletionType'. Record purge is noted as a hard 'deletionType'.",
+ "state": "ACTIVE",
+ "example": {
+ "message": {
+ "data": [
+ {
+ "id": "osdu:abc:123",
+ "kind": "osdu:petrel:regularheightfieldsurface:1.0.0",
+ "op": "create"
+ }
+ ],
+ "account-id": "opendes",
+ "data-partition-id": "opendes",
+ "correlation-id": "4f1982a2-cbdf-438a-b5a1-e0c6239d46fc",
+ "x-collaboration": "id=1e1c4e74-3b9b-4b17-a0d5-67766558ec65,application=Test App"
+ }
+ }
+ }
]
```
</details>
-
When the feature flag is set to true and the consumer provides "x-collaboration" header in the request for creating, updating, and deleting a record. the message sent will contain the collaboration context header as an attribute.
#### Example of a message when the x-collaboration header is provided:
+
```json
{
- "message": {
- "data": [
- {
- "id": "<message-id>",
- "version": "1617915304347525",
- "modifiedBy": "abc@xyz.com",
- "kind": "common:welldb:wellbore:1.0.0",
- "op": "create"
- }
- ],
- "account-id": "opendes",
- "data-partition-id": "opendes",
- "correlation-id": "<corrilation-id>",
- "x-collaboration": "id=<collaboration-id>,application=<app-name>"
+ "message": {
+ "data": [
+ {
+ "id": "<message-id>",
+ "version": "1617915304347525",
+ "modifiedBy": "abc@xyz.com",
+ "kind": "common:welldb:wellbore:1.0.0",
+ "op": "create"
}
+ ],
+ "account-id": "opendes",
+ "data-partition-id": "opendes",
+ "correlation-id": "<corrilation-id>",
+ "x-collaboration": "id=<collaboration-id>,application=<app-name>"
+ }
}
```
-
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
new file mode 100644
index 000000000..583c9afa1
--- /dev/null
+++ b/docs/mkdocs.yml
@@ -0,0 +1,38 @@
+site_name: OSDU Notification Documentation
+site_description: OSDU Notification Service
+site_author: Chad Leong [SLB]
+
+# Repository
+repo_url: https://community.opengroup.org/osdu/platform/system/notification
+repo_name: notification
+
+# Copyright
+copyright: Copyright © 2024 Open Subsurface Data Universe Software
+
+theme:
+ name: material
+ palette:
+ - scheme: default
+ primary: indigo
+ accent: indigo
+ toggle:
+ icon: material/brightness-7
+ name: Switch to dark mode
+ - scheme: slate
+ primary: indigo
+ accent: indigo
+ toggle:
+ icon: material/brightness-4
+ name: Switch to light mode
+ font:
+ text: Roboto
+ code: Roboto Mono
+
+markdown_extensions:
+ - admonition
+ - sane_lists
+ - tables
+
+plugins:
+ - search
+ - git-revision-date
diff --git a/publish.yml b/publish.yml
new file mode 100644
index 000000000..ed0e5259f
--- /dev/null
+++ b/publish.yml
@@ -0,0 +1,30 @@
+run-test-pages:
+ stage: build
+ image: python:latest
+ script:
+ - cd docs
+ - pip install mkdocs-material
+ - pip install mkdocs-git-revision-date-plugin
+ - mkdocs build --site-dir ../test
+ allow_failure: true
+ artifacts:
+ paths:
+ - test
+ when: manual
+
+pages:
+ stage: publish
+ image: python:latest
+ tags: ["osdu-medium"]
+ script:
+ - cd docs
+ - pip install --default-timeout=1000 mkdocs-material
+ - pip install mkdocs-git-revision-date-plugin
+ - mkdocs build --site-dir ../public
+ allow_failure: true
+ needs: []
+ artifacts:
+ paths:
+ - public
+# rules:
+# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
--
GitLab