Merge branch 'gcp-push-subscriber-mode' into 'master'

Gcp push subscriber mode, tenant & iam migration(GONRG-2742)

See merge request !81

.gitlab-ci.yml

@@ -7,9 +7,11 @@ variables:
   OSDU_GCP_SERVICE: wks
   OSDU_GCP_VENDOR: gcp
   OSDU_GCP_APPLICATION_NAME: os-wks
+  OSDU_GCP_OS_TARGET_SCHEMA_KIND_TENANT: osdu:wks:testWellbore:1.3.1
   OSDU_GCP_WKS_MAPPING: WksMapping
   OSDU_GCP_WKS_BUCKET: nice-etching-277309-wks-mapping-definitions
-  OSDU_GCP_ENV_VARS: GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_V2_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,WKS_GCP_DATASTORE_MAPPINGINFOKIND=$OSDU_GCP_WKS_MAPPING,WKS_GCP_TENANTNAME=$TENANT_NAME,WKS_GCP_STORAGE_BUCKETNAME=$OSDU_GCP_WKS_BUCKET,PARTITION_API=$OSDU_GCP_PARTITION_API,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE
+  OSDU_GCP_ENV_VARS: WKS_GCP_AUDIENCES=$GOOGLE_AUDIENCE,WKS_GCP_ENTITLEMENTS_URL=$OSDU_GCP_ENTITLEMENTS_V2_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,WKS_GCP_DATASTORE_MAPPING_INFO_KIND=$OSDU_GCP_WKS_MAPPING,WKS_GCP_TENANT_NAME=$TENANT_NAME,WKS_GCP_STORAGE_BUCKET_NAME=$OSDU_GCP_WKS_BUCKET,PARTITION_API=$OSDU_GCP_PARTITION_API,SEARCH_API=$OSDU_GCP_WKS_SEARCH_API,STORAGE_API=$OSDU_GCP_WKS_STORAGE_API,SCHEMA_API=$OSDU_GCP_WKS_SCHEMA_API,WKS_GCP_SUBSCRIBER_MODE=push,WKS_GCP_REDIS_HOST=$REDIS_SEARCH_HOST --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
+  OSDU_GCP_DEPLOYMENTS_SUBDIR: deployments/scripts/gcp
   
 
 include:
@@ -38,7 +40,9 @@ include:
     file: 'cloud-providers/osdu-gcp-cloudrun.yml'
 
   - local: "/devops/azure/bootstrap.yml"
+  - local: "/devops/gcp/bootstrap.yml"
 
 osdu-gcp-test:
-  allow_failure: true
+  variables:
+    OS_TARGET_SCHEMA_KIND_TENANT: $OSDU_GCP_OS_TARGET_SCHEMA_KIND_TENANT
 

NOTICE

@@ -129,10 +129,10 @@ The following software have components provided under the terms of this license:
 - Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter)
 - Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
 - Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
+- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
 - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
 - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
 - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
-- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
 - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
 - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
 - Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries)
@@ -157,7 +157,6 @@ The following software have components provided under the terms of this license:
 - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
-- Mockito (from http://www.mockito.org)
 - Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer)
 - Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec)
 - Netty/Codec/DNS (from https://repo1.maven.org/maven2/io/netty/netty-codec-dns)
@@ -243,6 +242,7 @@ The following software have components provided under the terms of this license:
 - elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
 - elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
 - error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
+- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
 - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
 - io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
 - io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
@@ -267,6 +267,7 @@ The following software have components provided under the terms of this license:
 - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
 - mockito-core (from https://github.com/mockito/mockito)
 - mockito-core (from https://github.com/mockito/mockito)
+- mockito-core (from https://github.com/mockito/mockito)
 - org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
 - org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/)
 - org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
@@ -306,6 +307,7 @@ The following software have components provided under the terms of this license:
 - spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot)
 - spring-boot-starter-logging (from https://spring.io/projects/spring-boot)
 - spring-boot-starter-security (from https://spring.io/projects/spring-boot)
+- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
 - spring-boot-starter-test (from https://spring.io/projects/spring-boot)
 - spring-boot-starter-test (from https://spring.io/projects/spring-boot)
 - spring-boot-starter-tomcat (from https://spring.io/projects/spring-boot)
@@ -319,8 +321,11 @@ The following software have components provided under the terms of this license:
 - spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
 - spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
 - spring-security-config (from http://spring.io/spring-security)
+- spring-security-config (from http://spring.io/spring-security)
+- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
 - spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
 - spring-security-web (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-web)
+- spring-security-web (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-web)
 - swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations)
 - swagger-jaxrs (from )
 - tomcat-embed-core (from http://tomcat.apache.org/)
@@ -378,7 +383,6 @@ The following software have components provided under the terms of this license:
 - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Web with Auto Registration Module (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Mockito (from http://www.mockito.org)
 - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
 - Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
 - Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
@@ -634,7 +638,6 @@ The following software have components provided under the terms of this license:
 - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
-- Mockito (from http://www.mockito.org)
 - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
 - Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
 - Project Lombok (from https://projectlombok.org)
@@ -652,11 +655,14 @@ The following software have components provided under the terms of this license:
 - micrometer-core (from https://github.com/micrometer-metrics/micrometer)
 - mockito-core (from https://github.com/mockito/mockito)
 - mockito-core (from https://github.com/mockito/mockito)
+- mockito-core (from https://github.com/mockito/mockito)
+- mockito-inline (from https://github.com/mockito/mockito)
 - mockito-junit-jupiter (from https://github.com/mockito/mockito)
 - mockito-junit-jupiter (from https://github.com/mockito/mockito)
 - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
 - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java)
 - spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
+- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
 
 ========================================================================
 MPL-1.1

deployments/scripts/gcp/deploy_mapping_info_records.py

+#  Copyright 2021 Google LLC
+#  Copyright 2021 EPAM Systems, Inc
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import json
+import os
+from os import listdir
+from os.path import isfile, join
+from google.cloud import datastore
+
+local_folder = os.path.join(os.path.dirname(__file__),"../../mapping_info_records/opendes")
+wks_namespace = os.environ.get("WKS_NAMESPACE")
+kind = os.environ.get("WKS_KIND")
+tenant = os.environ.get("WKS_TENANT")
+datastore_client = datastore.Client(namespace=wks_namespace)
+
+
+def upload_records():
+  files = [f for f in listdir(local_folder) if isfile(join(local_folder, f))]
+  for file in files:
+    local_file = join(local_folder, file)
+    with open(local_file) as f:
+      data = json.load(f)
+      data['kind'] = data['kind'].replace('opendes', tenant)
+      data['sourceSchemaAuthority'] = data['sourceSchemaAuthority'].replace('opendes', tenant)
+      data['sourceSchemaKind'] = data['sourceSchemaKind'].replace('opendes', tenant)
+      complete_key = datastore_client.key(kind, data['id'])
+      mapping = datastore.Entity(key=complete_key)
+      mapping.update(data)
+      datastore_client.put(mapping)
+
+
+if __name__ == "__main__":
+  upload_records()

deployments/scripts/gcp/deploy_mappings.py

+#  Copyright 2021 Google LLC
+#  Copyright 2021 EPAM Systems, Inc
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+from os import listdir
+from os.path import isfile, join
+from google.cloud import storage
+
+local_folder = os.path.join(os.path.dirname(__file__), "../../mappings/opendes")
+storage_client = storage.Client()
+bucket_name = os.environ.get("WKS_BUCKET")
+tenant = os.environ.get("WKS_TENANT")
+bucket = storage_client.get_bucket(bucket_name)
+
+
+def upload_files():
+  files = [f for f in listdir(local_folder) if isfile(join(local_folder, f))]
+  for file in files:
+    local_file = join(local_folder, file)
+    fin = open(local_file, 'rt')
+    data = fin.read()
+    data = data.replace("opendes", tenant)
+    fin.close()
+    fin = open(local_file, 'wt')
+    fin.write(data)
+    fin.close()
+    blob = bucket.blob(file)
+    blob.upload_from_filename(local_file)
+
+
+if __name__ == "__main__":
+  upload_files()

deployments/scripts/gcp/requirements.txt

+google-cloud-datastore==2.1.3
+google-cloud-storage==1.40.0
\ No newline at end of file

devops/gcp/bootstrap.yml

+osdu_gcp_bootstrap:
+  stage: bootstrap
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  needs: ["osdu-gcp-deploy"]
+  extends:
+    - .osdu-gcp-variables
+  variables:
+    WKS_KIND: $OSDU_GCP_WKS_MAPPING
+    WKS_NAMESPACE: $OSDU_TENANT
+    WKS_TENANT: $OSDU_TENANT
+    WKS_BUCKET: $OSDU_GCP_WKS_BUCKET
+  script:
+    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
+    - gcloud config set project $OSDU_GCP_PROJECT
+    - export GOOGLE_APPLICATION_CREDENTIALS=$OSDU_GCP_DEPLOY_FILE
+    - pip install -r $OSDU_GCP_DEPLOYMENTS_SUBDIR/requirements.txt
+    - python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/deploy_mappings.py
+    - python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/deploy_mapping_info_records.py
+  only:
+    variables:
+      - $OSDU_GCP == 'true'
+
+osdu-gcp-test:
+  needs: ["osdu_gcp_bootstrap"]
+  only:
+    variables:
+      - $OSDU_GCP == 'true'

provider/wks-gcp/README.md

@@ -17,13 +17,22 @@ In order to run the service locally, you will need to have the following environ
 
 | name | value | description | sensitive? | source |
 | ---  | ---   | ---         | ---        | ---    |
-| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
 | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | yes | output of infrastructure deployment |
 | `GOOGLE_CLOUD_PROJECT` | ex `osdu-cicd-epam` | Google Cloud Project Id | yes | - |
-| `wks.gcp.datastore-mappingInfoKind` | WksMapping | kind for Datastore | yes | - |
-| `wks.gcp.tenantName` | ex `opendes` | Tenant name | yes | - |
-| `wks.gcp.storage-bucketName` | ex `osdu-cicd-epam-wks-mapping-definitions` | Storage bucket name for mapping definitions | yes | - |
+| `WKS_GCP_AUDIENCES` | ex `1234.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+| `WKS_GCP_DATASTORE_MAPPING_INFO_KIND` | WksMapping | kind for Datastore | yes | - |
+| `WKS_GCP_TENANT_NAME` | ex `opendes` | Tenant name | yes | - |
+| `WKS_GCP_STORAGE_BUCKET_NAME` | ex `osdu-cicd-epam-wks-mapping-definitions` | Storage bucket name for mapping definitions | yes | - |
+| `PARTITION_API` | ex `https://dev.osdu-gcp.go3-nrg.projects.epam.com/api/partition/v1/` | Partition service endpoint | no | - |
+| `SEARCH_API` | ex `https://os-search-jvmvia5dea-uc.a.run.app/api/search/v2` | Search service endpoint | no | - |
+| `STORAGE_API` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2` | Storage service endpoint | no | - |
+| `SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema service endpoint | no | - |
+| `WKS_GCP_ENTITLEMENTS_URL` | ex `https://dev.osdu-gcp.go3-nrg.projects.epam.com/api/entitlements/v2/` | Entitlements service endpoint | no | - |
+| `WKS_GCP_REDIS_HOST` | ex `127.0.0.1` | Redis host | no | - |
+| `WKS_GCP_SUBSCRIBER_MODE` | `push` OR `pull` | Pubsub subscriber mode, `push` mode turn on http endpoint in WKS service and allow to use it in envs that require from deployed service to serve HTTP requests (Cloud Run for example), `pull` mode turn off endpoint and WKS listen events from PubSub by itself | no | - |
+
+Depending on which subscriber mode chosen, the environment must have configured PubSub subscription for records-changed topic, for push mode there must be push subscription with configured endpoint `https://wks-service/api/wks-service/v1/_ah/push-handlers/enqueue` and Service account that can be authorized at entitlements. 
+For pull mode in PubSub must be present subscriptions for each partition in following format `records-changed-sub-<partition>-wks`.
 
 ### Run Locally
 Check that maven is installed:
@@ -118,6 +127,21 @@ You will need to have the following environment variables defined.
 | `STORAGE_URL` | ex `********/api/storage/v2/`| storage service URL | yes | - |
 | `HOST` | `` | schema service URL | yes | - |
 
+The mapping files must be added to the Storage and Datastore for the tests to run successfully, they can be found in [deployments](/deployments).
+Scripts for bootstrapping files can be found in [gcp scripts](/deployments/scripts/gcp).
+```bash
+pip install -r requrments.txt
+```
+You will need to have the following environment variables defined to run scripts.
+
+| name | value | description | sensitive? | source |
+| ---  | ---   | ---         | ---        | ---    |
+| `WKS_KIND` | ex `WksMapping`| Kind in Datastore used by WKS service | no | - |
+| `WKS_NAMESPACE` | ex `osdu`| Namespace in Datastore used by WKS service | no | - |
+| `WKS_TENANT` | ex `osdu`| WKS service tenant | no | - |
+| `WKS_BUCKET` | ex `nice-etching-277309-wks-mapping-definitions`| Bucket used by WKS service to store mapping files | no | - |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex`usr/key.json` | Google Service account credentials with write access to Google Storage and Datastore | yes | - |
+
 Execute following command to build code and run all the integration tests:
 
 ```bash

provider/wks-gcp/pom.xml

-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-	<parent>
-		<groupId>org.opengroup.osdu</groupId>
-		<artifactId>os-wks</artifactId>
-		<version>0.11.0-SNAPSHOT</version>
-		<relativePath>../../pom.xml</relativePath>
-	</parent>
-	<artifactId>os-wks-gcp</artifactId>
-	<description>Google cloud related implementation stuff.</description>
-	<packaging>jar</packaging>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.opengroup.osdu</groupId>
+        <artifactId>os-wks</artifactId>
+        <version>0.11.0-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+    <artifactId>os-wks-gcp</artifactId>
+    <description>Google cloud related implementation stuff.</description>
+    <packaging>jar</packaging>
 
-	<dependencies>
-		<dependency>
-			<groupId>org.opengroup.osdu</groupId>
-			<artifactId>os-wks-core</artifactId>
-			<version>0.11.0-SNAPSHOT</version>
-		</dependency>
-		<dependency>
-			<groupId>org.opengroup.osdu</groupId>
-			<artifactId>os-core-common</artifactId>
-			<version>0.10.0</version>
-		</dependency>
-		<dependency>
-			<groupId>org.opengroup.osdu</groupId>
-			<artifactId>core-lib-gcp</artifactId>
-			<version>0.10.0</version>
-		</dependency>
-		<dependency>
-			<groupId>com.google.cloud</groupId>
-			<artifactId>google-cloud-storage</artifactId>
-			<version>1.108.0</version>
-		</dependency>
-		<dependency>
-			<groupId>com.google.cloud</groupId>
-			<artifactId>google-cloud-datastore</artifactId>
-			<version>1.102.2</version>
-		</dependency>
-		<dependency>
-			<groupId>org.springframework.boot</groupId>
-			<artifactId>spring-boot-starter-test</artifactId>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.mockito</groupId>
-			<artifactId>mockito-all</artifactId>
-			<version>1.10.19</version>
-			<scope>test</scope>
-		</dependency>
+    <dependencies>
+        <dependency>
+            <groupId>org.opengroup.osdu</groupId>
+            <artifactId>os-wks-core</artifactId>
+            <version>0.11.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-security</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.opengroup.osdu</groupId>
+            <artifactId>os-core-common</artifactId>
+            <version>0.10.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.opengroup.osdu</groupId>
+            <artifactId>core-lib-gcp</artifactId>
+            <version>0.10.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-core</artifactId>
+            <version>1.38.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <version>1.108.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-datastore</artifactId>
+            <version>1.102.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+        </dependency>
 
-		<dependency>
-			<groupId>com.google.apis</groupId>
-			<artifactId>google-api-services-iam</artifactId>
-			<version>v1-rev310-1.25.0</version>
-		</dependency>
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+            <version>4.2.0</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>3.11.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-inline</artifactId>
+            <version>3.11.2</version>
+            <scope>test</scope>
+        </dependency>
 
-		<dependency>
-			<groupId>io.cucumber</groupId>
-			<artifactId>cucumber-java8</artifactId>
-			<version>5.4.0</version>
-			<scope>test</scope>
-		</dependency>
+    </dependencies>
 
-		<dependency>
-			<groupId>io.cucumber</groupId>
-			<artifactId>cucumber-junit</artifactId>
-			<version>5.4.0</version>
-			<scope>test</scope>
-		</dependency>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                        <configuration>
+                            <classifier>spring-boot</classifier>
+                            <mainClass>
+                                org.opengroup.osdu.wks.WksServiceApplicationGcp
+                            </mainClass>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
 
-		<dependency>
-			<groupId>io.cucumber</groupId>
-			<artifactId>cucumber-guice</artifactId>
-			<version>5.4.0</version>
-			<scope>test</scope>
-		</dependency>
+            <plugin>
+                <artifactId>maven-war-plugin</artifactId>
+                <configuration>
+                    <failOnMissingWebXml>false</failOnMissingWebXml>
+                </configuration>
+            </plugin>
 
-		<dependency>
-			<groupId>com.google.inject</groupId>
-			<artifactId>guice</artifactId>
-			<version>4.2.0</version>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.projectlombok</groupId>
-			<artifactId>lombok</artifactId>
-		</dependency>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.0.0-M3</version>
+                <configuration>
+                    <skipTests>${skipUnitTests}</skipTests>
+                    <forkCount>3</forkCount>
+                    <reuseForks>true</reuseForks>
+                    <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
+                </configuration>
+            </plugin>
 
-	</dependencies>
-
-	<build>
-		<plugins>
-			<plugin>
-				<groupId>org.springframework.boot</groupId>
-				<artifactId>spring-boot-maven-plugin</artifactId>
-				<executions>
-					<execution>
-						<goals>
-							<goal>repackage</goal>
-						</goals>
-						<configuration>
-							<classifier>spring-boot</classifier>
-							<mainClass>
-								org.opengroup.osdu.wks.WksServiceApplication
-							</mainClass>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
-
-			<plugin>
-				<artifactId>maven-war-plugin</artifactId>
-				<configuration>
-					<failOnMissingWebXml>false</failOnMissingWebXml>
-				</configuration>
-			</plugin>
-
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-surefire-plugin</artifactId>
-				<version>3.0.0-M3</version>
-				<configuration>
-					<skipTests>${skipUnitTests}</skipTests>
-					<forkCount>3</forkCount>
-					<reuseForks>true</reuseForks>
-					<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
-				</configuration>
-			</plugin>
-
-		</plugins>
-	</build>
+        </plugins>
+    </build>
 
 
 </project>

provider/wks-gcp/src/main/java/org/opengroup/osdu/wks/WksServiceApplicationGcp.java

 /*
-  Copyright 2021 Google LLC
-  Copyright 2021 EPAM Systems, Inc
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-*/
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.opengroup.osdu.wks;
 
-import org.opengroup.osdu.wks.provider.interfaces.SubscriptionManager;
-
+import org.opengroup.osdu.core.gcp.di.PartitionTenantInfoFactoryBean;
+import org.opengroup.osdu.core.gcp.multitenancy.StorageFactory;
+import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
 
 @SpringBootApplication
-@ComponentScan(value = {"org.opengroup.osdu"})
+@ComponentScan(value = {
+    "org.opengroup.osdu"
+}, excludeFilters = {
+    @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = {
+        PartitionTenantInfoFactoryBean.class, WksServiceApplication.class, TenantFactory.class, StorageFactory.class})})
 public class WksServiceApplicationGcp {
 
-  public static void main(String[] args) {
-    ApplicationContext context = SpringApplication
-        .run(WksServiceApplicationGcp.class, args);
-
-    SubscriptionManager subscriptionManager = context.getBean(SubscriptionManager.class);
-    subscriptionManager.subscribeRecordsChangeEvent();
-  }
+    public static void main(String[] args) {
+        SpringApplication.run(WksServiceApplicationGcp.class, args);
+    }
 }

provider/wks-gcp/src/main/java/org/opengroup/osdu/wks/api/EnqueueApi.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.