diff --git a/devops/gc/pipeline/override-stages.yml b/devops/gc/pipeline/override-stages.yml
index 8ddb799329f9eb16d6d152bef69bfed416e0e2b5..4cbb7fed6feb280ec9e985c75fd06d31a1c36548 100644
--- a/devops/gc/pipeline/override-stages.yml
+++ b/devops/gc/pipeline/override-stages.yml
@@ -12,3 +12,36 @@ gc-dev2-test:
gc-test:
variables:
OPA_INTEGRATION_ENABLED: "true"
+
+
+download_gc_plugins:
+ image: maven:3.8.3-openjdk-17-slim
+ stage: build
+ variables:
+ GC_OSM_PACKAGE_REGISTRY_URL: "https://community.opengroup.org/api/v4/projects/1476/packages/maven"
+ GC_OSM_VERSION: "0.27.2"
+ GC_OBM_PACKAGE_REGISTRY_URL: "https://community.opengroup.org/api/v4/projects/1475/packages/maven"
+ GC_OBM_VERSION: "0.27.2"
+ GC_OQM_PACKAGE_REGISRTY_URL: "https://community.opengroup.org/api/v4/projects/1477/packages/maven"
+ GC_OQM_VERSION: "0.27.2"
+ GC_APD_PACKAGE_REGISTRY_URL: "https://community.opengroup.org/api/v4/projects/1480/packages/maven"
+ GC_APD_VERSION: "0.27.2"
+ artifacts:
+ paths:
+ - ./tmp-gc/*.jar
+ when: always
+ expire_in: 1 days
+ script:
+ - mvn dependency:copy -DrepoUrl=$GC_OSM_PACKAGE_REGISTRY_URL -Dartifact="org.opengroup.osdu:gc-osm-datastore:$GC_OSM_VERSION:jar:plugin" -Dtransitive=false -DoutputDirectory="./tmp-gc"
+ - mvn dependency:copy -DrepoUrl=$GC_OBM_PACKAGE_REGISTRY_URL -Dartifact="org.opengroup.osdu:gc-obm-gs:$GC_OBM_VERSION:jar:plugin" -Dtransitive=false -DoutputDirectory="./tmp-gc"
+ - mvn dependency:copy -DrepoUrl=$GC_OQM_PACKAGE_REGISRTY_URL -Dartifact="org.opengroup.osdu:gc-oqm-pubsub:$GC_OQM_VERSION:jar:plugin" -Dtransitive=false -DoutputDirectory="./tmp-gc"
+ - mvn dependency:copy -DrepoUrl=$GC_APD_PACKAGE_REGISRTY_URL -Dartifact="org.opengroup.osdu:gc-apd-acc:$GC_APD_VERSION:jar:plugin" -Dtransitive=false -DoutputDirectory="./tmp-gc"
+ only:
+ variables:
+ - $GC == '1'
+
+gc-containerize-gitlab:
+ needs: ["gc-compile-and-unit-test", "download_gc_plugins"]
+
+gc-containerize-gcr:
+ needs: ["gc-compile-and-unit-test", "download_gc_plugins"]
diff --git a/provider/storage-gc/README.md b/provider/storage-gc/README.md
index d4e2f89514f3c767b996033c7445d9245728d2b6..330b61653cc24f48c16ea252153c5678660c5f70 100644
--- a/provider/storage-gc/README.md
+++ b/provider/storage-gc/README.md
@@ -5,7 +5,6 @@ entire metadata life-cycle such as ingestion (persistence), modification, deleti
## Table of Contents <a name="TOC"></a>
* [Getting started](#Getting-started)
-* [Mappers](#Mappers)
* [Settings and Configuration](#Settings-and-Configuration)
* [Run service](#Run-service)
* [Testing](#Testing)
@@ -19,24 +18,6 @@ entire metadata life-cycle such as ingestion (persistence), modification, deleti
These instructions will get you a copy of the project up and running on your local machine for development and testing
purposes. See deployment for notes on how to deploy the project on a live system.
-## Mappers
-
-This is a universal solution created using EPAM OSM, OBM and OQM mappers technology. It allows you to work with various
-implementations of KV stores, Blob stores and message brokers.
-
-For more information about mappers:
-- [OSM Readme](https://community.opengroup.org/osdu/platform/system/lib/cloud/gcp/osm/-/blob/main/README.md)
-- [OBM Readme](https://community.opengroup.org/osdu/platform/system/lib/cloud/gcp/obm/-/blob/master/README.md)
-- [OQM Readme](https://community.opengroup.org/osdu/platform/system/lib/cloud/gcp/oqm/-/blob/master/README.md)
-
-### Limitations of the current version
-
-In the current version, the mappers are equipped with several drivers to the stores and the message broker:
-
-- OSM (mapper for KV-data): Google Datastore; Postgres
-- OBM (mapper to Blob stores): Google Cloud Storage (GCS); MinIO
-- OQM (mapper to message brokers): Google PubSub; RabbitMQ
-
## Settings and Configuration
### Requirements:
@@ -48,8 +29,6 @@ In the current version, the mappers are equipped with several drivers to the sto
2. For Google Cloud only
- GCloud SDK with java (latest version)
-### Baremetal Service Configuration:
-[Baremetal service configuration ](docs/baremetal/README.md)
### Google Cloud Service Configuration:
[Google Cloud service configuration ](docs/gc/README.md)
@@ -139,12 +118,9 @@ cd provider/storage-gc/ && mvn spring-boot:run
This section describes how to run cloud OSDU E2E tests.
-### Baremetal test configuration:
-[Baremetal service configuration ](docs/baremetal/README.md)
### Google Cloud test configuration:
[Google Cloud service configuration ](docs/gc/README.md)
-
## Deployment
Storage Service is compatible with App Engine Flexible Environment and Cloud Run.
@@ -152,8 +128,8 @@ Storage Service is compatible with App Engine Flexible Environment and Cloud Run
* To deploy into Cloud run, please, use this documentation:
https://cloud.google.com/run/docs/quickstarts/build-and-deploy
-* To deploy into App Engine, please, use this documentation:
- https://cloud.google.com/appengine/docs/flexible/java/quickstart
+* To deploy into GKE, please, use this documentation:
+ https://cloud.google.com/kubernetes-engine/docs/deploy-app-cluster
## Tutorial
diff --git a/provider/storage-gc/cloudbuild/Dockerfile.cloudbuild b/provider/storage-gc/cloudbuild/Dockerfile.cloudbuild
index a10c353c80730eb96038ddb31b39b34dc3ac082d..99b62633d9e18df8273a276e7eed04254d4df7d2 100644
--- a/provider/storage-gc/cloudbuild/Dockerfile.cloudbuild
+++ b/provider/storage-gc/cloudbuild/Dockerfile.cloudbuild
@@ -8,6 +8,13 @@ ENV PROVIDER_NAME $PROVIDER_NAME
ARG PORT
ENV PORT $PORT
+ENV LOADER_PATH="gc/"
+
+COPY tmp-gc/gc-oqm-pubsub-*.jar gc/oqm-pubsub.jar
+COPY tmp-gc/gc-obm-gs-*.jar gc/obm-gs.jar
+COPY tmp-gc/gc-osm-datastore-*.jar gc/osm-datastore.jar
+COPY tmp-gc/gc-apd-acc-*.jar gc/apd-acc.jar
+
# Copy the jar to the production image from the builder stage.
COPY provider/storage-${PROVIDER_NAME}/target/storage-${PROVIDER_NAME}-*-spring-boot.jar storage-${PROVIDER_NAME}.jar
@@ -18,4 +25,10 @@ RUN groupadd -g 10001 -r nonroot \
USER 10001:10001
# Run the web service on container startup.
-CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=${PORT} -Dlog4j.formatMsgNoLookups=true -jar /app/storage-${PROVIDER_NAME}.jar
+CMD java --add-opens java.base/java.lang=ALL-UNNAMED \
+ --add-opens java.base/java.lang.reflect=ALL-UNNAMED \
+ -Djava.security.egd=file:/dev/./urandom \
+ -Dserver.port=${PORT} \
+ -Dlog4j.formatMsgNoLookups=true \
+ -Dloader.main=org.opengroup.osdu.storage.provider.gcp.StorageApplicationGCP \
+ -jar /app/storage-${PROVIDER_NAME}.jar
\ No newline at end of file
diff --git a/provider/storage-gc/docs/anthos/README.md b/provider/storage-gc/docs/anthos/README.md
deleted file mode 100644
index 3afb8034045d1c987824692fe6e935448a7af7b7..0000000000000000000000000000000000000000
--- a/provider/storage-gc/docs/anthos/README.md
+++ /dev/null
@@ -1,565 +0,0 @@
-# Service Configuration for Baremetal
-
-## Table of Contents <a name="TOC"></a>
-* [Environment variables](#Environment-variables)
- * [Common properties for all environments](#Common-properties-for-all-environments)
- * [For Mappers to activate drivers](#For-Mappers-to-activate-drivers)
-* [Requirements for requests](#Requirements-for-requests)
-* [Configuring mappers Datasources](#Configuring-mappers-DataSources)
- * [For OSM Postgres](#For-OSM-Postgres)
- * [Postgres schema configuration](#Postgres-schema-configuration)
- * [For OBM MinIO](#For-OBM-MinIO)
- * [Object store configuration](#ObjectStoreConfig)
- * [For OQM RabbitMQ](#For-OQM-RabbitMQ)
- * [Exchanges and queues configuration](#Exchanges-and-queues-configuration)
-* [Interaction with message brokers](#Interaction-with-message-brokers)
-* [Keycloak configuration](#Keycloak-configuration)
-* [Running E2E Tests](#Running-E2E-Tests)
-* [Running locally](#Running-locally)
-* [License](#License)
-
-## Environment variables
-
-### Common properties for all environments
-
-Define the following environment variables.
-
-Must have:
-
-| name | value | description | sensitive? | source |
-|-------------------------------------------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|--------|
-| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for Google Cloud environment | false | - |
-| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` | yes | - |
-| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens | yes | - |
-| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - |
-| `<POSTGRES_PASSWORD_ENV_VARIABLE_NAME>` | ex `POSTGRES_PASS_OSDU` | Postgres password env name, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Storage service | yes | - |
-| `<MINIO_SECRETKEY_ENV_VARIABLE_NAME>` | ex `MINIO_SECRET_OSDU` | Minio secret env name, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Storage service | yes | - |
-| `<AMQP_PASSWORD_ENV_VARIABLE_NAME>` | ex `AMQP_PASS_OSDU` | Amqp password env name, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Storage service | yes | - |
-| `<AMQP_ADMIN_PASSWORD_ENV_VARIABLE_NAME>` | ex `AMQP_ADMIN_PASS_OSDU` | Amqp admin password env name, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Storage service | yes | - |
-| `STORAGE_SERVICE_ACCOUNT_EMAIL` | `workload-storage@keycloak.com` | Storage service account email, used during OQM events processing | no | - |
-
-| name | value | description | sensitive? | source |
-|--------------------------------------------|-----------------------------------------------|---------------------------------------------------------------------------------------|------------|-------------------------------------|
-| `LOG_PREFIX` | `storage` | Logging prefix | no | - |
-| `SERVER_SERVLET_CONTEXPATH` | `/api/storage/v2/` | Servlet context path | no | - |
-| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
-| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment |
-| `PUBSUB_SEARCH_TOPIC` | ex `records-changed` | RabbitMQ topic name | no | |
-| `REDIS_STORAGE_HOST` | ex `127.0.0.1` | Redis host for storage | no | |
-| `REDIS_STORAGE_PASSWORD` | ex `*****` | Redis storage host password | yes | |
-| `REDIS_STORAGE_WITH_SSL` | ex `true` or `false` | Redis storage host ssl config | no | |
-| `REDIS_STORAGE_EXPIRATION` | ex `30` | Redis storage cache expiration in seconds | no | |
-| `POLICY_API` | ex `http://localhost:8080/api/policy/v1/` | Police service endpoint | no | output of infrastructure deployment |
-| `POLICY_ID` | ex `search` | policeId from ex `http://localhost:8080/api/policy/v1/policies`. Look at `POLICY_API` | no | - |
-| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
-| `PARTITION_PROPERTIES_STORAGE_BUCKET_NAME` | ex `storage.bucket.name` | Name of partition property for storage bucket name value | yes | - |
-| `SYSTEM_PARTITION_ID` | ex `system` | System partition ID |
-
-These variables define service behavior, and are used to switch between `Reference` or `Google Cloud` environments, their overriding and usage in mixed mode was not tested.
-Usage of spring profiles is preferred.
-
-| name | value | description | sensitive? | source |
-|--------------------------|----------------------|---------------------------------------------------------------------------------------------------------------------------|------------|--------|
-| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
-| `OSMDRIVER` | `postgres` | Osm driver mode that defines which KV storage will be used | no | - |
-| `OBMDRIVER` | `minio` | Obm driver mode that defines which object storage will be used | no | - |
-| `OQMDRIVER` | `rabbitmq` | Oqm driver mode that defines which message broker will be used | no | - |
-| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` | Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
-
-### Properties set in Partition service:
-
-Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present **not value itself**, but **ENV variable name**.
-This variable should be present in environment of service that need that variable.
-
-Example:
-```
- "elasticsearch.port": {
- "sensitive": false, <- value not sensitive
- "value": "9243" <- will be used as is.
- },
- "elasticsearch.password": {
- "sensitive": true, <- value is sensitive
- "value": "ELASTIC_SEARCH_PASSWORD_OSDU" <- service consumer should have env variable ELASTIC_SEARCH_PASSWORD_OSDU with elastic search password
- }
-```
-
-### For Mappers to activate drivers
-
-| name | value | description |
-|-----------|-----------|---------------------------------------------------------|
-| OSMDRIVER | datastore | to activate **OSM** driver for **Google Datastore** |
-| OSMDRIVER | postgres | to activate **OSM** driver for **PostgreSQL** |
-| OBMDRIVER | gcs | to activate **OBM** driver for **Google Cloud Storage** |
-| OBMDRIVER | minio | to activate **OBM** driver for **MinIO** |
-| OQMDRIVER | pubsub | to activate **OQM** driver for **Google PubSub** |
-| OQMDRIVER | rabbitmq | to activate **OQM** driver for **Rabbit MQ** |
-
-## Requirements for requests
-
-Record identifiers cannot contain a space character. At the same time, they may contain a % character, which, when
-combined with subsequent numeric characters, may cause the application to misinterpret that combination. For example,
-the "%20" combination will be interpreted as a space " " character. To correctly transfer such an identifier, you should
-additionally perform the url-encode operation on it. This functionality can be built into the front-end application, or
-you can use an online url-encoder tool ( eg.: https://www.urlencoder.org/). Thus, having ID "osdu:
-work-product-component--WellboreMarkerSet:3D%20Kirchhoff%20DepthMigration" (with %20 combination)
-you should url-encode it and request
-"osdu%3Awork-product-component--WellboreMarkerSet%3A3D%2520Kirchhoff%2520DepthMigration" instead.
-
-## Configuring mappers DataSources
-
-When using non-Google-Cloud-native technologies, property sets must be defined on the Partition service as part of
-PartitionInfo for each Tenant.
-
-They are specific to each storage technology:
-
-### For OSM Postgres
-
-**database structure**
-OSM works with data logically organized as "partition"->"namespace"->"kind"->"record"->"columns". The above sequence
-describes how it is named in Google Datastore, where "partition" maps to "Google Cloud project".
-
-For example, this is how **Datastore** OSM driver contains records for "RecordsChanged" data register:
-
-| hierarchy level | value |
-|---------------------|----------------------------------|
-| partition (opendes) | osdu-cicd-epam |
-| namespace | opendes |
-| kind | StorageRecord |
-| record | `<multiple kind records>` |
-| columns | acl; bucket; kind; legal; etc... |
-
-And this is how **Postgres** OSM driver does. Notice, the above hierarchy is kept, but Postgres uses alternative entities
-for it.
-
-| Datastore hierarchy level | | Postgres alternative used |
-|------------------------------------|-----|----------------------------|
-| partition (Google Cloud project) | == | Postgres server URL |
-| namespace | == | Schema |
-| kind | == | Table |
-| record | == | '<multiple table records>' |
-| columns | == | id, data (jsonb) |
-
-As we can see in the above table, Postgres uses different approach in storing business data in records. Not like
-Datastore, which segments data into multiple physical columns, Postgres organises them into the single JSONB "data"
-column. It allows provisioning new data registers easily not taking care about specifics of certain registers structure.
-In the current OSM version (as on December'21) the Postgres OSM driver is not able to create new tables in runtime.
-
-So this is a responsibility of DevOps / CI/CD to provision all required SQL tables (for all required data kinds) when on new
-environment or tenant provisioning when using Postgres. Detailed instructions (with examples) for creating new tables is
-in the **OSM module Postgres driver README.md** `org/opengroup/osdu/core/gcp/osm/translate/postgresql/README.md`
-
-As a quick shortcut, this example snippet can be used by DevOps DBA:
-```postgres-psql
---CREATE SCHEMA "exampleschema";
-CREATE TABLE exampleschema."ExampleKind"(
- id text COLLATE pg_catalog."default" NOT NULL,
- pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
- data jsonb NOT NULL,
- CONSTRAINT ExampleKind_id UNIQUE (id)
-);
-CREATE INDEX ExampleKind_datagin ON exampleschema."ExampleKind" USING GIN (data);
-```
-
-**prefix:** `osm.postgres`
-It can be overridden by:
-
-- through the Spring Boot property `osm.postgres.partition-properties-prefix`
-- environment variable `OSM_POSTGRES_PARTITION_PROPERTIES_PREFIX`
-
-**PropertySet:**
-
-| Property | Description |
-|----------------------------------|-------------|
-| osm.postgres.datasource.url | server URL |
-| osm.postgres.datasource.username | username |
-| osm.postgres.datasource.password | password |
-
-<details><summary>Example of a definition for a single tenant</summary>
-
-```
-
-curl -L -X PATCH 'https://api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
- "properties": {
- "osm.postgres.datasource.url": {
- "sensitive": false,
- "value": "jdbc:postgresql://127.0.0.1:5432/postgres"
- },
- "osm.postgres.datasource.username": {
- "sensitive": false,
- "value": "postgres"
- },
- "osm.postgres.datasource.password": {
- "sensitive": true,
- "value": "<POSTGRES_PASSWORD_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
- }
- }
-}'
-
-```
-
-</details>
-
-#### Postgres schema configuration
-
-```
---CREATE SCHEMA "opendes";
---DROP TABLE opendes."StorageRecord";
-CREATE TABLE opendes."StorageRecord"(
- id text COLLATE pg_catalog."default" NOT NULL,
- pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
- data jsonb NOT NULL,
- CONSTRAINT StorageRecord_id UNIQUE (id)
-);
-CREATE INDEX StorageRecord_datagin ON opendes."StorageRecord" USING GIN (data);
------------------
---DROP TABLE opendes."StorageSchema";
-CREATE TABLE opendes."StorageSchema"(
- id text COLLATE pg_catalog."default" NOT NULL,
- pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
- data jsonb NOT NULL,
- CONSTRAINT StorageSchema_id UNIQUE (id)
-);
-CREATE INDEX StorageSchema_datagin ON opendes."StorageSchema" USING GIN (data);
-
-
-```
-
-Example of filling tables
-
-```
-
-INSERT INTO <partitionId>."StorageRecord"(
-id, data)
-VALUES ('726612843', '{
- "id": 726612843,
- ...
-}');
-
-```
-
-### For OBM MinIO
-
-**prefix:** `obm.minio`
-It can be overridden by:
-
-- through the Spring Boot property `osm.postgres.partition-properties-prefix`
-- environment variable `OBM_MINIO_PARTITION_PROPERTIES_PREFIX`
-
-**PropertySet:**
-
-| Property | Description |
-|---------------------|------------------------|
-| obm.minio.endpoint | server URL |
-| obm.minio.accessKey | credentials access key |
-| obm.minio.secretKey | credentials secret key |
-
-<details><summary>Example of a definition for a single tenant</summary>
-
-```
-
-curl -L -X PATCH 'https:///api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
- "properties": {
- "obm.minio.endpoint": {
- "sensitive": false,
- "value": "http://localhost:9000"
- },
- "obm.minio.accessKey": {
- "sensitive": false,
- "value": "minioadmin"
- },
- "obm.minio.secretKey": {
- "sensitive": true,
- "value": "<MINIO_SECRETKEY_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
- }
- }
-}'
-
-```
-
-</details>
-
-### Object store configuration <a name="ObjectStoreConfig"></a>
-
-#### Used Technology
-
-MinIO (or any other supported by OBM)
-
-#### Per-tenant buckets configuration
-
-These buckets must be defined in tenants’ dedicated object store servers.
-OBM connection properties of these servers (url, etc.) are defined as specific properties in tenants’ PartitionInfo registration objects at the Partition service as described in accordant sections of this document.
-
-
-<table>
- <tr>
- <td>Bucket Naming template
- </td>
- <td>Permissions required
- </td>
- </tr>
- <tr>
- <td><PartitionInfo.projectId-PartitionInfo.name><strong>-records</strong>
- </td>
- <td>ListObjects, CRUDObject
- </td>
- </tr>
-</table>
-
-or
-
-We can use Partition Service to get a bucket name:
-
-```
-curl -L -X PATCH 'https:///api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
- "properties": {
- "partition.properties.storage.bucket.name": {
- "sensitive": true,
- "value": "PARTITION_PROPERTIES_STORAGE_BUCKET_NAME"
- }
- }
-}'
-
-```
-
-### For OQM RabbitMQ
-
-**prefix:** `oqm.rabbitmq`
-It can be overridden by:
-
-- through the Spring Boot property `oqm.rabbitmq.partition-properties-prefix`
-- environment variable `OQM_RABBITMQ_PARTITION_PROPERTIES_PREFIX``
-
-**PropertySet** (for two types of connection: messaging and admin operations):
-
-| Property | Description |
-|-----------------------------|--------------------------|
-| oqm.rabbitmq.amqp.host | messaging hostname or IP |
-| oqm.rabbitmq.amqp.port | - port |
-| oqm.rabbitmq.amqp.path | - path |
-| oqm.rabbitmq.amqp.username | - username |
-| oqm.rabbitmq.amqp.password | - password |
-| oqm.rabbitmq.admin.schema | admin host schema |
-| oqm.rabbitmq.admin.host | - host name |
-| oqm.rabbitmq.admin.port | - port |
-| oqm.rabbitmq.admin.path | - path |
-| oqm.rabbitmq.admin.username | - username |
-| oqm.rabbitmq.admin.password | - password |
-
-<details><summary>Example of a single tenant definition</summary>
-
-```
-
-curl -L -X PATCH 'https://api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
- "properties": {
- "oqm.rabbitmq.amqp.host": {
- "sensitive": false,
- "value": "localhost"
- },
- "oqm.rabbitmq.amqp.port": {
- "sensitive": false,
- "value": "5672"
- },
- "oqm.rabbitmq.amqp.path": {
- "sensitive": false,
- "value": ""
- },
- "oqm.rabbitmq.amqp.username": {
- "sensitive": false,
- "value": "guest"
- },
- "oqm.rabbitmq.amqp.password": {
- "sensitive": true,
- "value": "<AMQP_PASSWORD_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
- },
- "oqm.rabbitmq.admin.schema": {
- "sensitive": false,
- "value": "http"
- },
- "oqm.rabbitmq.admin.host": {
- "sensitive": false,
- "value": "localhost"
- },
- "oqm.rabbitmq.admin.port": {
- "sensitive": false,
- "value": "9002"
- },
- "oqm.rabbitmq.admin.path": {
- "sensitive": false,
- "value": "/api"
- },
- "oqm.rabbitmq.admin.username": {
- "sensitive": false,
- "value": "guest"
- },
- "oqm.rabbitmq.admin.password": {
- "sensitive": true,
- "value": "<AMQP_ADMIN_PASSWORD_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
- }
- }
-}'
-
-```
-
-</details>
-
-## Exchanges and queues configuration
-
-At RabbitMq should be created set of exchanges and queues.
-
-| topic name | subscription name | description | sensitive? | env var to override |
-|------------------------------------------|---------------------------------|------------------------------------------------------------|------------|-------------------------------------------------------------------------|
-| `records-changed` | - | Search topic for pushing | yes | `PUBSUB_SEARCH_TOPIC` |
-| `legaltags-changed` | `storage-oqm-legaltags-changed` | Legaltags topic for consuming | yes | `LEGAL_TAGS_CHANGED_TOPIC_NAME`, `LEGAL_TAGS_CHANGED_SUBSCRIPTION_NAME` |
-| `storage-oqm-legaltags-changed-exchange` | - | Service topic for delaying failed legal tag changed events | - | - |
-
-
-
-## Interaction with message brokers
-
-### Specifics of work through PULL subscription
-
-To receive messages from brokers, this solution uses the PULL-subscriber mechanism to get 'record_changed' messages.
-This is its cardinal difference from other implementations that use PUSH-subscribers (webhooks). This opens a wide
-choice when choosing brokers.
-
-When using PULL-subscribers, there is a need to restore Storage service subscribers at the start of Storage service.
-This magic happens in the `OqmSubscriberManager.java` class from `OQM` in the @PostConstruct method.
-
-## Keycloak configuration
-
-[Keycloak service accounts setup](https://www.keycloak.org/docs/latest/server_admin/#_service_accounts)
-
-Configure Clients. One Client per OSDU service. Set them “confidentialâ€.
-
-
-
-Each Client has embedded Service Account (SA) option. Enable SAs for Clients, make “Authorization enabledâ€:
-
-
-
-Add `partition-and-entitlements` scope to `Default Client Scopes` and generate Keys.
-
-Give `client-id` and `client-secret` to services, which should be authorized within the platform.
-
-## Running E2E Tests
-
-You will need to have the following environment variables defined.
-
-| name | value | description | sensitive? | source |
-|------------------------------------------------|------------------------------------------------|------------------------------------------------------------------|---------------------------------------------------|--------|
-| `DEPLOY_ENV` | `empty` | Required but not used, should be set up with string "empty" | no | - |
-| `GROUP_ID` | ex`opendes-gc.projects.com` | OSDU R2 to run tests under | no | - |
-| `LEGAL_URL` | ex`http://localhsot:8080/api/legal/v1/` | Legal API endpoint | no | - |
-| `STORAGE_URL` | ex`http://localhost:8080/api/storage/v2/` | Endpoint of storage service | no | - |
-| `TENANT_NAME` | ex `opendes` | OSDU tenant used for testing | no | -- |
-| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
-| `TEST_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$INTEGRATION_TESTER` | -- |
-| `TEST_NO_ACCESS_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$NO_ACCESS_INTEGRATION_TESTER` | yes | -- |
-| `TEST_NO_ACCESS_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$NO_ACCESS_INTEGRATION_TESTER` | -- |
-| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | -- |
-| `OPA_INTEGRATION_ENABLED` | `true` OR `false` | Should be update if integration with OPA\Policy enabled\disabled | no | -- |
-| `ENTITLEMENTS_URL` | ex`http://localhost:8080/api/entitlements/v2/` | Endpoint of entitlements service | no | - |
-| `DATA_ROOT_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for data root tester | yes | - |
-| `DATA_ROOT_OPENID_PROVIDER_CLIENT_SECRET` | `********` | Client secret for data root tester | yes | - |
-
-**Entitlements configuration for integration accounts**
-
-| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER | DATA_ROOT_TESTER |
-|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------|-----------------------------------------------------------------------------------------|
-| users<br/>service.entitlements.user<br/>service.storage.admin<br/>service.storage.creator<br/>service.storage.viewer<br/>service.legal.admin<br/>service.legal.editor<br/>data.test1<br/>data.integration.test | users<br/>service.entitlements.user<br/>service.storage.admin | users<br/>users.data.root<br/>service.entitlements.user<br/>service.storage.viewer<br/> |
-
-Execute following command to build code and run all the integration tests:
-
- ```bash
- # Note: this assumes that the environment variables for integration tests as outlined
- # above are already exported in your environment.
- # build + install integration test core
- $ (cd testing/storage-test-core/ && mvn clean install)
- ```
-
- ```bash
- # build + run Google Cloud integration tests.
- $ (cd testing/storage-test-baremetal/ && mvn clean test)
- ```
-
-
-## Running locally
-To run storage service locally connected with baremetal environment:
-#### Specify mappers drivers property or run `SPRING_PROFILES_ACTIVE=anthos`
-```properties
-obmDriver=minio
-osmDriver=postgres
-oqmDriver=rabbitmq
-```
-#### Specify osdu services urls:
-```properties
-HOST=https://osdu.ref.gcp.gnrg-osdu.projects.epam.com
-AUTHORIZE_API=${HOST}/api/entitlements/v2
-CRS_API=${HOST}/api/crs/v2
-LEGALTAG_API=${HOST}/api/legal/v1
-PARTITION_API=${HOST}/api/partition/v1/
-```
-#### Auth variables:
-```properties
-opa.enabled=false;
-partition-auth-enabled=false
-service.token.provider=OPENID
-OPENID_PROVIDER_CLIENT_ID=${CLIENT_ID};
-OPENID_PROVIDER_CLIENT_SECRET=${CLIENT_SECRET};
-OPENID_PROVIDER_URL=${OPENID_PROVIDER_URL};
-```
-#### Redis variables:
-```properties
-REDIS_STORAGE_HOST=127.0.0.1
-REDIS_GROUP_HOST=127.0.0.1
-```
-#### Partition service should contain non-production sensitive properties to override them with localhost and use through a custom prefixes.
-##### OSM:
-```properties
-POSTGRES_DATASOURCE_URL_OSDU=jdbc:postgresql://localhost:5432/storage;
-POSTGRES_DB_USERNAME_OSDU=${POSTGRES_USERNAME};
-POSTGRES_DB_PASSWORD_OSDU=${POSTGRES_PASSWORD};
-```
-##### OBM:
-```properties
-OBM_MINIO_PARTITION_PROPERTIES_PREFIX=obm.minio.localDebug;
-MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY};
-MINIO_SECRET_KEY=${MINIO_SECRET_KEY};
-```
-##### OQM:
-```properties
-OQM_RABBITMQ_PARTITION_PROPERTIES_PREFIX=oqm.rabbitmq.localDebug;
-RABBIT_MQ_HOST=localhost;
-RABBITMQ_ADMIN_HOST=localhost;
-RABBITMQ_ADMIN_USERNAME=${RABBITMQ_ADMIN_USERNAME};
-RABBITMQ_ADMIN_PASSWORD=${RABBITMQ_ADMIN_PASSWORD};
-```
-#### To connect postgres or minio or rabbitmq port-forwarding should be used:
-```shell
-gcloud auth list
-gcloud config set account <account_name>
-gcloud config set project <project_name>
-kubectl port-forward <rabbit_pod_name> 15672:15672
-kubectl port-forward <rabbit_pod_name> 5672:5672
-kubectl port-forward <minio_pod_name> 9000:9000
-gcloud components install cloud_sql_proxy
-cloud_sql_proxy -instances=<instance_connection_string> -credential_file=<baremetal_service_account_json_file>
-```
-
-## License
-
-Copyright © Google LLC
-
-Copyright © EPAM Systems
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
diff --git a/provider/storage-gc/docs/anthos/pics/client.png b/provider/storage-gc/docs/anthos/pics/client.png
deleted file mode 100644
index 8a2014e672b65ddf644558e2c9e8cb8823f549dd..0000000000000000000000000000000000000000
Binary files a/provider/storage-gc/docs/anthos/pics/client.png and /dev/null differ
diff --git a/provider/storage-gc/docs/anthos/pics/rabbit.PNG b/provider/storage-gc/docs/anthos/pics/rabbit.PNG
deleted file mode 100644
index e1c67734b91a536659fdf0b940a93208d749e3c8..0000000000000000000000000000000000000000
Binary files a/provider/storage-gc/docs/anthos/pics/rabbit.PNG and /dev/null differ
diff --git a/provider/storage-gc/docs/anthos/pics/sa.png b/provider/storage-gc/docs/anthos/pics/sa.png
deleted file mode 100644
index b9c8124047b5834a2cf79baecbf751dc106feeff..0000000000000000000000000000000000000000
Binary files a/provider/storage-gc/docs/anthos/pics/sa.png and /dev/null differ
diff --git a/provider/storage-gc/docs/gc/README.md b/provider/storage-gc/docs/gc/README.md
index ff7a0fcfa9191f6e0fc78a0c0b2fa798ad4335d1..df4b84901ea28eb819d4af67fb2a152142074e95 100644
--- a/provider/storage-gc/docs/gc/README.md
+++ b/provider/storage-gc/docs/gc/README.md
@@ -4,12 +4,14 @@
* [Environment variables](#Environment-variables)
* [Common properties for all environments](#Common-properties-for-all-environments)
-* [For Mappers to activate drivers](#For-Mappers-to-activate-drivers)
* [Datastore configuration](#Datastore-configuration)
-* [Pubsub configuration](#Pubsub-configuration)
+* [Pub/Sub configuration](#Pubsub-configuration)
* [Google Cloud service configuration](#ObjectStoreConfig)
-* [Google loud service account configuration](#Google-cloud-service-account-configuration)
+* [Google_Cloud service account configuration](#Google-cloud-service-account-configuration)
+* [Run args](#run-args)
+* [Running E2E Tests](#running-e2e-tests)
* [License](#License)
+*
## Environment variables
Define the following environment variables.
@@ -50,7 +52,6 @@ Defined in default application property file but possible to override:
| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
| `PARTITION_PROPERTIES_STORAGE_BUCKET_NAME` | ex `storage.bucket.name` | Name of partition property for storage bucket name value | yes | - |
-| `SYSTEM_PARTITION_ID` | ex `system` | System partition ID |
These variables define service behavior, and are used to switch between `Reference` or `Google Cloud` environments, their overriding and usage in mixed mode was not tested.
Usage of spring profiles is preferred.
@@ -58,22 +59,8 @@ Usage of spring profiles is preferred.
| name | value | description | sensitive? | source |
|--------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------------|------------|--------|
| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
-| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
-| `OSMDRIVER` | `datastore` or `postgres` | Osm driver mode that defines which KV storage will be used | no | - |
-| `OBMDRIVER` | `gcs` or `minio` | Obm driver mode that defines which object storage will be used | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` | Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
-### For Mappers to activate drivers
-
-| name | value | description |
-|-----------|-----------|---------------------------------------------------------|
-| OSMDRIVER | datastore | to activate **OSM** driver for **Google Datastore** |
-| OSMDRIVER | postgres | to activate **OSM** driver for **PostgreSQL** |
-| OBMDRIVER | gcs | to activate **OBM** driver for **Google Cloud Storage** |
-| OBMDRIVER | minio | to activate **OBM** driver for **MinIO** |
-| OQMDRIVER | pubsub | to activate **OQM** driver for **Google PubSub** |
-| OQMDRIVER | rabbitmq | to activate **OQM** driver for **Rabbit MQ** |
-
## Datastore configuration
There must be a namespace for each tenant, which is the same as the tenant name.
@@ -100,9 +87,9 @@ indexes:
```
-## PubSub configuration
+## Pub/Sub configuration
-At PubSub should be created set of topics and subscriptions.
+At Pub/Sub should be created set of topics and subscriptions.
| topic name | subscription name | description | sensitive? | env var to override |
|------------------------------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-------------------------------------------------------------------------|
@@ -154,6 +141,21 @@ TBD
|----------------|
| - |
+### Run args
+
+In order to run Legal with Java 17 additional run args must be provided:
+
+```bash
+--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED
+```
+
+```bash
+CMD java --add-opens java.base/java.lang=ALL-UNNAMED \
+ --add-opens java.base/java.lang.reflect=ALL-UNNAMED \
+ -Dloader.main=org.opengroup.osdu.storage.provider.gcp.StorageApplicationGCP \
+ -jar /app/secret-${PROVIDER_NAME}.jar
+```
+
### Running E2E Tests
You will need to have the following environment variables defined.
diff --git a/provider/storage-gc/maven/settings.xml b/provider/storage-gc/maven/settings.xml
deleted file mode 100644
index b7a15a01004e3079c32548b080a7c49d2861d9df..0000000000000000000000000000000000000000
--- a/provider/storage-gc/maven/settings.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
- <servers>
- <server>
- <id>os-core</id>
- <username>slb-des-ext-collaboration</username>
- <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
- <!-- The generated token expires on or before 11/14/2019 -->
- <password>${VSTS_FEED_TOKEN}</password>
- </server>
- </servers>
-</settings>
diff --git a/provider/storage-gc/pom.xml b/provider/storage-gc/pom.xml
index cf1c0e8c305688285bd1211876d7cc2a3ad70aac..2247071a076e19038028cc0d94e83d33ae563f48 100644
--- a/provider/storage-gc/pom.xml
+++ b/provider/storage-gc/pom.xml
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- ~ Copyright 2020-2023 Google LLC
- ~ Copyright 2020-2023 EPAM Systems, Inc
+ ~ Copyright 2020-2025 Google LLC
+ ~ Copyright 2020-2025 EPAM Systems, Inc
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -16,8 +16,8 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -28,7 +28,7 @@
</parent>
<artifactId>storage-gc</artifactId>
- <description>Google cloud related implementation staff.</description>
+ <description>Storage Service</description>
<packaging>jar</packaging>
<properties>
@@ -58,79 +58,15 @@
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
- <artifactId>core-lib-gc</artifactId>
- <version>0.27.0-rc8</version>
- </dependency>
- <!-- Mappers -->
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>osm</artifactId>
- <version>0.27.0-rc3</version>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>oqm</artifactId>
- <version>0.26.0</version>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>obm</artifactId>
- <version>0.27.2</version>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>storage-core</artifactId>
+ <artifactId>storage-core-plus</artifactId>
<version>0.28.0-SNAPSHOT</version>
- <exclusions>
- <exclusion>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- </exclusion>
- </exclusions>
</dependency>
+
<dependency>
<groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
+ <artifactId>logback-jackson</artifactId>
<version>0.1.5</version>
</dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-webmvc</artifactId>
- <version>${spring-webmvc.version}</version>
- </dependency>
-
- <dependency>
- <groupId>jakarta.annotation</groupId>
- <artifactId>jakarta.annotation-api</artifactId>
- <version>2.1.1</version>
- </dependency>
-
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-test</artifactId>
- <scope>test</scope>
- </dependency>
- <!-- Testing packages -->
- <dependency>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
- <version>5.10.2</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-actuator</artifactId>
- </dependency>
-
</dependencies>
<build>
@@ -139,7 +75,7 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
- <version>3.0.0</version>
+ <version>3.2.2</version>
<configuration>
<profiles>
<profile>
@@ -151,12 +87,6 @@
<spring.profiles.active>local</spring.profiles.active>
</properties>
</profile>
- <profile>
- <id>dev</id>
- <properties>
- <spring.profiles.active>dev</spring.profiles.active>
- </properties>
- </profile>
</profiles>
</configuration>
<executions>
@@ -167,20 +97,13 @@
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
- org.opengroup.osdu.storage.provider.gcp.StorageApplicationGCP
+ org.springframework.boot.loader.launch.PropertiesLauncher
</mainClass>
</configuration>
</execution>
</executions>
</plugin>
- <plugin>
- <artifactId>maven-war-plugin</artifactId>
- <configuration>
- <failOnMissingWebXml>false</failOnMissingWebXml>
- </configuration>
- </plugin>
-
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
@@ -203,4 +126,4 @@
</plugins>
</build>
-</project>
+</project>
\ No newline at end of file
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/StorageApplicationGCP.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/StorageApplicationGCP.java
index a3b482578ebd5b35af0f9d6450db3c65f6186565..3d556cbaa3f72ff34281eaea125ca6021ad185f9 100644
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/StorageApplicationGCP.java
+++ b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/StorageApplicationGCP.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2025 Google LLC
+ * Copyright 2020-2025 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/config/MessagingConfigurationProperties.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/logging/formatter/GoogleJsonFormatter.java
similarity index 51%
rename from provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/config/MessagingConfigurationProperties.java
rename to provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/logging/formatter/GoogleJsonFormatter.java
index 1c61908b4411757c40e8941a0a60acaf3473f4a5..d88fe08a948766c43b0acad34caa2b5ac29447f6 100644
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/config/MessagingConfigurationProperties.java
+++ b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/logging/formatter/GoogleJsonFormatter.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2025 Google LLC
+ * Copyright 2020-2025 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,19 +15,16 @@
* limitations under the License.
*/
-package org.opengroup.osdu.storage.provider.gcp.messaging.config;
+package org.opengroup.osdu.storage.provider.gcp.logging.formatter;
-import lombok.Data;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@ConfigurationProperties
-@Data
-public class MessagingConfigurationProperties {
-
- private String legalTagsChangedTopicName;
- private String legalTagsChangedSubscriptionName;
- private String storageServiceAccountEmail;
+import ch.qos.logback.contrib.jackson.JacksonJsonFormatter;
+import java.io.IOException;
+import java.util.Map;
+public class GoogleJsonFormatter extends JacksonJsonFormatter {
+ @Override
+ public String toJsonString(Map map) throws IOException {
+ map.put("severity", map.remove("level"));
+ return super.toJsonString(map);
+ }
}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/config/MessagingCustomContextConfiguration.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/config/MessagingCustomContextConfiguration.java
deleted file mode 100644
index 97bd8b11e726d032b9173dc3888a0b3c8bce0a93..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/config/MessagingCustomContextConfiguration.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.config;
-
-import java.util.Arrays;
-import jakarta.annotation.PostConstruct;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.storage.StorageApplication;
-import org.opengroup.osdu.storage.provider.gcp.web.config.WebAppMainContextConfiguration;
-import org.opengroup.osdu.storage.swagger.SwaggerConfiguration;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.PropertySource;
-
-/**
- * This context configuration serves for message receiving configuration not bounded to request scope.
- * Also, configured context does not serve incoming requests.
- */
-@Slf4j
-@Configuration
-@EnableConfigurationProperties
-@PropertySource("classpath:application.properties")
-@RequiredArgsConstructor
-@ComponentScan(value = {
- "org.opengroup.osdu"
-},
- excludeFilters = {
- @ComponentScan.Filter(
- type = FilterType.ASSIGNABLE_TYPE,
- value = {
- StorageApplication.class,
- WebAppMainContextConfiguration.class,
- SwaggerConfiguration.class
- }
- )
- }
-)
-public class MessagingCustomContextConfiguration {
-
- private final ApplicationContext applicationContext;
-
- @PostConstruct
- public void setUp() {
- log.debug("Messaging context initialized with id: {}.", applicationContext.getId());
- log.debug("Messaging context status: {}.", applicationContext);
- String[] allBeansNames = applicationContext.getBeanDefinitionNames();
- log.debug("Messaging context beans definitions: {}.", Arrays.toString(allBeansNames));
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/LegalComplianceChangeServiceGcpImpl.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/LegalComplianceChangeServiceGcpImpl.java
deleted file mode 100644
index 9b6bfaa6b50eac05a07ab66a917f5264901d7acb..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/LegalComplianceChangeServiceGcpImpl.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.jobs;
-
-import static java.util.Collections.singletonList;
-
-import java.util.AbstractMap;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import lombok.RequiredArgsConstructor;
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
-import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceChangeInfo;
-import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceUpdateStoppedException;
-import org.opengroup.osdu.core.common.model.legal.jobs.ILegalComplianceChangeService;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChanged;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChangedCollection;
-import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.RecordState;
-import org.opengroup.osdu.storage.logging.StorageAuditLogger;
-import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
-import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
-import org.springframework.stereotype.Service;
-
-@Service
-@RequiredArgsConstructor
-public class LegalComplianceChangeServiceGcpImpl implements ILegalComplianceChangeService {
-
- private final IRecordsMetadataRepository recordsRepo;
- private final IMessageBus messageBus;
- private final StorageAuditLogger auditLogger;
- private final JaxRsDpsLog logger;
- // not conventional field name due to bean qualifiers in a core module
- private final ICache<String, String> LegalTagCache;
-
- private final long maxRunningTimeMills = 115000;
-
- @Override
- public Map<String, LegalCompliance> updateComplianceOnRecords(LegalTagChangedCollection legalTagsChanged,
- DpsHeaders headers) throws ComplianceUpdateStoppedException {
- Map<String, LegalCompliance> output = new HashMap<>();
- long currentTimeMills;
- long start = System.currentTimeMillis();
-
- for (LegalTagChanged lt : legalTagsChanged.getStatusChangedTags()) {
-
- ComplianceChangeInfo complianceChangeInfo = this.getComplianceChangeInfo(lt);
- if (complianceChangeInfo == null) {
- continue;
- }
- AbstractMap.SimpleEntry<String, List<RecordMetadata>> results = this.recordsRepo
- .queryByLegal(lt.getChangedTagName(), complianceChangeInfo.getCurrent(), 500);
-
- while (results.getValue() != null && !results.getValue().isEmpty()) {
- currentTimeMills = System.currentTimeMillis() - start;
- if (currentTimeMills >= maxRunningTimeMills) {
- throw new ComplianceUpdateStoppedException(currentTimeMills / 1000);
- }
- List<RecordMetadata> recordsMetadata = results.getValue();
- PubSubInfo[] pubsubInfos = this.updateComplianceStatus(complianceChangeInfo, recordsMetadata, output);
- StringBuilder recordsId = new StringBuilder();
- for (RecordMetadata recordMetadata : recordsMetadata) {
- recordsId.append(", ").append(recordMetadata.getId());
- }
- this.recordsRepo.createOrUpdate(recordsMetadata, Optional.empty());
- this.messageBus.publishMessage(headers, pubsubInfos);
- this.auditLogger.updateRecordsComplianceStateSuccess(
- singletonList("[" + recordsId.substring(2) + "]"));
- results = this.recordsRepo.queryByLegal(lt.getChangedTagName(), complianceChangeInfo.getCurrent(), 500);
- }
- }
-
- return output;
- }
-
- private PubSubInfo[] updateComplianceStatus(ComplianceChangeInfo complianceChangeInfo,
- List<RecordMetadata> recordMetadata, Map<String, LegalCompliance> output) {
-
- PubSubInfo[] pubsubInfo = new PubSubInfo[recordMetadata.size()];
-
- int i = 0;
- for (RecordMetadata rm : recordMetadata) {
- rm.getLegal().setStatus(complianceChangeInfo.getNewState());
- rm.setStatus(complianceChangeInfo.getNewRecordState());
- pubsubInfo[i] = new PubSubInfo(rm.getId(), rm.getKind(), complianceChangeInfo.getPubSubEvent());
- output.put(rm.getId(), complianceChangeInfo.getNewState());
- i++;
- }
-
- return pubsubInfo;
- }
-
- private ComplianceChangeInfo getComplianceChangeInfo(LegalTagChanged lt) {
- ComplianceChangeInfo output = null;
-
- if (lt.getChangedTagStatus().equalsIgnoreCase("compliant")) {
- output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.update, RecordState.active);
- } else if (lt.getChangedTagStatus().equalsIgnoreCase("incompliant")) {
- this.LegalTagCache.delete(lt.getChangedTagName());
- output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, RecordState.deleted);
- } else {
- this.logger.warning(String.format("Unknown LegalTag compliance status received %s %s",
- lt.getChangedTagStatus(), lt.getChangedTagName()));
- }
-
- return output;
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/LegalTagChangedProcessing.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/LegalTagChangedProcessing.java
deleted file mode 100644
index d84c4cff22bd49ef0fd901671c1c9714e773338c..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/LegalTagChangedProcessing.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.jobs;
-
-import com.google.gson.Gson;
-import java.util.Map;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
-import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceUpdateStoppedException;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChangedCollection;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagConsistencyValidator;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.storage.provider.gcp.messaging.scope.override.ThreadDpsHeaders;
-
-@Slf4j
-@RequiredArgsConstructor
-public class LegalTagChangedProcessing {
-
- private final LegalTagConsistencyValidator legalTagConsistencyValidator;
- private final LegalComplianceChangeServiceGcpImpl legalComplianceChangeServiceGcp;
- private final ThreadDpsHeaders dpsHeaders;
-
- public void process(OqmMessage oqmMessage) throws ComplianceUpdateStoppedException {
- String pubsubMessage = oqmMessage.getData();
- LegalTagChangedCollection dto = (new Gson()).fromJson(pubsubMessage, LegalTagChangedCollection.class);
-
- LegalTagChangedCollection validDto = this.legalTagConsistencyValidator.checkLegalTagStatusWithLegalService(dto);
- log.debug("LegalTags changed status validation via Legal service: {}.", validDto);
- Map<String, LegalCompliance> stringLegalComplianceMap = this.legalComplianceChangeServiceGcp.updateComplianceOnRecords(validDto, dpsHeaders);
- log.debug("Updated compliance on records: {}.", stringLegalComplianceMap);
-
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/OqmSubscriberManager.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/OqmSubscriberManager.java
deleted file mode 100644
index ca331684c335a4457226130b7aa2cfe1c3c0dc71..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/OqmSubscriberManager.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.jobs;
-
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
-import java.util.Map;
-import jakarta.annotation.PostConstruct;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagConsistencyValidator;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessageReceiver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriber;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscription;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriptionQuery;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
-import org.opengroup.osdu.storage.provider.gcp.messaging.config.MessagingConfigurationProperties;
-import org.opengroup.osdu.storage.provider.gcp.messaging.scope.override.ThreadDpsHeaders;
-import org.opengroup.osdu.storage.provider.gcp.messaging.thread.ThreadScopeContextHolder;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.context.annotation.Scope;
-import org.springframework.http.HttpStatus;
-import org.springframework.stereotype.Component;
-
-/**
- * Runs once on the service start. Fetches all tenants' oqm destinations for TOPIC existence. If exists - searches for pull SUBSCRIPTION existence. Creates
- * SUBSCRIPTION if doesn't exist. Then subscribe itself on SUBSCRIPTION.
- */
-
-@Slf4j
-@Component
-@Scope(SCOPE_SINGLETON)
-@ConditionalOnProperty(name = "oqmDriver")
-@RequiredArgsConstructor
-public class OqmSubscriberManager {
-
- private final MessagingConfigurationProperties configurationProperties;
-
- private final ITenantFactory tenantInfoFactory;
- private final OqmDriver driver;
-
- private final LegalTagConsistencyValidator legalTagConsistencyValidator;
- private final LegalComplianceChangeServiceGcpImpl legalComplianceChangeServiceGcp;
- private final ThreadDpsHeaders dpsHeaders;
-
- @PostConstruct
- void postConstruct() {
- log.debug("OqmSubscriberManager bean constructed. Provisioning STARTED.");
-
- //Get all Tenant infos
- for (TenantInfo tenantInfo : tenantInfoFactory.listTenantInfo()) {
- String dataPartitionId = tenantInfo.getDataPartitionId();
- String tagsChangedTopicName = configurationProperties.getLegalTagsChangedTopicName();
- log.debug("* OqmSubscriberManager on provisioning tenant {}:", dataPartitionId);
-
- log.debug("* * OqmSubscriberManager on check for topic {} existence:",
- tagsChangedTopicName);
- OqmTopic topic = driver.getTopic(tagsChangedTopicName, getDestination(tenantInfo)).orElse(null);
- if (topic == null) {
- log.error("* * OqmSubscriberManager on check for topic {} existence: ABSENT.",
- tagsChangedTopicName);
- throw new AppException(
- HttpStatus.INTERNAL_SERVER_ERROR.value(),
- "Required topic not exists.",
- String.format(
- "Required topic not exists. Create topic: %s for tenant: %s and restart service.",
- tagsChangedTopicName, dataPartitionId
- )
- );
- }
-
- String legalTagsChangedSubscriptionName = configurationProperties.getLegalTagsChangedSubscriptionName();
-
- log.debug("* * OqmSubscriberManager on check for subscription {} existence:", legalTagsChangedSubscriptionName);
-
- OqmSubscriptionQuery query = OqmSubscriptionQuery.builder()
- .namePrefix(legalTagsChangedSubscriptionName)
- .subscriberable(true)
- .build();
-
- OqmSubscription subscription = driver.listSubscriptions(topic, query, getDestination(tenantInfo)).stream().findAny().orElse(null);
-
- if (subscription == null) {
- log.error("* * OqmSubscriberManager on check for subscription {} existence: ABSENT. Will create.", legalTagsChangedSubscriptionName);
- throw new AppException(
- HttpStatus.INTERNAL_SERVER_ERROR.value(),
- "Required subscription not exists.",
- String.format(
- "Required subscription not exists. Create subscription: %s for tenant: %s and restart service.",
- legalTagsChangedSubscriptionName,
- dataPartitionId
- )
- );
- } else {
- log.debug("* * OqmSubscriberManager on check for subscription {} existence: PRESENT.", legalTagsChangedSubscriptionName);
- }
-
- log.debug("* * OqmSubscriberManager on registering Subscriber for tenant {}, subscription {}",
- dataPartitionId,
- legalTagsChangedSubscriptionName);
- registerSubscriber(tenantInfo, subscription);
- log.debug("* * OqmSubscriberManager on provisioning for tenant {}, subscription {}: Subscriber REGISTERED.",
- dataPartitionId,
- subscription.getName());
-
- log.debug("* OqmSubscriberManager on provisioning tenant {}: COMPLETED.",
- dataPartitionId);
- }
-
- log.debug("OqmSubscriberManager bean constructed. Provisioning COMPLETED.");
- }
-
- private void registerSubscriber(TenantInfo tenantInfo, OqmSubscription subscription) {
- OqmDestination destination = getDestination(tenantInfo);
-
- OqmMessageReceiver receiver = (oqmMessage, oqmAckReplier) -> {
-
- String pubsubMessage = oqmMessage.getData();
- Map<String, String> headerAttributes = oqmMessage.getAttributes();
- log.debug(pubsubMessage + " " + headerAttributes + " " + oqmMessage.getId());
-
- boolean ackedNacked = false;
- try {
- dpsHeaders.setThreadContext(headerAttributes);
- LegalTagChangedProcessing legalTagChangedProcessing =
- new LegalTagChangedProcessing(legalTagConsistencyValidator, legalComplianceChangeServiceGcp, dpsHeaders);
- legalTagChangedProcessing.process(oqmMessage);
- log.debug("OQM message handling for tenant {} topic {} subscription {}. ACK. Message: -data: {}, attributes: {}.",
- dpsHeaders.getPartitionId(),
- configurationProperties.getLegalTagsChangedTopicName(),
- configurationProperties.getLegalTagsChangedSubscriptionName(),
- pubsubMessage,
- StringUtils.join(headerAttributes)
- );
- oqmAckReplier.ack();
- ackedNacked = true;
- } catch (Exception e) {
- log.error("OQM message handling error for tenant {} topic {} subscription {}. Message: -data: {}, attributes: {}, error: {}.",
- dpsHeaders.getPartitionId(),
- configurationProperties.getLegalTagsChangedTopicName(),
- configurationProperties.getLegalTagsChangedSubscriptionName(),
- pubsubMessage,
- StringUtils.join(headerAttributes),
- e
- );
- } finally {
- if (!ackedNacked) {
- oqmAckReplier.nack();
- }
- ThreadScopeContextHolder.currentThreadScopeAttributes().clear();
- }
- };
-
- OqmSubscriber subscriber = OqmSubscriber.builder().subscription(subscription).messageReceiver(receiver).build();
- driver.subscribe(subscriber, destination);
- log.debug("Just subscribed at topic {} subscription {} for tenant {}.",
- subscription.getTopics().get(0).getName(), subscription.getName(), tenantInfo.getDataPartitionId());
- }
-
- private OqmDestination getDestination(TenantInfo tenantInfo) {
- return OqmDestination.builder().partitionId(tenantInfo.getDataPartitionId()).build();
- }
-
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/scope/override/ScopeModifierPostProcessor.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/scope/override/ScopeModifierPostProcessor.java
deleted file mode 100644
index 0c0656ae27b75476bb47fed6419f49442f1cb087..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/scope/override/ScopeModifierPostProcessor.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.scope.override;
-
-import java.util.Objects;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.storage.provider.gcp.messaging.thread.ThreadScope;
-import org.springframework.beans.BeansException;
-import org.springframework.beans.factory.config.BeanDefinition;
-import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
-import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class ScopeModifierPostProcessor implements BeanFactoryPostProcessor {
-
- public static final String SCOPE_THREAD = "scope_thread";
-
- @Override
- public void postProcessBeanFactory(ConfigurableListableBeanFactory factory) throws BeansException {
- factory.registerScope(SCOPE_THREAD, new ThreadScope());
-
- for (String beanName : factory.getBeanDefinitionNames()) {
- BeanDefinition beanDef = factory.getBeanDefinition(beanName);
- if (Objects.equals(beanDef.getScope(), "request")) {
- beanDef.setScope(SCOPE_THREAD);
- log.debug("Scope has been overridden for bean: {}", beanDef.getBeanClassName());
- }
- }
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/scope/override/ThreadDpsHeaders.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/scope/override/ThreadDpsHeaders.java
deleted file mode 100644
index ad212673e6cded2d0d8c416b11cfdc53577ab848..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/scope/override/ThreadDpsHeaders.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.scope.override;
-
-import static org.springframework.context.annotation.ScopedProxyMode.TARGET_CLASS;
-
-import java.util.Map;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.storage.provider.gcp.messaging.config.MessagingConfigurationProperties;
-import org.springframework.context.annotation.Primary;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-/**
- * Original class bean configuration bounded to request scope, extend purpose is to unbound it. Due to OQM specific, we cannot rely on events transferred via
- * HTTP requests, only pull subscriptions works with OQM. But this bean is configured only for Messaging context, original bean keeps working as usual for the
- * web app context.
- */
-
-@Slf4j
-@Primary
-@Component
-@Scope(value = ScopeModifierPostProcessor.SCOPE_THREAD, proxyMode = TARGET_CLASS)
-@RequiredArgsConstructor
-public class ThreadDpsHeaders extends DpsHeaders {
-
- private final MessagingConfigurationProperties properties;
- private final TokenProvider tokenProvider;
-
- public void setThreadContext(Map<String, String> headers) {
- this.put(DpsHeaders.AUTHORIZATION, "Bearer " + tokenProvider.getIdToken());
- this.put(DpsHeaders.USER_EMAIL, properties.getStorageServiceAccountEmail());
- this.addFromMap(headers);
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScope.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScope.java
deleted file mode 100644
index 91c87ee7b4cb54d603b03ed3ba1d58b121c60d34..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScope.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.thread;
-
-import java.util.Map;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.ObjectFactory;
-import org.springframework.beans.factory.config.Scope;
-
-@Slf4j
-public class ThreadScope implements Scope {
-
- public Object get(String name, ObjectFactory<?> factory) {
- log.trace("Get bean:{} with factory: {} current Thread: {}", name, factory, Thread.currentThread().getName());
- Object result = null;
- Map<String, Object> hBeans = ThreadScopeContextHolder.currentThreadScopeAttributes().getBeanMap();
- if (!hBeans.containsKey(name)) {
- result = factory.getObject();
- log.trace("No bean in context with name: {} factory provisioning result is: {} current Thread: {}", name, result, Thread.currentThread().getName());
- hBeans.put(name, result);
- } else {
- result = hBeans.get(name);
- }
-
- return result;
- }
-
- public Object remove(String name) {
- log.trace("Removing bean : {} current Thread: {}", name, Thread.currentThread().getName());
- Object result = null;
- Map<String, Object> hBeans = ThreadScopeContextHolder.currentThreadScopeAttributes().getBeanMap();
- if (hBeans.containsKey(name)) {
- result = hBeans.get(name);
- hBeans.remove(name);
- }
-
- return result;
- }
-
- public void registerDestructionCallback(String name, Runnable callback) {
- ThreadScopeContextHolder.currentThreadScopeAttributes().registerRequestDestructionCallback(name, callback);
- }
-
- public Object resolveContextualObject(String key) {
- return null;
- }
-
- public String getConversationId() {
- return Thread.currentThread().getName();
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScopeAttributes.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScopeAttributes.java
deleted file mode 100644
index 42b8ca4dfb1b93aaf8dd34eecc1805830faa0e35..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScopeAttributes.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.thread;
-
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.lang.NonNull;
-
-@Slf4j
-@RequiredArgsConstructor
-public class ThreadScopeAttributes {
-
- protected final Map<String, Object> hBeans = new HashMap();
- protected final Map<String, Runnable> hRequestDestructionCallbacks = new LinkedHashMap();
-
- protected final Map<String, Object> getBeanMap() {
- return this.hBeans;
- }
-
- protected final void registerRequestDestructionCallback(@NonNull String name, @NonNull Runnable callback) {
- log.trace("Registering callback for: {} on runnable: {}", name, callback);
- this.hRequestDestructionCallbacks.put(name, callback);
- }
-
- public final void clear() {
- this.processDestructionCallbacks();
- this.hBeans.clear();
- }
-
- private void processDestructionCallbacks() {
- for (Map.Entry<String, Runnable> mapEntry : this.hRequestDestructionCallbacks.entrySet()) {
- Runnable callback = mapEntry.getValue();
- log.trace("Performing destruction callback for: {} on thread: {}", mapEntry.getKey(), Thread.currentThread().getName());
- callback.run();
- }
- this.hRequestDestructionCallbacks.clear();
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScopeContextHolder.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScopeContextHolder.java
deleted file mode 100644
index cd90169491e493e1d0b11929a4b1cb8aaa5f6b74..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/messaging/thread/ThreadScopeContextHolder.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.thread;
-
-public class ThreadScopeContextHolder {
-
- private static final ThreadLocal<ThreadScopeAttributes> threadScopeAttributesHolder = new InheritableThreadLocal<ThreadScopeAttributes>() {
- @Override
- protected ThreadScopeAttributes initialValue() {
- return new ThreadScopeAttributes();
- }
- };
-
- private ThreadScopeContextHolder() {
- }
-
- public static ThreadScopeAttributes getThreadScopeAttributes() {
- return threadScopeAttributesHolder.get();
- }
-
- public static void setThreadScopeAttributes(ThreadScopeAttributes accessor) {
- threadScopeAttributesHolder.set(accessor);
- }
-
- public static ThreadScopeAttributes currentThreadScopeAttributes() throws IllegalStateException {
- ThreadScopeAttributes accessor = threadScopeAttributesHolder.get();
- if (accessor == null) {
- throw new IllegalStateException("No thread scoped attributes.");
- } else {
- return accessor;
- }
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java
deleted file mode 100644
index 5115bbd5d7463ba3b32e6388148a3b5a428ae8b6..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.cache;
-
-import lombok.RequiredArgsConstructor;
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.common.model.entitlements.Groups;
-import org.opengroup.osdu.core.common.model.storage.Schema;
-import org.opengroup.osdu.core.common.partition.PartitionInfo;
-import org.opengroup.osdu.core.gcp.cache.RedisCacheBuilder;
-import org.opengroup.osdu.storage.provider.gcp.web.config.GcpAppServiceConfig;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@RequiredArgsConstructor
-public class CacheConfig {
-
- private final RedisCacheBuilder<String, String> legalRedisCacheBuilder;
- private final RedisCacheBuilder<String, Schema> schemaRedisCacheBuilder;
- private final RedisCacheBuilder<String, Groups> groupsRedisCacheBuilder;
-
- @Bean("LegalTagCache")
- public ICache<String, String> legalTagCache(GcpAppServiceConfig gcpAppServiceConfig) {
- RedisCache<String, String> storageCache = legalRedisCacheBuilder.buildRedisCache(
- gcpAppServiceConfig.getRedisStorageHost(),
- gcpAppServiceConfig.getRedisStoragePort(),
- gcpAppServiceConfig.getRedisStoragePassword(),
- gcpAppServiceConfig.getRedisStorageExpiration(),
- gcpAppServiceConfig.getRedisStorageWithSsl(),
- String.class,
- String.class
- );
- return new LegalTagMultiTenantCache(storageCache);
- }
-
- @Bean
- public RedisCache<String, Schema> schemaCache(GcpAppServiceConfig gcpAppServiceConfig) {
- return schemaRedisCacheBuilder.buildRedisCache(
- gcpAppServiceConfig.getRedisStorageHost(),
- gcpAppServiceConfig.getRedisStoragePort(),
- gcpAppServiceConfig.getRedisStoragePassword(),
- gcpAppServiceConfig.getRedisStorageExpiration(),
- gcpAppServiceConfig.getRedisStorageWithSsl(),
- String.class,
- Schema.class
- );
- }
-
- @Bean
- public RedisCache<String, Groups> groupsCache(GcpAppServiceConfig gcpAppServiceConfig){
- return groupsRedisCacheBuilder.buildRedisCache(
- gcpAppServiceConfig.getRedisGroupHost(),
- gcpAppServiceConfig.getRedisGroupPort(),
- gcpAppServiceConfig.getRedisGroupPassword(),
- gcpAppServiceConfig.getRedisGroupExpiration(),
- gcpAppServiceConfig.getRedisGroupWithSsl(),
- String.class,
- Groups.class
- );
- }
-
- @Bean
- public ICache<String, PartitionInfo> partitionInfoCache() {
- return new VmCache<>(600, 2000);
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/LegalTagMultiTenantCache.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/LegalTagMultiTenantCache.java
deleted file mode 100644
index 961f411265828b19c1f04199d319ac5b7388745c..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/LegalTagMultiTenantCache.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.cache;
-
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.cache.MultiTenantCache;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.springframework.beans.factory.annotation.Autowired;
-
-public class LegalTagMultiTenantCache implements ICache<String, String> {
-
- @Autowired
- private TenantInfo tenant;
-
- private final MultiTenantCache<String> caches;
-
- public LegalTagMultiTenantCache(RedisCache<String, String> redisCache) {
- this.caches = new MultiTenantCache<>(redisCache);
- }
-
- @Override
- public void put(String key, String val) {
- this.partitionCache().put(key, val);
- }
-
- @Override
- public String get(String key) {
- return this.partitionCache().get(key);
- }
-
- @Override
- public void delete(String key) {
- this.partitionCache().delete(key);
- }
-
- @Override
- public void clearAll() {
- this.partitionCache().clearAll();
- }
-
- private ICache<String, String> partitionCache() {
- return this.caches.get(String.format("%s:legalTag", this.tenant));
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java
deleted file mode 100644
index 40f80770e08b9d77fd043e80d7a0fe341c661ce2..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.config;
-
-import lombok.Data;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@ConfigurationProperties
-@Data
-public class GcpAppServiceConfig {
-
- private String pubsubSearchTopic;
-
- private String redisStorageHost;
- private Integer redisStoragePort;
- private String redisStoragePassword;
- private Integer redisStorageExpiration = 60 * 60;
- private Boolean redisStorageWithSsl = false;
-
- private String redisGroupHost;
- private Integer redisGroupPort;
- private String redisGroupPassword;
- private Integer redisGroupExpiration = 30;
- private Boolean redisGroupWithSsl = false;
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/PartitionPropertyNames.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/PartitionPropertyNames.java
deleted file mode 100644
index 3038ac7a78d391bb170702bec9849aaadae4120d..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/PartitionPropertyNames.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.config;
-
-import lombok.Data;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Data
-@Configuration
-@ConfigurationProperties(prefix = "partition.properties")
-public class PartitionPropertyNames {
- private String storageBucketName;
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/WebAppMainContextConfiguration.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/WebAppMainContextConfiguration.java
deleted file mode 100644
index 577f64ff0728dc506624e9aacb22d3f5990d2fb0..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/WebAppMainContextConfiguration.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.config;
-
-import java.util.Arrays;
-import jakarta.annotation.PostConstruct;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.storage.StorageApplication;
-import org.opengroup.osdu.storage.provider.gcp.messaging.config.MessagingCustomContextConfiguration;
-import org.opengroup.osdu.storage.provider.gcp.messaging.jobs.OqmSubscriberManager;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.PropertySource;
-
-/**
- * This context configuration excludes pulling message receiving and thread scope configuration.
- */
-@Slf4j
-@Configuration
-@EnableAutoConfiguration
-@PropertySource("classpath:application.properties")
-@ComponentScan(
- value = {"org.opengroup.osdu"},
- excludeFilters = {
- @ComponentScan.Filter(
- type = FilterType.ASSIGNABLE_TYPE,
- value = {
- StorageApplication.class
- }
- ),
- @ComponentScan.Filter(
- type = FilterType.REGEX,
- pattern = {"org\\.opengroup\\.osdu\\.storage\\.provider\\.gcp\\.messaging\\..*"}
- )
- }
-)
-public class WebAppMainContextConfiguration {
-
- @Autowired
- private ApplicationContext applicationContext;
-
- @PostConstruct
- public void setUp() {
- log.debug("Main web app context initialized with id: {}.", applicationContext.getId());
- log.debug("Main web app context status: {}.", applicationContext);
- String[] allBeansNames = applicationContext.getBeanDefinitionNames();
- log.debug("Main web app context beans definitions: {}.", Arrays.toString(allBeansNames));
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/mappers/osm/OsmTypeMapperImpl.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/mappers/osm/OsmTypeMapperImpl.java
deleted file mode 100644
index ca85277791fd84a9b5883e8299ce9d31021555c1..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/mappers/osm/OsmTypeMapperImpl.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.opengroup.osdu.storage.provider.gcp.web.mappers.osm;
-
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
-import com.google.cloud.Timestamp;
-import com.google.cloud.datastore.Blob;
-import com.google.cloud.datastore.Key;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.Schema;
-import org.opengroup.osdu.core.gcp.osm.persistence.IdentityTranslator;
-import org.opengroup.osdu.core.gcp.osm.translate.Instrumentation;
-import org.opengroup.osdu.core.gcp.osm.translate.TypeMapper;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-/**
- * All Entity classes used in translation should be registered here even if don't need custom settings. Each class is represented as Instrumentation object. At
- * least class objects and rules for Identity translation should be provided.oi
- */
-@Component
-@Scope(SCOPE_SINGLETON)
-@ConditionalOnProperty(name = "osmDriver")
-public class OsmTypeMapperImpl extends TypeMapper {
-
- public OsmTypeMapperImpl() {
- super(Arrays.asList(
- new Instrumentation<>(RecordMetadata.class,
- new HashMap<String, String>() {{
- put("user", "createUser");
- put("gcsVersionPaths", "bucket");
- }},
- new HashMap<String, Class<?>>() {{
- put("createTime", Timestamp.class);
- put("modifyTime", Timestamp.class);
- }},
- new IdentityTranslator<>(
- RecordMetadata::getId,
- (r, o) -> r.setId(((Key) o).getName())
- ),
- Collections.singletonList("id")
- ),
- new Instrumentation<>(Schema.class,
- new HashMap<String, String>() {{
- put("ext", "extension");
- }},
- new HashMap<String, Class<?>>() {{
- put("schema", Blob.class);
- put("ext", Blob.class);
- }},
- new IdentityTranslator<>(
- Schema::getKind,
- (r, o) -> r.setKind(((Key) o).getName())
- ),
- Collections.singletonList("kind")
- )
- ));
- }
-
-
-}
\ No newline at end of file
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/middleware/GcpExceptionMapper.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/middleware/GcpExceptionMapper.java
deleted file mode 100644
index 3b580843c7b05c5f984bd5b275cd7bc8731b152a..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/middleware/GcpExceptionMapper.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.middleware;
-
-import com.google.cloud.storage.StorageException;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.storage.util.GlobalExceptionMapper;
-import org.springframework.core.Ordered;
-import org.springframework.core.annotation.Order;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.ControllerAdvice;
-import org.springframework.web.bind.annotation.ExceptionHandler;
-
-@Order(Ordered.HIGHEST_PRECEDENCE + 1)
-@ControllerAdvice
-public class GcpExceptionMapper {
-
- public static final String ACCESS_DENIED_REASON = "Access denied";
- public static final String ACCESS_DENIED_MESSAGE = "The user is not authorized to perform this action";
-
- private final GlobalExceptionMapper mapper;
-
- public GcpExceptionMapper(GlobalExceptionMapper mapper) {
- this.mapper = mapper;
- }
-
- @ExceptionHandler(StorageException.class)
- protected ResponseEntity<Object> handleStorageException(StorageException e) {
- return mapper.getErrorResponse(
- new AppException(HttpStatus.FORBIDDEN.value(), ACCESS_DENIED_REASON, ACCESS_DENIED_MESSAGE, e));
- }
-
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/pubsub/OqmPubSub.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/pubsub/OqmPubSub.java
deleted file mode 100644
index 687d239fa5d9ce91b2b2a6228f1d1be519e3278e..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/pubsub/OqmPubSub.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.pubsub;
-
-import com.google.gson.Gson;
-import lombok.RequiredArgsConstructor;
-import org.apache.commons.lang3.NotImplementedException;
-import org.opengroup.osdu.core.common.model.http.CollaborationContext;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
-import org.opengroup.osdu.storage.model.RecordChangedV2;
-import org.opengroup.osdu.storage.provider.gcp.web.config.GcpAppServiceConfig;
-import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
-import org.springframework.stereotype.Repository;
-
-import jakarta.annotation.PostConstruct;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-@Repository
-@RequiredArgsConstructor
-public class OqmPubSub implements IMessageBus {
-
- private final GcpAppServiceConfig config;
- private final OqmDriver driver;
- private final TenantInfo tenant;
- private final int BATCH_SIZE = 50;
-
- private OqmTopic oqmTopic = null;
-
- @PostConstruct
- void postConstruct() {
- oqmTopic = OqmTopic.builder().name(config.getPubsubSearchTopic()).build();
- }
-
- @Override
- public void publishMessage(DpsHeaders headers, PubSubInfo... messages) {
-
- OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId()).build();
-
- for (int i = 0; i < messages.length; i += BATCH_SIZE) {
-
- PubSubInfo[] batch = Arrays.copyOfRange(messages, i, Math.min(messages.length, i + BATCH_SIZE));
-
- String json = new Gson().toJson(batch);
-
- Map<String, String> attributes = new HashMap<>();
- attributes.put(DpsHeaders.USER_EMAIL, headers.getUserEmail());
- attributes.put(DpsHeaders.ACCOUNT_ID, this.tenant.getName());
- attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- headers.addCorrelationIdIfMissing();
- attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
-
- OqmMessage oqmMessage = OqmMessage.builder().data(json).attributes(attributes).build();
-
- driver.publish(oqmMessage, oqmTopic, oqmDestination);
- }
- }
-
- @Override
- public void publishMessage(Optional<CollaborationContext> collaborationContext, DpsHeaders headers, RecordChangedV2... messages) {
- throw new NotImplementedException();
- }
-
- @Override
- public void publishMessage(DpsHeaders headers, Map<String, String> routingInfo, List<?> messageList) {
- throw new NotImplementedException();
- }
-
- @Override
- public void publishMessage(DpsHeaders headers, Map<String, String> routingInfo, PubSubInfo... messages) {
- throw new NotImplementedException();
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java
deleted file mode 100644
index b664cf98819638f7111b4d01a1a3a30b4b4404b7..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java
+++ /dev/null
@@ -1,439 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.repository;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.gson.Gson;
-import lombok.RequiredArgsConstructor;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.model.entitlements.GroupInfo;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.CollaborationContext;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.storage.RecordData;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.RecordProcessing;
-import org.opengroup.osdu.core.common.model.storage.RecordState;
-import org.opengroup.osdu.core.common.model.storage.TransferInfo;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.partition.PartitionPropertyResolver;
-import org.opengroup.osdu.core.gcp.obm.driver.Driver;
-import org.opengroup.osdu.core.gcp.obm.driver.ObmDriverRuntimeException;
-import org.opengroup.osdu.core.gcp.obm.driver.S3CompatibleErrors;
-import org.opengroup.osdu.core.gcp.obm.model.Blob;
-import org.opengroup.osdu.core.gcp.obm.persistence.ObmDestination;
-import org.opengroup.osdu.storage.provider.gcp.web.config.PartitionPropertyNames;
-import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage;
-import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
-import org.opengroup.osdu.storage.service.DataAuthorizationService;
-import org.opengroup.osdu.storage.service.IEntitlementsExtensionService;
-import org.springframework.http.MediaType;
-import org.springframework.stereotype.Repository;
-
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.stream.Collectors;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-@Repository
-@RequiredArgsConstructor
-public class ObmStorage implements ICloudStorage {
-
- private static final String RECORD_WRITING_ERROR_REASON = "Error on writing record";
- private static final String RECORD_DOES_NOT_HAVE_VERSIONS_AVAILABLE_MSG = "Record %s does not have versions available";
- private static final String ERROR_ON_WRITING_THE_RECORD_HAS_OCCURRED_MSG = "An unexpected error on writing the record has occurred";
-
- private final Driver storage;
- private final DataAuthorizationService dataAuthorizationService;
- private final DpsHeaders headers;
- private final TenantInfo tenantInfo;
- private final IRecordsMetadataRepository<?> recordRepository;
- private final IEntitlementsExtensionService entitlementsService;
- private final ExecutorService threadPool;
- private final JaxRsDpsLog log;
-
- private final PartitionPropertyResolver partitionPropertyResolver;
-
- private final PartitionPropertyNames partitionPropertyNames;
-
- @Override
- public void write(RecordProcessing... records) {
- String bucket = getBucketName(this.tenantInfo);
- String dataPartitionId = tenantInfo.getDataPartitionId();
-
- ObjectMapper mapper = new ObjectMapper();
-
- List<Callable<Boolean>> tasks = new ArrayList<>();
-
- for (RecordProcessing record : records) {
-
- RecordMetadata metadata = record.getRecordMetadata();
- validateMetadata(metadata);
-
- tasks.add(() -> this.writeBlobThread(dataPartitionId, record, mapper, bucket));
- }
-
- try {
- List<Future<Boolean>> results = this.threadPool.invokeAll(tasks);
-
- for (Future<Boolean> future : results) {
- future.get();
- }
-
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- } catch (ExecutionException e) {
- if (e.getCause() instanceof AppException) {
- throw (AppException) e.getCause();
- } else {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record ingestion",
- ERROR_ON_WRITING_THE_RECORD_HAS_OCCURRED_MSG, e);
- }
- }
- }
-
- @Override
- public Map<String, Acl> updateObjectMetadata(List<RecordMetadata> recordsMetadata, List<String> recordsId, List<RecordMetadata> validMetadata,
- List<String> lockedRecords, Map<String, String> recordsIdMap, Optional<CollaborationContext> collaborationContext) {
- String bucket = getBucketName(this.tenantInfo);
- Map<String, Acl> originalAcls = new HashMap<>();
- Map<String, RecordMetadata> currentRecords = this.recordRepository.get(recordsId, collaborationContext);
-
- for (RecordMetadata recordMetadata : recordsMetadata) {
- String id = recordMetadata.getId();
- String idWithVersion = recordsIdMap.get(id);
-
- if (!id.equalsIgnoreCase(idWithVersion)) {
- long previousVersion = Long.parseLong(idWithVersion.split(":")[3]);
- long currentVersion = currentRecords.get(id).getLatestVersion();
- if (previousVersion != currentVersion) {
- lockedRecords.add(idWithVersion);
- continue;
- }
- }
- validMetadata.add(recordMetadata);
- storage.getBlob(bucket, recordMetadata.getVersionPath(recordMetadata.getLatestVersion()), getDestination());
- originalAcls.put(recordMetadata.getId(), currentRecords.get(id).getAcl());
- }
-
- return originalAcls;
- }
-
- @Override
- public void revertObjectMetadata(List<RecordMetadata> recordsMetadata, Map<String, Acl> originalAcls, Optional<CollaborationContext> collaborationContext) {
- String bucket = getBucketName(this.tenantInfo);
-
- for (RecordMetadata recordMetadata : recordsMetadata) {
- storage.getBlob(bucket, recordMetadata.getVersionPath(recordMetadata.getLatestVersion()), getDestination());
- }
- }
-
- @Override
- public boolean hasAccess(RecordMetadata... records) {
-
- if (ArrayUtils.isEmpty(records)) {
- return true;
- }
-
- String bucket = getBucketName(this.tenantInfo);
- for (RecordMetadata record : records) {
-
- try {
- validateMetadata(record);
- } catch (AppException e) {
- if (e.getError().getReason().equals(RECORD_WRITING_ERROR_REASON)) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG);
- } else {
- throw e;
- }
- }
-
- if (!record.getStatus().equals(RecordState.active)) {
- continue;
- }
-
- if (!record.hasVersion()) {
- this.log.warning(String.format(RECORD_DOES_NOT_HAVE_VERSIONS_AVAILABLE_MSG, record.getId()));
- continue;
- }
-
- try {
- String path = record.getVersionPath(record.getLatestVersion());
- Blob blob = storage.getBlob(bucket, path, getDestination());
- if (blob == null) {
- throw new ObmDriverRuntimeException(S3CompatibleErrors.NO_SUCH_KEY_CODE, new RuntimeException(String.format("'%s' not found", path)));
- }
- } catch (ObmDriverRuntimeException exception) {
- throw new AppException(exception.getError().getHttpStatusCode(), exception.getCause().getMessage(), exception.getMessage(), exception);
- }
- }
-
- return true;
- }
-
- @Override
- public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) {
-
- if (!this.dataAuthorizationService.validateViewerOrOwnerAccess(record, OperationType.view)) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG);
- }
-
- try {
- String path = record.getVersionPath(version);
-
- byte[] blob = storage.getBlobContent(getBucketName(this.tenantInfo), path, getDestination());
-
- return new String(blob, UTF_8);
-
- } catch (ObmDriverRuntimeException e) {
- if (e.getError().getHttpStatusCode() == HttpStatus.SC_FORBIDDEN) {
- //exception should be generic, logging can be informative
- throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG, e);
- } else if (e.getError().getHttpStatusCode() == HttpStatus.SC_NOT_FOUND) {
- String msg = String.format("Record with id '%s' does not exist", record.getId());
- throw new AppException(HttpStatus.SC_UNPROCESSABLE_ENTITY, "Record not found", msg);
- } else {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record retrieval",
- "An unexpected error on retrieving the record has occurred", e);
- }
- }
- }
-
- @Override
- public Map<String, String> read(Map<String, String> objects, Optional<CollaborationContext> collaborationContext) {
-
- String bucketName = getBucketName(this.tenantInfo);
- String dataPartitionId = tenantInfo.getDataPartitionId();
-
- Map<String, String> map = new ConcurrentHashMap<>();
-
- List<Callable<Boolean>> tasks = new ArrayList<>();
-
- for (Map.Entry<String, String> object : objects.entrySet()) {
- tasks.add(() -> this.readBlobThread(dataPartitionId, object.getValue(), bucketName, map));
- }
-
- try {
- this.threadPool.invokeAll(tasks);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
-
- return map;
- }
-
- @Override
- public Map<String, String> getHash(Collection<RecordMetadata> records) {
-
- String bucket = getBucketName(this.tenantInfo);
-
- String[] blobIds = records.stream().map(rm -> rm.getVersionPath(rm.getLatestVersion())).toArray(String[]::new);
- Iterable<Blob> blobs = storage.listBlobsByName(bucket, getDestination(), blobIds);
-
- Map<String, String> hashes = new HashMap<>();
-
- for (RecordMetadata rm : records) {
- Blob blob = blobs.iterator().next();
- String hash = blob == null ? "" : blob.getChecksum();
- hashes.put(rm.getId(), hash);
- }
-
- return hashes;
- }
-
- @Override
- public void delete(RecordMetadata record) {
- if (!record.hasVersion()) {
- this.log.warning(String.format(RECORD_DOES_NOT_HAVE_VERSIONS_AVAILABLE_MSG, record.getId()));
- return;
- }
-
- String bucket = getBucketName(this.tenantInfo);
- try {
- Blob blob = storage.getBlob(bucket, record.getVersionPath(record.getLatestVersion()), getDestination());
-
- if (blob == null) {
- String msg = String.format("Record with id '%s' does not exist", record.getId());
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", msg);
- }
-
- String[] versionFiles = record.getGcsVersionPaths().toArray(new String[0]);
-
- storage.deleteBlobs(bucket, getDestination(), versionFiles);
-
- } catch (ObmDriverRuntimeException e) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG, e);
- }
-
- }
-
- @Override
- public void deleteVersion(RecordMetadata record, Long version) {
-
- String bucket = getBucketName(this.tenantInfo);
-
- try {
- if (!record.hasVersion()) {
- this.log.warning(String.format(RECORD_DOES_NOT_HAVE_VERSIONS_AVAILABLE_MSG, record.getId()));
- }
-
- Blob blob = storage.getBlob(bucket, record.getVersionPath(version), getDestination());
-
- if (blob == null) {
- this.log.warning(String.format("Record with id '%s' does not exist, unable to purge version: %s", record.getId(), version));
- }
- storage.deleteBlob(bucket, record.getVersionPath(version), getDestination());
-
- } catch (ObmDriverRuntimeException e) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG, e);
- }
- }
-
- @Override
- public void deleteVersions(List<String> versionPaths) {
- String bucket = getBucketName(this.tenantInfo);
- versionPaths.stream().forEach(versionPath -> {
- try {
- storage.deleteBlob(bucket, versionPath, getDestination());
- } catch (ObmDriverRuntimeException e) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG, e);
- }
- });
- }
-
- @Override
- public boolean isDuplicateRecord(TransferInfo transfer, Map<String, String> hashMap, Map.Entry<RecordMetadata, RecordData> kv) {
- Gson gson = new Gson();
- RecordMetadata updatedRecordMetadata = kv.getKey();
- RecordData recordData = kv.getValue();
- String recordHash = hashMap.get(updatedRecordMetadata.getId());
-
- String newRecordStr = gson.toJson(recordData);
- byte[] bytes = newRecordStr.getBytes(StandardCharsets.UTF_8);
- String newHash = storage.getCalculatedChecksum(bytes);
-
- if (newHash.equals(recordHash)) {
- transfer.getSkippedRecords().add(updatedRecordMetadata.getId());
- return true;
- } else {
- return false;
- }
- }
-
- private boolean writeBlobThread(String dataPartitionId, RecordProcessing processing, ObjectMapper mapper, String bucket) {
-
- RecordMetadata metadata = processing.getRecordMetadata();
- String objectPath = metadata.getVersionPath(metadata.getLatestVersion());
-
- Blob blob = Blob.builder().bucket(bucket).name(objectPath).contentType(MediaType.APPLICATION_JSON_VALUE).build();
-
- try {
- String content = mapper.writeValueAsString(processing.getRecordData());
- storage.createAndGetBlob(blob, content.getBytes(UTF_8), getDestination(dataPartitionId));
- } catch (ObmDriverRuntimeException e) {
- if (e.getError().getHttpStatusCode() == HttpStatus.SC_BAD_REQUEST) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, RECORD_WRITING_ERROR_REASON, e.getMessage(), e);
- }
-
- if (e.getError().getHttpStatusCode() == HttpStatus.SC_FORBIDDEN) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, RECORD_WRITING_ERROR_REASON,
- "User does not have permission to write the records.", e);
- }
-
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, RECORD_WRITING_ERROR_REASON,
- "An unexpected error on writing the record has occurred", e);
- } catch (JsonProcessingException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, RECORD_WRITING_ERROR_REASON,
- "An unexpected error on writing the record has occurred", e);
- }
-
- return true;
- }
-
- private void validateMetadata(RecordMetadata metadata) {
- if (entitlementsService.isDataManager(headers)) {
- return;
- }
-
- List<String> aclGroups = new ArrayList<>();
-
- Collections.addAll(aclGroups, metadata.getAcl().getViewers());
- Collections.addAll(aclGroups, metadata.getAcl().getOwners());
-
- List<String> groups = entitlementsService.getGroups(headers)
- .getGroups()
- .stream()
- .map(GroupInfo::getEmail)
- .collect(Collectors.toList());
-
- Optional<String> missingGroup = aclGroups.stream().filter((s) -> !groups.contains(s)).findFirst();
-
- if (missingGroup.isPresent()) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST,
- RECORD_WRITING_ERROR_REASON,
- String.format("Could not find group \"%s\".", missingGroup.get()));
- }
- }
-
- private boolean readBlobThread(String dataPartitionId, String object, String bucket, Map<String, String> map) {
- String[] tokens = object.split("/");
- String key = tokens[tokens.length - 2];
-
- try {
- String value = new String(storage.getBlobContent(bucket, object, getDestination(dataPartitionId)), UTF_8);
- map.put(key, value);
- } catch (ObmDriverRuntimeException e) {
- map.put(key, null);
- }
-
- return true;
- }
-
- private String getBucketName(TenantInfo tenant) {
- return partitionPropertyResolver
- .getOptionalPropertyValue(
- partitionPropertyNames.getStorageBucketName(), tenantInfo.getDataPartitionId())
- .orElseGet(
- () -> String.format("%s-%s-records", tenant.getProjectId(), tenant.getName()));
- }
-
- private ObmDestination getDestination() {
- return getDestination(tenantInfo.getDataPartitionId());
- }
-
- private ObmDestination getDestination(String dataPartitionId) {
- return ObmDestination.builder().partitionId(dataPartitionId).build();
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmQueryRepository.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmQueryRepository.java
deleted file mode 100644
index ce427114e3d7fc636021cd3b546e19e6b47ddb38..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmQueryRepository.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.repository;
-
-import static org.opengroup.osdu.core.gcp.osm.model.where.condition.And.and;
-import static org.opengroup.osdu.core.gcp.osm.model.where.predicate.Eq.eq;
-import static org.opengroup.osdu.storage.provider.gcp.web.repository.OsmRecordsMetadataRepository.KIND;
-import static org.opengroup.osdu.storage.provider.gcp.web.repository.OsmRecordsMetadataRepository.RECORD_KIND;
-import static org.opengroup.osdu.storage.provider.gcp.web.repository.OsmRecordsMetadataRepository.STATUS;
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.java.Log;
-import org.apache.commons.lang3.NotImplementedException;
-import org.opengroup.osdu.core.common.model.http.CollaborationContext;
-import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.RecordState;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.gcp.osm.model.Destination;
-import org.opengroup.osdu.core.gcp.osm.model.Namespace;
-import org.opengroup.osdu.core.gcp.osm.model.order.OrderBy;
-import org.opengroup.osdu.core.gcp.osm.model.query.GetQuery;
-import org.opengroup.osdu.core.gcp.osm.service.Context;
-import org.opengroup.osdu.core.gcp.osm.translate.Outcome;
-import org.opengroup.osdu.core.gcp.osm.translate.ViewResult;
-import org.opengroup.osdu.storage.model.RecordId;
-import org.opengroup.osdu.storage.model.RecordIdAndKind;
-import org.opengroup.osdu.storage.model.RecordInfoQueryResult;
-import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Repository;
-
-@Repository
-@Scope(SCOPE_SINGLETON)
-@Log
-@RequiredArgsConstructor
-public class OsmQueryRepository implements IQueryRepository {
-
- private final Context context;
- private final TenantInfo tenantInfo;
-
- @Override
- public DatastoreQueryResult getAllKinds(Integer limit, String cursor) {
-
- GetQuery<RecordMetadata> q = new GetQuery<>(RecordMetadata.class, getDestination(),
- eq(STATUS, RecordState.active), OrderBy.builder().addAsc(KIND).build());
- Outcome<ViewResult> out = context.getViewResults(q, null, getLimitTuned(limit), Collections.singletonList(KIND), true, cursor).outcome();
- List<String> kinds = out.getList().stream().map(e -> (String) e.get(KIND)).collect(Collectors.toList());
- return new DatastoreQueryResult(out.getPointer(), kinds);
- }
-
- @Override
- public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor, Optional<CollaborationContext> collaborationContext) {
-
- GetQuery<RecordMetadata> q = new GetQuery<>(RecordMetadata.class, getDestination(), and(eq(KIND, kind), eq(STATUS, RecordState.active)));
- Outcome<ViewResult> out = context.getViewResults(q, null, getLimitTuned(limit), Collections.singletonList("id"), false, cursor).outcome();
- return new DatastoreQueryResult(out.getPointer(), out.getList().stream().map(e -> (String) e.get("id")).collect(Collectors.toList()));
- }
-
- @Override
- public RecordInfoQueryResult<RecordIdAndKind> getAllRecordIdAndKind(Integer limit, String cursor) {
- throw new NotImplementedException();
- }
-
- @Override
- public RecordInfoQueryResult<RecordId> getAllRecordIdsFromKind(Integer limit, String cursor, String kind) {
- throw new NotImplementedException();
- }
-
- @Override
- public HashMap<String, Long> getActiveRecordsCount() {
- throw new NotImplementedException();
- }
-
- @Override
- public Map<String, Long> getActiveRecordsCountForKinds(List<String> kinds) {
- throw new NotImplementedException();
- }
-
- private int getLimitTuned(Integer limit) {
- return limit == null ? PAGE_SIZE : (limit > 0 ? limit : PAGE_SIZE);
- }
-
- private Destination getDestination() {
- return Destination.builder().partitionId(tenantInfo.getDataPartitionId())
- .namespace(new Namespace(tenantInfo.getName()))
- .kind(RECORD_KIND)
- .build();
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmRecordsMetadataRepository.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmRecordsMetadataRepository.java
deleted file mode 100644
index 2cdd936450987940951a1843d5085431d3aec165..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmRecordsMetadataRepository.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.repository;
-
-import com.github.fge.jsonpatch.JsonPatch;
-import java.util.Map.Entry;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.model.http.CollaborationContext;
-import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.gcp.osm.model.Destination;
-import org.opengroup.osdu.core.gcp.osm.model.Kind;
-import org.opengroup.osdu.core.gcp.osm.model.Namespace;
-import org.opengroup.osdu.core.gcp.osm.model.query.GetQuery;
-import org.opengroup.osdu.core.gcp.osm.service.Context;
-import org.opengroup.osdu.core.gcp.osm.translate.Outcome;
-import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
-import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository;
-import org.opengroup.osdu.storage.util.JsonPatchUtil;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Repository;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.opengroup.osdu.core.gcp.osm.model.where.condition.And.and;
-import static org.opengroup.osdu.core.gcp.osm.model.where.predicate.Eq.eq;
-import static org.opengroup.osdu.core.gcp.osm.model.where.predicate.In.in;
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
-@Repository
-@Scope(SCOPE_SINGLETON)
-@Log
-@RequiredArgsConstructor
-public class OsmRecordsMetadataRepository implements IRecordsMetadataRepository<String> {
-
- private final Context context;
- private final TenantInfo tenantInfo;
-
- public static final Kind RECORD_KIND = new Kind("StorageRecord");
- public static final Kind SCHEMA_KIND = new Kind(ISchemaRepository.SCHEMA_KIND);
-
- public static final String KIND = "kind";
- public static final String LEGAL_TAGS = "legal.legaltags";
- public static final String LEGAL_COMPLIANCE = "legal.status";
- public static final String STATUS = "status";
-
- @Override
- public List<RecordMetadata> createOrUpdate(List<RecordMetadata> recordsMetadata,
- Optional<CollaborationContext> collaborationContext) {
- if (recordsMetadata != null) {
- RecordMetadata[] metadata = recordsMetadata.toArray(RecordMetadata[]::new);
- context.upsert(getDestination(), metadata);
- }
- return recordsMetadata;
- }
-
- @Override
- public void delete(String id, Optional<CollaborationContext> collaborationContext) {
- context.deleteById(RecordMetadata.class, getDestination(), id);
- }
-
- @Override
- public RecordMetadata get(String id, Optional<CollaborationContext> collaborationContext) {
- GetQuery<RecordMetadata> osmQuery = new GetQuery<>(RecordMetadata.class, getDestination(), eq("id", id));
- return context.getResultsAsList(osmQuery).stream()
- .filter(Objects::nonNull)
- .findFirst()
- .orElse(null);
- }
-
- @Override
- public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegal(String legalTagName, LegalCompliance status, int limit) {
-
- GetQuery<RecordMetadata>.GetQueryBuilder<RecordMetadata> builder = new GetQuery<>(RecordMetadata.class, getDestination()).toBuilder();
- if (status == null) {
- builder.where(eq(LEGAL_TAGS, legalTagName));
- } else {
- builder.where(and(eq(LEGAL_TAGS, legalTagName), eq(LEGAL_COMPLIANCE, status.name())));
- }
-
- Outcome<RecordMetadata> out = context.getResults(builder.build(), null, limit, null).outcome();
- return new AbstractMap.SimpleEntry<>(out.getPointer(), out.getList());
- }
-
- @Override
- public Map<String, RecordMetadata> get(List<String> ids,
- Optional<CollaborationContext> collaborationContext) {
- GetQuery<RecordMetadata> recordsMetadataInQuery = new GetQuery<>(
- RecordMetadata.class,
- getDestination(),
- in("id", ids)
- );
- return context.getResultsAsList(recordsMetadataInQuery).stream()
- .filter(Objects::nonNull)
- .collect(Collectors.toMap(RecordMetadata::getId, recordMetadata -> recordMetadata));
- }
-
- //TODO remove when other providers replace with new method queryByLegal
- @Override
- public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegalTagName(String legalTagName, int limit, String cursor) {
- return queryByLegal(legalTagName, null, limit);
- }
-
- @Override
- public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegalTagName(String legalTagName[], int limit, String cursor) {
- throw new UnsupportedOperationException("Method not implemented.");
- }
-
- @Override
- public Map<String, String> patch(
- Map<RecordMetadata, JsonPatch> jsonPatchPerRecord,
- Optional<CollaborationContext> collaborationContext) {
- if (Objects.nonNull(jsonPatchPerRecord)) {
- RecordMetadata[] newRecordMetadata = new RecordMetadata[jsonPatchPerRecord.size()];
- int count = 0;
- for (Entry<RecordMetadata, JsonPatch> entry : jsonPatchPerRecord.entrySet()) {
- JsonPatch jsonPatch = entry.getValue();
- newRecordMetadata[count] = JsonPatchUtil.applyPatch(jsonPatch, RecordMetadata.class, entry.getKey());
- count++;
- }
- context.upsert(getDestination(), newRecordMetadata);
- }
- return new HashMap<>();
- }
-
- private Destination getDestination() {
- return Destination.builder().partitionId(tenantInfo.getDataPartitionId())
- .namespace(new Namespace(tenantInfo.getName())).kind(RECORD_KIND).build();
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmSchemaRepository.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmSchemaRepository.java
deleted file mode 100644
index 4345d2d3620d614073f036f8a2681517015d8e36..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/OsmSchemaRepository.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.repository;
-
-import static org.opengroup.osdu.core.gcp.osm.model.where.predicate.Eq.eq;
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.model.storage.Schema;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.gcp.osm.model.Destination;
-import org.opengroup.osdu.core.gcp.osm.model.Namespace;
-import org.opengroup.osdu.core.gcp.osm.model.query.GetQuery;
-import org.opengroup.osdu.core.gcp.osm.service.Context;
-import org.opengroup.osdu.core.gcp.osm.service.Transaction;
-import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Repository;
-
-@Repository
-@Scope(SCOPE_SINGLETON)
-@Log
-@RequiredArgsConstructor
-public class OsmSchemaRepository implements ISchemaRepository {
-
- private final Context context;
- private final TenantInfo tenantInfo;
-
- @Override
- public void add(Schema schema, String user) {
- GetQuery<Schema> q = new GetQuery<>(Schema.class, getDestination(), eq("kind", schema.getKind()));
- Transaction txn = context.beginTransaction(getDestination());
- try {
- if (context.findOne(q).isPresent()) {
- txn.rollbackIfActive();
- throw new IllegalArgumentException("A schema for the specified kind has already been registered.");
- } else {
- context.create(getDestination(), schema);
- txn.commitIfActive();
- }
- } finally {
- if (txn != null) {
- txn.rollbackIfActive();
- }
- }
- }
-
- @Override
- public Schema get(String kind) {
- GetQuery<Schema> q = new GetQuery<>(Schema.class, getDestination(), eq("kind", kind));
- return context.getResultsAsList(q).stream().findFirst().orElse(null);
- }
-
- @Override
- public void delete(String kind) {
- context.deleteById(Schema.class, getDestination(), kind);
- }
-
- private Destination getDestination() {
- return Destination.builder()
- .partitionId(tenantInfo.getDataPartitionId())
- .namespace(new Namespace(tenantInfo.getName()))
- .kind(OsmRecordsMetadataRepository.SCHEMA_KIND)
- .build();
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/security/GSuiteSecurityConfig.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/security/GSuiteSecurityConfig.java
deleted file mode 100644
index 8b0a1e930e2af0b5ab46cbb5ebe36218a5382d7c..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/security/GSuiteSecurityConfig.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.security;
-
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity;
-import org.springframework.security.config.annotation.web.builders.HttpSecurity;
-import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
-import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
-import org.springframework.security.config.http.SessionCreationPolicy;
-import org.springframework.security.web.SecurityFilterChain;
-
-import static org.springframework.security.config.Customizer.withDefaults;
-
-@Configuration
-@EnableWebSecurity
-@EnableMethodSecurity
-public class GSuiteSecurityConfig {
-
-
- @Bean
- public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
- http
- .cors(AbstractHttpConfigurer::disable)
- .csrf(AbstractHttpConfigurer::disable)
- .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
- .authorizeHttpRequests(authorize -> authorize.anyRequest().permitAll())
- .httpBasic(withDefaults());
- return http.build();
- }
-
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/service/BatchServiceGcpImpl.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/service/BatchServiceGcpImpl.java
deleted file mode 100644
index 6124a62924716a9c2f215f63e73a6473d1cbcfb7..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/service/BatchServiceGcpImpl.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.service;
-
-import static java.util.Collections.singletonList;
-
-import lombok.RequiredArgsConstructor;
-import org.opengroup.osdu.core.common.model.http.CollaborationContext;
-import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
-import org.opengroup.osdu.storage.logging.StorageAuditLogger;
-import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
-import org.opengroup.osdu.storage.service.BatchServiceImpl;
-import org.springframework.stereotype.Service;
-
-import java.util.Optional;
-
-@Service
-@RequiredArgsConstructor
-public class BatchServiceGcpImpl extends BatchServiceImpl {
-
- private final IQueryRepository queryRepository;
- private final StorageAuditLogger auditLogger;
-
- @Override
- public DatastoreQueryResult getAllKinds(String cursor, Integer limit) {
- DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor);
- this.auditLogger.readAllKindsSuccess(result.getResults());
- return result;
- }
-
- @Override
- public DatastoreQueryResult getAllRecords(String cursor, String kind, Integer limit, Optional<CollaborationContext> collaborationContext) {
- DatastoreQueryResult result = this.queryRepository.getAllRecordIdsFromKind(kind, limit, cursor, collaborationContext);
- if (!result.getResults().isEmpty()) {
- this.auditLogger.readAllRecordsOfGivenKindSuccess(singletonList(kind));
- }
- return result;
- }
-}
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/util/ServiceAccountJwtClientImpl.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/util/ServiceAccountJwtClientImpl.java
deleted file mode 100644
index 3989219a25bb88fca302d03444433772e878cddb..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/util/ServiceAccountJwtClientImpl.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.web.util;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Primary
-@Component
-@RequestScope
-@Slf4j
-@RequiredArgsConstructor
-public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
-
- private final TokenProvider tokenProvider;
-
- @Override
- public String getIdToken(String tenantName) {
- log.debug("Tenant name received for auth token is: {}", tenantName);
- return "Bearer " + tokenProvider.getIdToken();
- }
-}
\ No newline at end of file
diff --git a/provider/storage-gc/src/main/resources/application-anthos.properties b/provider/storage-gc/src/main/resources/application-anthos.properties
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/provider/storage-gc/src/main/resources/application-gcp.properties b/provider/storage-gc/src/main/resources/application-gcp.properties
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/provider/storage-gc/src/main/resources/application-local.properties b/provider/storage-gc/src/main/resources/application-local.properties
deleted file mode 100644
index 2fdec1f53f1d93231c299e3c6def5dd00cd318da..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/main/resources/application-local.properties
+++ /dev/null
@@ -1,13 +0,0 @@
-osmDriver=datastore
-obmDriver=gcs
-oqmDriver=pubsub
-
-REDIS_GROUP_HOST=127.0.0.1
-REDIS_STORAGE_HOST=127.0.0.1
-
-PARTITION_API=https://community.gcp.gnrg-osdu.projects.epam.com/api/partition/v1/
-AUTHORIZE_API=https://community.gcp.gnrg-osdu.projects.epam.com/entitlements/v2
-LEGALTAG_API=https://community.gcp.gnrg-osdu.projects.epam.com/api/legal/v1
-CRS_API=https://community.gcp.gnrg-osdu.projects.epam.com/api/crs/v2
-
-DEFAULT_DATA_COUNTRY=US
diff --git a/provider/storage-gc/src/main/resources/application.properties b/provider/storage-gc/src/main/resources/application.properties
index 8306f694791eb6f2491ab62e841d900a5de7eb06..71eaa84449425f7c4fda3bb8ec1368f09d0c83c6 100644
--- a/provider/storage-gc/src/main/resources/application.properties
+++ b/provider/storage-gc/src/main/resources/application.properties
@@ -15,7 +15,6 @@ osdu.spring.config.enableEncodedSpecialCharacters=true
# Security config
opa.opa-endpoint=${OPA_ENDPOINT:notused}
-opa.enabled=true
# OQM config
legal-tags-changed-topic-name=legaltags-changed
@@ -58,16 +57,15 @@ management.endpoints.web.base-path=${MANAGEMENT_ENDPOINTS_WEB_BASE:/}
management.endpoints.web.exposure.include=health
management.health.probes.enabled=true
-featureFlag.strategy=systemPartition
-featureFlag.opa.enabled=false
-SYSTEM_PARTITION_ID=system
+featureFlag.strategy=appProperty
+featureFlag.opa.enabled=${OPA_ENABLED:false}
+
+destination.resolver=partition
-# GCP specific properties
-osmDriver=datastore
-obmDriver=gcs
-oqmDriver=pubsub
service.token.provider=GCP
partition-auth-enabled=true
datastore-beta-enabled=false
dead-lettering-required=true
+googleAudiences=${GOOGLE_AUDIENCES:osdu}
+
diff --git a/provider/storage-gc/src/main/resources/logback.xml b/provider/storage-gc/src/main/resources/logback.xml
index cdfa2e625e54da4cfb64929655942bfd3b739cbb..c32f3088eaf002ebd063e52993700511a083372a 100644
--- a/provider/storage-gc/src/main/resources/logback.xml
+++ b/provider/storage-gc/src/main/resources/logback.xml
@@ -22,7 +22,7 @@
<timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId>
<appendLineSeparator>true</appendLineSeparator>
- <jsonFormatter class="org.opengroup.osdu.core.gcp.logging.formatter.GoogleJsonFormatter">
+ <jsonFormatter class="org.opengroup.osdu.storage.provider.gcp.logging.formatter.GoogleJsonFormatter">
<prettyPrint>false</prettyPrint>
</jsonFormatter>
</layout>
diff --git a/provider/storage-gc/src/main/resources/type-mapper-config.json b/provider/storage-gc/src/main/resources/type-mapper-config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d2d406fded81c240f1eda2695708ba89e6daa40b
--- /dev/null
+++ b/provider/storage-gc/src/main/resources/type-mapper-config.json
@@ -0,0 +1,39 @@
+{
+ "instrumentations": [
+ {
+ "entityType": "org.opengroup.osdu.core.common.model.storage.RecordMetadata",
+ "fieldNameCustomMapping": {
+ "user": "createUser",
+ "gcsVersionPaths": "bucket"
+ },
+ "fieldTypeCustomMapping": {
+ "createTime": "shaded.osm.com.google.cloud.Timestamp",
+ "modifyTime": "shaded.osm.com.google.cloud.Timestamp"
+ },
+ "identityTranslator": {
+ "getIdMethod": "getId",
+ "setIdMethod": "setId",
+ "keyClass": "java.lang.String"
+ },
+ "keyFields": ["id"]
+ },
+
+ {
+ "entityType": "org.opengroup.osdu.core.common.model.storage.Schema",
+ "fieldNameCustomMapping": {
+ "ext": "extension",
+ "gcsVersionPaths": "bucket"
+ },
+ "fieldTypeCustomMapping": {
+ "schema": "shaded.osm.com.google.cloud.datastore.Blob",
+ "ext": "shaded.osm.com.google.cloud.datastore.Blob"
+ },
+ "identityTranslator": {
+ "getIdMethod": "getKind",
+ "setIdMethod": "setKind",
+ "keyClass": "java.lang.String"
+ },
+ "keyFields": ["kind"]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/MultiThreadingLegalTagChangedProcessingTest.java b/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/MultiThreadingLegalTagChangedProcessingTest.java
deleted file mode 100644
index a0316fb37e08c8207e84d5ff05434cf18e9cdeec..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/MultiThreadingLegalTagChangedProcessingTest.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.jobs;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.when;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import java.util.AbstractMap.SimpleEntry;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.logging.DefaultLogger;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.entitlements.Groups;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.legal.InvalidTagWithReason;
-import org.opengroup.osdu.core.common.model.legal.Legal;
-import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagConsistencyValidator;
-import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.Schema;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmAckReplier;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.storage.logging.StorageAuditLogger;
-import org.opengroup.osdu.storage.provider.gcp.messaging.config.MessagingConfigurationProperties;
-import org.opengroup.osdu.storage.provider.gcp.messaging.jobs.config.PullConfigStub;
-import org.opengroup.osdu.storage.provider.gcp.messaging.jobs.stub.OqmPubSubStub;
-import org.opengroup.osdu.storage.provider.gcp.messaging.scope.override.ThreadDpsHeaders;
-import org.opengroup.osdu.storage.provider.gcp.messaging.thread.ThreadScopeContextHolder;
-import org.opengroup.osdu.storage.provider.gcp.web.cache.LegalTagMultiTenantCache;
-import org.opengroup.osdu.storage.provider.gcp.web.repository.OsmRecordsMetadataRepository;
-import org.opengroup.osdu.storage.service.LegalServiceImpl;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.test.context.junit4.SpringRunner;
-
-@RunWith(SpringRunner.class)
-@SpringBootTest(classes = {PullConfigStub.class}, webEnvironment = WebEnvironment.NONE, properties = {"oqmDriver=any"})
-public class MultiThreadingLegalTagChangedProcessingTest {
-
- private static final String FIRST_TEST_TENANT = "test";
- private static final String FIRST_TEST_PROJECT = "test.project";
- private static final String SECOND_TEST_PROJECT = "test2.project";
- private static final String SECOND_TEST_TENANT = "test2";
- private static final String FIRST_TEST_LEGALTAG = "test-dataset-tag";
- private static final String SECOND_TEST_LEGALTAG = "test2-dataset-tag";
- private static final String REASON = "LegalTag not found";
- private static final String TEST_CORRELATION = "123";
- private static final int numberOfAllRuns = 500;
- private static final int numberOfTenantRuns = numberOfAllRuns / 2;
- private final CountDownLatch latch = new CountDownLatch(numberOfAllRuns);
-
- @MockBean
- private ICache<String, Groups> groupCache;
-
- @MockBean(name = "LegalTagCache")
- private LegalTagMultiTenantCache legalTagMultiTenantCache;
-
- @MockBean
- private ICache<String, Schema> schemaCache;
-
- @MockBean
- private TokenProvider tokenProvider;
-
- @MockBean
- private ITenantFactory tenantInfoFactory;
-
- @MockBean
- private OqmAckReplier oqmAckReplier;
-
- @MockBean
- private LegalServiceImpl legalService;
-
- @MockBean
- private OsmRecordsMetadataRepository recordsMetadataRepository;
-
- @MockBean
- private OqmDriver oqmDriver;
-
- @MockBean
- private DefaultLogger iLogger;
-
- @MockBean
- private JaxRsDpsLog jaxRsDpsLog;
-
- @MockBean
- private StorageAuditLogger auditLogger;
-
- @Autowired
- private OqmPubSubStub pubSubStub;
-
- @Autowired
- private PullConfigStub pullConfigStub;
-
- @Autowired
- private LegalTagConsistencyValidator legalTagConsistencyValidator;
-
- @Autowired
- private LegalComplianceChangeServiceGcpImpl complianceChangeServiceGcp;
-
- @Autowired
- private ThreadDpsHeaders threadDpsHeaders;
-
- @Autowired
- private MessagingConfigurationProperties configurationProperties;
-
- @Before
- public void setUp() {
- configurationProperties.setStorageServiceAccountEmail("storage");
-
- setUpTenants();
-
- for (int i = 0; i < numberOfAllRuns; i++) {
- setUpLegalServiceGetInvalidLegalTags(FIRST_TEST_LEGALTAG + i);
- setUpLegalServiceGetInvalidLegalTags(SECOND_TEST_LEGALTAG + i);
- setStorageRepoQueryByLegal(FIRST_TEST_LEGALTAG + i);
- setStorageRepoQueryByLegal(SECOND_TEST_LEGALTAG + i);
- }
- }
-
- @Test
- public void testMultithreadingMessageProcessingWithDifferentTenants() throws InterruptedException {
- ExecutorService service = Executors.newFixedThreadPool(numberOfTenantRuns);
- for (int i = 0; i < numberOfTenantRuns; i++) {
- int finalI = i;
- service.execute(() -> messageRun(FIRST_TEST_LEGALTAG + finalI, FIRST_TEST_TENANT));
- service.execute(() -> messageRun(SECOND_TEST_LEGALTAG + finalI, SECOND_TEST_TENANT));
- }
- latch.await();
-
- List<Map<DpsHeaders, PubSubInfo[]>> collector = pubSubStub.getCollector();
-
- assertEquals(numberOfAllRuns, collector.size());
-
- long firstTenantEvents = collector.stream().flatMap(entry -> entry.keySet().stream()).map(DpsHeaders::getPartitionId)
- .filter(partition -> partition.equals(FIRST_TEST_TENANT)).count();
-
- long secondTenantEvents = collector.stream().flatMap(entry -> entry.keySet().stream()).map(DpsHeaders::getPartitionId)
- .filter(partition -> partition.equals(SECOND_TEST_TENANT)).count();
-
- assertEquals(numberOfTenantRuns, firstTenantEvents);
- assertEquals(numberOfTenantRuns, secondTenantEvents);
- }
-
- private void messageRun(String legalTagName, String tenantName) {
- try {
- OqmMessage firstOqmMessage = new OqmMessage("1", "{\n"
- + " \"statusChangedTags\": [{\n"
- + " \"changedTagName\": \"" + legalTagName + "\",\n"
- + " \"changedTagStatus\": \"incompliant\"\n"
- + " }\n"
- + " ]\n"
- + "}",
- ImmutableMap.of(
- DpsHeaders.ACCOUNT_ID,
- tenantName,
- DpsHeaders.CORRELATION_ID,
- TEST_CORRELATION,
- DpsHeaders.DATA_PARTITION_ID,
- tenantName
- )
- );
-
- threadDpsHeaders.setThreadContext(firstOqmMessage.getAttributes());
- LegalTagChangedProcessing legalTagChangedProcessing =
- new LegalTagChangedProcessing(legalTagConsistencyValidator, complianceChangeServiceGcp, threadDpsHeaders);
-
- legalTagChangedProcessing.process(firstOqmMessage);
- } catch (Exception e) {
- System.out.println(Arrays.toString(e.getStackTrace()));
- } finally {
- ThreadScopeContextHolder.currentThreadScopeAttributes().clear();
- latch.countDown();
- }
- }
-
-
- private void setUpLegalServiceGetInvalidLegalTags(String legalTagName) {
- InvalidTagWithReason[] firstInvalidTagWithReasons = new InvalidTagWithReason[1];
- InvalidTagWithReason invalidTagWithReason = new InvalidTagWithReason();
- invalidTagWithReason.setName(legalTagName);
- invalidTagWithReason.setReason(REASON);
- firstInvalidTagWithReasons[0] = invalidTagWithReason;
-
- when(legalService.getInvalidLegalTags(Collections.singleton(legalTagName))).thenReturn(firstInvalidTagWithReasons);
-
- }
-
- private void setStorageRepoQueryByLegal(String legalTagName) {
- Legal legal = new Legal();
- legal.setLegaltags(Collections.singleton(legalTagName));
-
- RecordMetadata recordMetadata = new RecordMetadata();
- recordMetadata.setId("record-id" + legalTagName);
- recordMetadata.setLegal(legal);
-
- SimpleEntry<String, List<RecordMetadata>> mapWithValues = new SimpleEntry<>("empty", Collections.singletonList(recordMetadata));
- SimpleEntry<String, List<RecordMetadata>> emptyMap = new SimpleEntry<>("empty", Collections.emptyList());
-
- when(recordsMetadataRepository.queryByLegal(legalTagName, LegalCompliance.compliant, 500))
- .thenReturn(mapWithValues)
- .thenReturn(emptyMap);
-
- }
-
- private void setUpTenants() {
- TenantInfo testTenant = new TenantInfo();
- testTenant.setProjectId(FIRST_TEST_PROJECT);
- testTenant.setDataPartitionId(FIRST_TEST_TENANT);
- testTenant.setName(FIRST_TEST_TENANT);
-
- TenantInfo secondTestTenant = new TenantInfo();
- secondTestTenant.setProjectId(SECOND_TEST_PROJECT);
- secondTestTenant.setDataPartitionId(SECOND_TEST_TENANT);
- secondTestTenant.setName(SECOND_TEST_TENANT);
-
- when(tenantInfoFactory.listTenantInfo()).thenReturn(ImmutableList.of(testTenant, secondTestTenant));
- }
-}
diff --git a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/config/PullConfigStub.java b/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/config/PullConfigStub.java
deleted file mode 100644
index f3370b5059888bc30cb2e5407c813c8705837d0c..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/config/PullConfigStub.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.jobs.config;
-
-import lombok.Getter;
-import org.opengroup.osdu.core.common.legal.ILegalService;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagConsistencyValidator;
-import org.opengroup.osdu.storage.provider.gcp.messaging.config.MessagingCustomContextConfiguration;
-import org.opengroup.osdu.storage.provider.gcp.messaging.jobs.stub.OqmPubSubStub;
-import org.opengroup.osdu.storage.provider.gcp.messaging.scope.override.ScopeModifierPostProcessor;
-import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
-import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.test.util.ReflectionTestUtils;
-
-@Getter
-@Configuration
-@EnableConfigurationProperties
-@ComponentScan(value = {
- "org.opengroup.osdu.storage.provider.gcp.messaging"
-},
- excludeFilters = {
- @ComponentScan.Filter(
- type = FilterType.ASSIGNABLE_TYPE,
- value = {
- MessagingCustomContextConfiguration.class
- }
- )
- }
-)
-public class PullConfigStub {
-
- @Bean
- public IMessageBus messageBusStub() {
- return new OqmPubSubStub();
- }
-
- @Bean
- public LegalTagConsistencyValidator legalTagConsistencyValidator(ILegalService legalService,
- JaxRsDpsLog jaxRsDpsLog) {
- LegalTagConsistencyValidator legalTagConsistencyValidator = new LegalTagConsistencyValidator();
- ReflectionTestUtils.setField(legalTagConsistencyValidator, "legalService", legalService);
- ReflectionTestUtils.setField(legalTagConsistencyValidator, "logger", jaxRsDpsLog);
- return legalTagConsistencyValidator;
- }
-
- @Bean
- public BeanFactoryPostProcessor beanFactoryPostProcessor() {
- return new ScopeModifierPostProcessor();
- }
-}
diff --git a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/stub/OqmPubSubStub.java b/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/stub/OqmPubSubStub.java
deleted file mode 100644
index 40f3c8b8bd7b4740db07575aafb5c7aa42aaea19..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/messaging/jobs/stub/OqmPubSubStub.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.storage.provider.gcp.messaging.jobs.stub;
-
-import com.google.common.collect.ImmutableMap;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import lombok.Getter;
-import org.apache.commons.lang3.NotImplementedException;
-import org.opengroup.osdu.core.common.model.http.CollaborationContext;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
-import org.opengroup.osdu.storage.model.RecordChangedV2;
-import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
-
-@Getter
-public class OqmPubSubStub implements IMessageBus {
-
- private final List<Map<DpsHeaders, PubSubInfo[]>> collector = new ArrayList<>();
-
- @Override
- public synchronized void publishMessage(DpsHeaders headers, PubSubInfo... messages) {
- collector.add(ImmutableMap.of(DpsHeaders.createFromMap(headers.getHeaders()), messages));
- }
-
- @Override
- public void publishMessage(Optional<CollaborationContext> collaborationContext, DpsHeaders headers, RecordChangedV2... messages) {
- throw new NotImplementedException();
- }
-
- @Override
- public void publishMessage(DpsHeaders headers, Map<String, String> routingInfo, List<?> messageList) {
- throw new NotImplementedException();
- }
-
- @Override
- public void publishMessage(DpsHeaders headers, Map<String, String> routingInfo, PubSubInfo... messages) {
- throw new NotImplementedException();
- }
-}
diff --git a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/web/middleware/GcpExceptionMapperTest.java b/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/web/middleware/GcpExceptionMapperTest.java
deleted file mode 100644
index e54a7bd1a31fc390c6980b0aeb77b33bcdc43af7..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/web/middleware/GcpExceptionMapperTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.opengroup.osdu.storage.provider.gcp.web.middleware;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.when;
-import static org.springframework.http.HttpStatus.FORBIDDEN;
-
-import com.google.cloud.storage.StorageException;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.model.http.AppError;
-import org.opengroup.osdu.storage.util.GlobalExceptionMapper;
-import org.springframework.http.ResponseEntity;
-
-@RunWith(MockitoJUnitRunner.class)
-public class GcpExceptionMapperTest {
-
- @InjectMocks
- private GcpExceptionMapper sut;
-
- @Mock
- private GlobalExceptionMapper mapper;
-
- @Test
- public void handleStorageExceptionShouldReturnForbiddenResponseWhenCatchesStorageException() {
- StorageException exception = new StorageException(403, "Access Denied");
- AppError expectedBody = new AppError(FORBIDDEN.value(),
- "Access Denied", "The user is not authorized to perform this action");
- when(mapper.getErrorResponse(ArgumentMatchers .argThat(
- e -> StorageException.class == e.getOriginalException().getClass())))
- .thenReturn(new ResponseEntity<>(expectedBody, FORBIDDEN));
-
- ResponseEntity<Object> response = sut.handleStorageException(exception);
-
- assertThat(response.getStatusCodeValue(), is(FORBIDDEN.value()));
- assertThat(response.getBody(), is(expectedBody));
- }
-}
diff --git a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorageTest.java b/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorageTest.java
deleted file mode 100644
index bd19300140609738070a3aea763a48ed0bac1702..0000000000000000000000000000000000000000
--- a/provider/storage-gc/src/test/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorageTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package org.opengroup.osdu.storage.provider.gcp.web.repository;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.util.Arrays;
-import java.util.List;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.partition.PartitionPropertyResolver;
-import org.opengroup.osdu.core.gcp.obm.driver.Driver;
-import org.opengroup.osdu.storage.provider.gcp.web.config.PartitionPropertyNames;
-
-@ExtendWith(MockitoExtension.class)
-class ObmStorageTest {
-
- private static final String PARTITION_ID = "dummyPartitionID";
- private static final String BUCKET_NAME = "dummy_bucket";
- private static final List<String> VERSION_PATHS = Arrays.asList("versionPath1", "versionPath2");
-
- @InjectMocks
- private ObmStorage storage;
-
- @Mock
- private Driver storageDriver;
-
- @Mock
- private PartitionPropertyNames partitionPropertyNames;
-
- @Mock
- private TenantInfo tenantInfo;
-
- @Mock
- private PartitionPropertyResolver partitionPropertyResolver;
-
-
- @Test
- void deleteSeveralVersions() {
- when(partitionPropertyNames.getStorageBucketName()).thenReturn(BUCKET_NAME);
- when(tenantInfo.getDataPartitionId()).thenReturn(PARTITION_ID);
-
- storage.deleteVersions(VERSION_PATHS);
-
- VERSION_PATHS.forEach(versionPath ->
- verify(storageDriver, times(1))
- .deleteBlob(any() , eq(versionPath), any()));
- }
-}
diff --git a/testing/storage-test-gc/src/test/resources/test.properties b/testing/storage-test-gc/src/test/resources/test.properties
index 4e97497f40685257e6c9f9bf3843436398bf2e72..4fd2b81759f4f000bb611ccf9eb46e9145093139 100644
--- a/testing/storage-test-gc/src/test/resources/test.properties
+++ b/testing/storage-test-gc/src/test/resources/test.properties
@@ -1,2 +1,2 @@
enableEncodedSpecialCharactersInURL=true
-opa.integration.enabled=true
+opa.integration.enabled=false