From 092801d65499d18e895dcf50bfb12873e654742b Mon Sep 17 00:00:00 2001
From: "Rustam Lotsmanenko (EPAM)" <rustam_lotsmanenko@epam.com>
Date: Wed, 9 Mar 2022 09:10:56 +0000
Subject: [PATCH] remove encryption from partition service, refactor properties
and docs(GONRG-4404)
---
.../templates/partition-variables.yml | 3 +-
devops/gcp/configmap/values.yaml | 3 +-
provider/partition-gcp/README.md | 120 ------------------
provider/partition-gcp/docs/anthos/README.md | 47 +++++++
provider/partition-gcp/docs/gcp/README.md | 38 ++++++
provider/partition-gcp/pom.xml | 7 +-
.../provider/gcp/PartitionGcpApplication.java | 10 +-
.../provider/gcp/cache/GroupCache.java | 30 -----
.../gcp/config/PostgresOsmConfiguration.java | 2 +
.../OsmPartitionDestinationProvider.java | 13 +-
.../OsmDatastoreDestinationResolver.java | 36 +++---
.../OsmPostgresDestinationResolver.java | 30 ++---
.../gcp/service/PartitionServiceImpl.java | 42 +-----
.../resources/application-anthos.properties | 25 ++++
.../main/resources/application-gcp.properties | 22 ++++
.../application-osm-postgres.properties | 53 --------
.../src/main/resources/application.properties | 29 ++---
.../PartitionServiceImplCacheTest.java | 23 ++--
.../gcp/service/PartitionServiceImplTest.java | 24 ++--
19 files changed, 223 insertions(+), 334 deletions(-)
create mode 100644 provider/partition-gcp/docs/anthos/README.md
create mode 100644 provider/partition-gcp/docs/gcp/README.md
delete mode 100644 provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/GroupCache.java
create mode 100644 provider/partition-gcp/src/main/resources/application-anthos.properties
create mode 100644 provider/partition-gcp/src/main/resources/application-gcp.properties
delete mode 100644 provider/partition-gcp/src/main/resources/application-osm-postgres.properties
diff --git a/devops/gcp/configmap/templates/partition-variables.yml b/devops/gcp/configmap/templates/partition-variables.yml
index 3d379e794..e76b8c0f5 100644
--- a/devops/gcp/configmap/templates/partition-variables.yml
+++ b/devops/gcp/configmap/templates/partition-variables.yml
@@ -9,7 +9,6 @@ data:
GOOGLE_CLOUD_PROJECT: "{{ .Values.data.google_cloud_project }}"
PARTITION_ADMIN_ACCOUNTS: "{{ .Values.data.partition_admin_accounts }}"
GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
- KEY_RING: "{{ .Values.data.key_ring }}"
- KMS_KEY: "{{ .Values.data.kms_key }}"
LOG_LEVEL: "{{ .Values.data.log_level }}"
SERVICE_ACCOUNT_TAIL: "{{ .Values.data.service_account_tail }}"
+ SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml
index 460bc78f2..61d4a23e0 100644
--- a/devops/gcp/configmap/values.yaml
+++ b/devops/gcp/configmap/values.yaml
@@ -2,10 +2,9 @@ data:
google_cloud_project: ""
partition_admin_accounts: ""
google_audiences: ""
- key_ring: ""
- kms_key: ""
log_level: "INFO"
service_account_tail: ".iam.gserviceaccount.com"
+ spring_profiles_active: "gcp"
conf:
diff --git a/provider/partition-gcp/README.md b/provider/partition-gcp/README.md
index 95b635050..ba368928f 100644
--- a/provider/partition-gcp/README.md
+++ b/provider/partition-gcp/README.md
@@ -33,106 +33,6 @@ Pre-requisites
* Lombok 1.16 or later
* Maven
-## Mapper tuning mechanisms
-
-This service uses specific implementations of DestinationResolvers. A total of 2 resolvers have been implemented, which are divided into two groups:
-
-### for universal technologies:
-
-- for Postgres: osm/config/resolver/OsmPostgresDestinationResolver.java
-
-#### Their algorithms are as follows:
-
-- incoming Destination carries data-partition-id
-- resolver accesses the Partition service and gets PartitionInfo
-- from PartitionInfo resolver retrieves properties for the connection: URL, username, password etc.
-- resolver creates a data source, connects to the resource, remembers the datasource
-- resolver gives the datasource to the mapper in the Resolution object
-
-### for native Google Cloud technologies:
-
-- for Datastore: osm/config/resolver/OsmDatastoreDestinationResolver.java
-
-#### Their algorithms are similar,
-
-Except that they do not receive special properties from the Partition service for connection, because the location of
-the resources is unambiguously known - they are in the GCP project. And credentials are also not needed - access to data
-is made on behalf of the Google Identity SA under which the service itself is launched. Therefore, resolver takes only
-the value of the **projectId** property from PartitionInfo and uses it to connect to a resource in the corresponding GCP
-project.
-
-### Installation
-In order to run the service locally or remotely, you will need to have the following environment variables defined.
-
-| name | value | description | sensitive? | source |
-| --- | --- | --- | --- | --- |
-| `LOG_PREFIX` | `service` | Logging prefix | no | - |
-| `SERVER_SERVLET_CONTEXPATH` | `/api/partition/v1` | Servlet context path | no | - |
-| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
-| `GOOGLE_CLOUD_PROJECT` | ex `osdu-cicd-epam` | Google Cloud Project Id| no | output of infrastructure deployment |
-| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
-| `PARTITION_ADMIN_ACCOUNTS` | ex `admin@domen.iam.gserviceaccount.com,osdu-gcp-sa,workload-identity` | List of partition admin account emails, could be in full form like `admin@domen.iam.gserviceaccount.com` or in `starts with` pattern like `osdu-gcp-sa`| no | - |
-| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `KEY_RING` | ex `csqp` | A key ring holds keys in a specific Google Cloud location and permit us to manage access control on groups of keys | yes | https://cloud.google.com/kms/docs/resource-hierarchy#key_rings |
-| `KMS_KEY` | ex `partitionService` | A key exists on one key ring linked to a specific location. | yes | https://cloud.google.com/kms/docs/resource-hierarchy#key_rings |
-| `PARTITION_PROPERTY_KIND` | ex `PartitionProperty` | Kind name to store the properties. | no | - |
-| `PARTITION_NAMESPACE` | ex `partition` | Namespace for database. | no | - |
-| `osmDriver` | ex `postgres` or `datastore` | Osm driver mode that defines which storage will be used | no | - |
-| `osm.postgres.url` | ex `jdbc:postgresql://127.0.0.1:5432/postgres` | Postgres server URL | no | - |
-| `osm.postgres.username` | ex `postgres` | Postgres admin username | no | - |
-| `osm.postgres.password` | ex `postgres` | Postgres admin password | yes | - |
-| `ENVIRONMENT` | `gcp` or `anthos` | If `anthos` then authorization is disabled | no | - |
-| `SERVICE_ACCOUNT_TAIL` | `****` |By default Partition service while authenticating the request, verifies that the email in provided token belongs to a service account from a specific project by email tail `<GOOGLE_CLOUD_PROJECT> + .iam.gserviceaccount.com`, this behavior can be changed with this variable, you may specify which email tail exactly expected.| no | - |
-
-## Configuring mappers' Datasources
-
-When using non-Google-Cloud-native technologies, property sets must be defined on the Partition service as part of
-PartitionInfo for each Tenant.
-
-They are specific to each storage technology:
-
-#### for OSM - Postgres:
-
-**database structure**
-OSM works with data logically organized as "partition"->"namespace"->"kind"->"record"->"columns". The above sequence
-describes how it is named in Google Datastore, where "partition" maps to "GCP project".
-
-This is how **Postgres** OSM driver does. Notice, the above hierarchy has been kept, but Postgres uses alternative entities
-for it.
-
-| Datastore hierarchy level | | Postgres alternative used |
-|---------------------------|-----|----------------------------|
-| partition (GCP project) | == | Postgres server URL |
-| namespace | == | Schema |
-| kind | == | Table |
-| record | == | '<multiple table records>' |
-| columns | == | id, data (jsonb) |
-
-As we can see in the above table, Postgres uses different approach in storing business data in records. Not like
-Datastore, which segments data into multiple physical columns, Postgres organises them into the single JSONB "data"
-column. It allows provisioning new data registers easily not taking care about specifics of certain registers structure.
-In the current OSM version (as on December'21) the Postgres OSM driver is not able to create new tables in runtime.
-
-So this is a responsibility of DevOps / CICD to provision all required SQL tables (for all required data kinds) when on new
-environment or tenant provisioning when using Postgres. Detailed instructions (with examples) for creating new tables is
-in the **OSM module Postgres driver README.md** `org/opengroup/osdu/core/gcp/osm/translate/postgresql/README.md`
-
-As a quick shortcut, this example snippet can be used by DevOps DBA:
-
-* `exampleschema` equals to `PARTITION_NAMESPACE`
-* `ExampleKind` equals to `PARTITION_PROPERTY_KIND`
-
-```postgres-psql
---CREATE SCHEMA "<exampleschema>";
-CREATE TABLE <exampleschema>."<ExampleKind>"(
- id text COLLATE pg_catalog."default" NOT NULL,
- pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
- data jsonb NOT NULL,
- CONSTRAINT <ExampleKind>_id UNIQUE (id)
-);
-CREATE INDEX <ExampleKind>_datagin ON <exampleschema>."<ExampleKind>" USING GIN (data);
-```
-
### Run Locally
Check that maven is installed:
@@ -246,26 +146,6 @@ Partition Service is compatible with App Engine Flexible Environment and Cloud R
* To deploy into App Engine, please, use this documentation:
https://cloud.google.com/appengine/docs/flexible/java/quickstart
-#### Cloud KMS Setup
-
-Enable cloud KMS on master project.
-
-Create king ring and key in the ***master project***
-
-```bash
- gcloud services enable cloudkms.googleapis.com
- export KEYRING_NAME="csqp"
- export CRYPTOKEY_NAME="partionService"
- gcloud kms keyrings create $KEYRING_NAME --location global
- gcloud kms keys create $CRYPTOKEY_NAME --location global \
- --keyring $KEYRING_NAME \
- --purpose encryption
-```
-
-Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the used **service account** by Partition Service of the ***master project*** through IAM - Role tab.
-
-Add "Cloud KMS Encrypt/Decrypt" role to the used **service account** by Partition Service of the ***master project*** through IAM - Role tab.
-
## Licence
Copyright © Google LLC
Copyright © EPAM Systems
diff --git a/provider/partition-gcp/docs/anthos/README.md b/provider/partition-gcp/docs/anthos/README.md
new file mode 100644
index 000000000..096537e74
--- /dev/null
+++ b/provider/partition-gcp/docs/anthos/README.md
@@ -0,0 +1,47 @@
+## Service Configuration for Anthos
+
+## Environment variables:
+
+Define the following environment variables.
+
+Must have:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for Anthos environment | false | - |
+| `OSM_POSTGRES_URL` | ex `jdbc:postgresql://127.0.0.1:5432/postgres` | Postgres server URL | no | - |
+| `OSM_POSTGRES_USERNAME` | ex `postgres` | Postgres admin username | no | - |
+| `OSM_POSTGRES_PASSWORD` | ex `postgres` | Postgres admin password | yes | - |
+
+Defined in default application property file but possible to override:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `SERVER_SERVLET_CONTEXPATH` | `/api/partition/v1` | Servlet context path | no | - |
+| `PARTITION_PROPERTY_KIND` | ex `PartitionProperty` | Kind name to store the properties. | no | - |
+| `PARTITION_NAMESPACE` | ex `partition` | Namespace for database. | no | - |
+| `SERVICE_ACCOUNT_TAIL` | `****` |By default Partition service while authenticating the request, verifies that the email in provided token belongs to a service account from a specific project by email tail `<GOOGLE_CLOUD_PROJECT> + .iam.gserviceaccount.com`, this behavior can be changed with this variable, you may specify which email tail exactly expected.| no | - |
+
+These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+Usage of spring profiles is preferred.
+
+| `OSMDRIVER` | ex `postgres` or `datastore` | Osm driver mode that defines which storage will be used | no | - |
+| `ENVIRONMENT` | `gcp` or `anthos` | If `anthos` then authorization is disabled | no | - |
+
+
+#### for OSM - Postgres:
+
+### Schema configuration:
+
+```
+CREATE TABLE partition."PartitionProperty"(
+id text COLLATE pg_catalog."default" NOT NULL,
+pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+data jsonb NOT NULL,
+CONSTRAINT PartitionProperty_id UNIQUE (id)
+);
+CREATE INDEX PartitionProperty_dataGin ON partition."PartitionProperty" USING GIN (data);
+
+```
diff --git a/provider/partition-gcp/docs/gcp/README.md b/provider/partition-gcp/docs/gcp/README.md
new file mode 100644
index 000000000..331676b32
--- /dev/null
+++ b/provider/partition-gcp/docs/gcp/README.md
@@ -0,0 +1,38 @@
+## Service Configuration for GCP
+
+## Environment variables:
+
+Define the following environment variables.
+
+Must have:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - |
+| `PARTITION_ADMIN_ACCOUNTS` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service | yes | - |
+| `GOOGLE_CLOUD_PROJECT` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service | false | - |
+| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+
+Defined in default application property file but possible to override:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `SERVER_SERVLET_CONTEXPATH` | `/api/partition/v1` | Servlet context path | no | - |
+| `PARTITION_PROPERTY_KIND` | ex `PartitionProperty` | Kind name to store the properties. | no | - |
+| `PARTITION_NAMESPACE` | ex `partition` | Namespace for database. | no | - |
+| `SERVICE_ACCOUNT_TAIL` | `****` |By default Partition service while authenticating the request, verifies that the email in provided token belongs to a service account from a specific project by email tail `<GOOGLE_CLOUD_PROJECT> + .iam.gserviceaccount.com`, this behavior can be changed with this variable, you may specify which email tail exactly expected.| no | - |
+
+These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+Usage of spring profiles is preferred.
+
+| `OSMDRIVER` | ex `postgres` or `datastore` | Osm driver mode that defines which storage will be used | no | - |
+| `ENVIRONMENT` | `gcp` or `anthos` | If `anthos` then authorization is disabled | no | - |
+
+## Google cloud service account configuration :
+TBD
+
+| Required roles |
+| --- |
+| - |
\ No newline at end of file
diff --git a/provider/partition-gcp/pom.xml b/provider/partition-gcp/pom.xml
index 4c8990912..b0c64239e 100644
--- a/provider/partition-gcp/pom.xml
+++ b/provider/partition-gcp/pom.xml
@@ -28,12 +28,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.13.0</version>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>osm</artifactId>
- <version>0.13.0</version>
+ <version>0.14.0-rc2</version>
</dependency>
<dependency>
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
index d61b123c9..edb106306 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
@@ -17,19 +17,19 @@
package org.opengroup.osdu.partition.provider.gcp;
-import org.opengroup.osdu.core.gcp.di.GcpPartitionClientFactory;
+import org.opengroup.osdu.core.di.GcpPartitionClientFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
@ComponentScan(basePackages = {"org.opengroup.osdu"}, excludeFilters =
- @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {GcpPartitionClientFactory.class})
+@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {GcpPartitionClientFactory.class})
)
@SpringBootApplication
public class PartitionGcpApplication {
- public static void main(String[] args) {
- SpringApplication.run(PartitionGcpApplication.class, args);
- }
+ public static void main(String[] args) {
+ SpringApplication.run(PartitionGcpApplication.class, args);
+ }
}
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/GroupCache.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/GroupCache.java
deleted file mode 100644
index 94d417131..000000000
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/GroupCache.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- Copyright 2002-2021 Google LLC
- Copyright 2002-2021 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.partition.provider.gcp.cache;
-
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.common.model.entitlements.Groups;
-import org.springframework.stereotype.Component;
-
-@Component
-public class GroupCache extends VmCache<String, Groups> {
-
- public GroupCache() {
- super(30, 1000);
- }
-}
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
index 6ee1a1d26..755ceaba5 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
@@ -22,7 +22,9 @@ import lombok.Setter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+@Primary
@Configuration
@ConditionalOnProperty(name = "osmDriver", havingValue = "postgres")
@ConfigurationProperties(prefix = "osm.postgres")
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
index 337d9b06a..1f5fa5cb9 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
@@ -22,18 +22,21 @@ import org.opengroup.osdu.core.gcp.osm.model.Destination;
import org.opengroup.osdu.core.gcp.osm.model.Kind;
import org.opengroup.osdu.core.gcp.osm.model.Namespace;
import org.opengroup.osdu.partition.provider.gcp.config.PropertiesConfiguration;
+import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
+@Primary
@Service
@RequiredArgsConstructor
public class OsmPartitionDestinationProvider {
+
private final PropertiesConfiguration config;
public Destination getDestination() {
- return Destination.builder()
- .partitionId(config.getGoogleCloudProject())
- .namespace(new Namespace(config.getPartitionNamespace()))
- .kind(new Kind(config.getPartitionPropertyKind()))
- .build();
+ return Destination.builder()
+ .partitionId(config.getGoogleCloudProject())
+ .namespace(new Namespace(config.getPartitionNamespace()))
+ .kind(new Kind(config.getPartitionPropertyKind()))
+ .build();
}
}
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
index faeb51096..28db1017a 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
@@ -17,32 +17,36 @@
package org.opengroup.osdu.partition.provider.gcp.osm.config.resolver;
+import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
+
import com.google.api.gax.retrying.RetrySettings;
import com.google.cloud.TransportOptions;
import com.google.cloud.datastore.Datastore;
import com.google.cloud.datastore.DatastoreOptions;
import com.google.cloud.http.HttpTransportOptions;
+import java.io.IOException;
+import java.util.Map;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.gcp.osm.model.Destination;
import org.opengroup.osdu.core.gcp.osm.translate.datastore.DsDestinationResolution;
import org.opengroup.osdu.core.gcp.osm.translate.datastore.DsDestinationResolver;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.threeten.bp.Duration;
-import java.io.IOException;
-import java.util.Map;
-
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
+@Primary
@Component
@Scope(SCOPE_SINGLETON)
@ConditionalOnProperty(name = "osmDriver", havingValue = "datastore")
@RequiredArgsConstructor
public class OsmDatastoreDestinationResolver implements DsDestinationResolver {
- protected static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder().setMaxAttempts(6).setInitialRetryDelay(Duration.ofSeconds(10L)).setMaxRetryDelay(Duration.ofSeconds(32L)).setRetryDelayMultiplier(2.0D).setTotalTimeout(Duration.ofSeconds(50L)).setInitialRpcTimeout(Duration.ofSeconds(50L)).setRpcTimeoutMultiplier(1.0D).setMaxRpcTimeout(Duration.ofSeconds(50L)).build();
+ protected static final RetrySettings RETRY_SETTINGS =
+ RetrySettings.newBuilder().setMaxAttempts(6).setInitialRetryDelay(Duration.ofSeconds(10L)).setMaxRetryDelay(Duration.ofSeconds(32L))
+ .setRetryDelayMultiplier(2.0D).setTotalTimeout(Duration.ofSeconds(50L)).setInitialRpcTimeout(Duration.ofSeconds(50L)).setRpcTimeoutMultiplier(1.0D)
+ .setMaxRpcTimeout(Duration.ofSeconds(50L)).build();
protected static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder().setReadTimeout(30000).build();
private final Map<String, Datastore> datastoreCache;
@@ -58,20 +62,20 @@ public class OsmDatastoreDestinationResolver implements DsDestinationResolver {
Datastore datastore = datastoreCache.computeIfAbsent(projectId, key -> getDatastoreFor(destination, key, projectId));
return DsDestinationResolution.builder()
- .projectId(datastore.getOptions().getProjectId())
- .datastore(datastore)
- .build();
+ .projectId(datastore.getOptions().getProjectId())
+ .datastore(datastore)
+ .build();
}
private Datastore getDatastoreFor(Destination destination, String key, String projectId) {
return datastoreCache.computeIfAbsent(key, k ->
- DatastoreOptions.newBuilder()
- .setRetrySettings(RETRY_SETTINGS)
- .setTransportOptions(TRANSPORT_OPTIONS)
- .setProjectId(projectId)
- .setNamespace(destination.getNamespace().getName())
- .build()
- .getService());
+ DatastoreOptions.newBuilder()
+ .setRetrySettings(RETRY_SETTINGS)
+ .setTransportOptions(TRANSPORT_OPTIONS)
+ .setProjectId(projectId)
+ .setNamespace(destination.getNamespace().getName())
+ .build()
+ .getService());
}
@Override
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
index ac3464f81..413859875 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
@@ -17,7 +17,13 @@
package org.opengroup.osdu.partition.provider.gcp.osm.config.resolver;
+import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
+
import com.zaxxer.hikari.HikariDataSource;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.PreDestroy;
+import javax.sql.DataSource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.gcp.osm.model.Destination;
@@ -26,17 +32,11 @@ import org.opengroup.osdu.core.gcp.osm.translate.postgresql.PgDestinationResolve
import org.opengroup.osdu.partition.provider.gcp.config.PostgresOsmConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
-import javax.annotation.PreDestroy;
-import javax.sql.DataSource;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
-
+@Primary
@Component
@Scope(SCOPE_SINGLETON)
@ConditionalOnProperty(name = "osmDriver", havingValue = "postgres")
@@ -66,18 +66,18 @@ public class OsmPostgresDestinationResolver implements PgDestinationResolver {
}
return PgDestinationResolution.builder()
- .datasource(dataSource)
- .build();
+ .datasource(dataSource)
+ .build();
}
private DataSource buildDataSourceFromProperties(String partitionId) {
DataSource dataSource;
dataSource = DataSourceBuilder.create()
- .driverClassName(DRIVER_CLASS_NAME)
- .url(properties.getUrl())
- .username(properties.getUsername())
- .password(properties.getPassword())
- .build();
+ .driverClassName(DRIVER_CLASS_NAME)
+ .url(properties.getUrl())
+ .username(properties.getUsername())
+ .password(properties.getPassword())
+ .build();
HikariDataSource hikariDataSource = (HikariDataSource) dataSource;
hikariDataSource.setMaximumPoolSize(properties.getMaximumPoolSize());
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
index e808d77f3..3de531179 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
@@ -17,11 +17,17 @@
package org.opengroup.osdu.partition.provider.gcp.service;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
import lombok.RequiredArgsConstructor;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.partition.logging.AuditLogger;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
@@ -31,9 +37,6 @@ import org.opengroup.osdu.partition.provider.interfaces.IPartitionService;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
-import java.io.IOException;
-import java.util.*;
-
@Service
@RequiredArgsConstructor
public class PartitionServiceImpl implements IPartitionService {
@@ -43,8 +46,6 @@ public class PartitionServiceImpl implements IPartitionService {
private final OsmPartitionPropertyRepository partitionPropertyEntityRepository;
- private final IKmsClient kmsClient;
-
private final AuditLogger auditLogger;
private final ICache<String, PartitionInfo> partitionServiceCache;
@@ -65,7 +66,6 @@ public class PartitionServiceImpl implements IPartitionService {
for (Map.Entry<String, Property> entry : partitionInfo.getProperties().entrySet()) {
PartitionPropertyEntity entity = new PartitionPropertyEntity(partitionId,
entry.getKey(), entry.getValue());
- encryptPartitionPropertyEntityIfNeeded(entity);
partitionProperties.add(entity);
}
this.partitionPropertyEntityRepository.saveAll(partitionProperties);
@@ -78,18 +78,6 @@ public class PartitionServiceImpl implements IPartitionService {
return pi;
}
- private void encryptPartitionPropertyEntityIfNeeded(PartitionPropertyEntity entity) {
- if (entity.isSensitive()) {
- String propertyValue = entity.getValue().toString();
- try {
- entity.setValue(this.kmsClient.encryptString(propertyValue));
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, UNKNOWN_ERROR_REASON,
- e.getMessage());
- }
- }
- }
-
@Override
public PartitionInfo updatePartition(String partitionId, PartitionInfo partitionInfo) {
if (partitionInfo.getProperties().containsKey("id")) {
@@ -114,7 +102,6 @@ public class PartitionServiceImpl implements IPartitionService {
} else {
entity = new PartitionPropertyEntity(partitionId, entry.getKey(), entry.getValue());
}
- encryptPartitionPropertyEntityIfNeeded(entity);
partitionProperties.add(entity);
}
this.partitionPropertyEntityRepository.saveAll(partitionProperties);
@@ -131,9 +118,6 @@ public class PartitionServiceImpl implements IPartitionService {
if (Objects.isNull(pi)) {
pi = getEncryptedPartition(partitionId);
- for (Property property : pi.getProperties().values()) {
- decryptPartitionPropertyIfNeeded(property);
- }
if (Objects.nonNull(pi)) {
partitionServiceCache.put(partitionId, pi);
@@ -163,18 +147,6 @@ public class PartitionServiceImpl implements IPartitionService {
return partitionInfo;
}
- private void decryptPartitionPropertyIfNeeded(Property property) {
- if (property.isSensitive()) {
- String propertyValue = property.getValue().toString();
- try {
- property.setValue(this.kmsClient.decryptString(propertyValue));
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, UNKNOWN_ERROR_REASON,
- e.getMessage());
- }
- }
- }
-
@Override
public boolean deletePartition(String partitionId) {
if (!this.partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) {
diff --git a/provider/partition-gcp/src/main/resources/application-anthos.properties b/provider/partition-gcp/src/main/resources/application-anthos.properties
new file mode 100644
index 000000000..6d9a9e8e7
--- /dev/null
+++ b/provider/partition-gcp/src/main/resources/application-anthos.properties
@@ -0,0 +1,25 @@
+#
+# Copyright 2020-2022 Google LLC
+# Copyright 2020-2022 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+osmDriver=postgres
+environment=anthos
+osm.postgres.url=jdbc:postgresql://127.0.0.1:5432/postgres
+osm.postgres.username=postgres
+osm.postgres.password=postgres
+
+#stub value, not used on anthos, but will cause NPE if not set
+google-cloud-project=anthos
\ No newline at end of file
diff --git a/provider/partition-gcp/src/main/resources/application-gcp.properties b/provider/partition-gcp/src/main/resources/application-gcp.properties
new file mode 100644
index 000000000..3828f9401
--- /dev/null
+++ b/provider/partition-gcp/src/main/resources/application-gcp.properties
@@ -0,0 +1,22 @@
+#
+# Copyright 2020-2022 Google LLC
+# Copyright 2020-2022 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+osmDriver=datastore
+environment=gcp
+partition-admin-accounts=${PARTITION_ADMIN_ACCOUNTS}
+google-cloud-project=${GOOGLE_CLOUD_PROJECT}
+google-audiences=${GOOGLE_AUDIENCES}
diff --git a/provider/partition-gcp/src/main/resources/application-osm-postgres.properties b/provider/partition-gcp/src/main/resources/application-osm-postgres.properties
deleted file mode 100644
index b700407e2..000000000
--- a/provider/partition-gcp/src/main/resources/application-osm-postgres.properties
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Copyright 2020-2021 Google LLC
-# Copyright 2020-2021 EPAM Systems, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-google-cloud-project=osdu-cicd-epam
-
-LOG_PREFIX=partition
-server.servlet.contextPath=/api/partition/v1
-server.port=8080
-springfox.documentation.swagger.v2.path=/api-docs
-authorize-api=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
-spring.cloud.gcp.datastore.namespace=${partition-namespace}
-#ACCEPT_HTTP=true
-environment=gcp
-
-cache-expiration=1
-cache-maxSize=1000
-
-key-ring=csqp
-kms-key=searchService
-
-KEY_RING=${key-ring}
-KMS_KEY=${kms-key}
-GOOGLE_CLOUD_PROJECT=${google-cloud-project}
-google-audiences=123.apps.googleusercontent.com
-partition-admin-accounts=osdu-gcp-sa
-partition-property-kind=PartitionProperty
-partition-namespace=partition
-
-#logging configuration
-logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
-
-management.endpoint.health.show-details=always
-management.health.diskspace.enabled=false
-
-osmDriver=postgres
-
-osm.postgres.url=jdbc:postgresql://127.0.0.1:5432/postgres
-osm.postgres.username=postgres
-osm.postgres.password=postgres
diff --git a/provider/partition-gcp/src/main/resources/application.properties b/provider/partition-gcp/src/main/resources/application.properties
index bce9e7f12..bff52e44e 100644
--- a/provider/partition-gcp/src/main/resources/application.properties
+++ b/provider/partition-gcp/src/main/resources/application.properties
@@ -15,35 +15,24 @@
# limitations under the License.
#
-google-cloud-project=osdu-cicd-epam
-
-LOG_PREFIX=partition
+#server config
server.servlet.contextPath=/api/partition/v1
server.port=8080
springfox.documentation.swagger.v2.path=/api-docs
-authorize-api=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
-spring.cloud.gcp.datastore.namespace=${partition-namespace}
-#ACCEPT_HTTP=true
-environment=gcp
+management.endpoint.health.show-details=always
+management.health.diskspace.enabled=false
+#logging configuration
+logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
+LOG_PREFIX=partition
+
+#cache config
cache-expiration=1
cache-maxSize=1000
-key-ring=csqp
-kms-key=searchService
-
-KEY_RING=${key-ring}
-KMS_KEY=${kms-key}
-GOOGLE_CLOUD_PROJECT=${google-cloud-project}
-google-audiences=123.apps.googleusercontent.com
-partition-admin-accounts=osdu-gcp-sa
+#service config
partition-property-kind=PartitionProperty
partition-namespace=partition
-#logging configuration
-logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
-management.endpoint.health.show-details=always
-management.health.diskspace.enabled=false
-osmDriver=datastore
diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
index 3715aee7f..050af8c25 100644
--- a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
+++ b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
@@ -17,6 +17,18 @@
package org.opengroup.osdu.partition.provider.gcp.service;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -24,22 +36,12 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.partition.logging.AuditLogger;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
import org.opengroup.osdu.partition.provider.gcp.model.PartitionPropertyEntity;
import org.opengroup.osdu.partition.provider.gcp.osm.repository.OsmPartitionPropertyRepository;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.*;
-
@ExtendWith(MockitoExtension.class)
public class PartitionServiceImplCacheTest {
@@ -61,7 +63,6 @@ public class PartitionServiceImplCacheTest {
public void setup() {
partitionServiceImpl = new PartitionServiceImpl(
partitionPropertyEntityRepository,
- mock(IKmsClient.class),
auditLogger,
partitionServiceCache,
partitionListCache
diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
index 213aafa89..1abda3af6 100644
--- a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
+++ b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
@@ -17,6 +17,16 @@
package org.opengroup.osdu.partition.provider.gcp.service;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -24,25 +34,12 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.partition.logging.AuditLogger;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
import org.opengroup.osdu.partition.provider.gcp.model.PartitionPropertyEntity;
import org.opengroup.osdu.partition.provider.gcp.osm.repository.OsmPartitionPropertyRepository;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
@ExtendWith(MockitoExtension.class)
public class PartitionServiceImplTest {
@@ -74,7 +71,6 @@ public class PartitionServiceImplTest {
public void setup() {
partitionServiceImpl = new PartitionServiceImpl(
partitionPropertyEntityRepository,
- mock(IKmsClient.class),
auditLogger,
partitionServiceCache,
partitionListCache
--
GitLab