diff --git a/devops/gcp/configmap/templates/partition-variables.yml b/devops/gcp/configmap/templates/partition-variables.yml
index 3d379e79405523eee44f3a9c8a020f9e38049e42..e76b8c0f5495fbb05230e3e8d05d3d6d88ac7781 100644
--- a/devops/gcp/configmap/templates/partition-variables.yml
+++ b/devops/gcp/configmap/templates/partition-variables.yml
@@ -9,7 +9,6 @@ data:
GOOGLE_CLOUD_PROJECT: "{{ .Values.data.google_cloud_project }}"
PARTITION_ADMIN_ACCOUNTS: "{{ .Values.data.partition_admin_accounts }}"
GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
- KEY_RING: "{{ .Values.data.key_ring }}"
- KMS_KEY: "{{ .Values.data.kms_key }}"
LOG_LEVEL: "{{ .Values.data.log_level }}"
SERVICE_ACCOUNT_TAIL: "{{ .Values.data.service_account_tail }}"
+ SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml
index 460bc78f2f88be73d08c8d24ce167b290ab0888f..61d4a23e087a945e4abba92b4f908e1bd9b50a6d 100644
--- a/devops/gcp/configmap/values.yaml
+++ b/devops/gcp/configmap/values.yaml
@@ -2,10 +2,9 @@ data:
google_cloud_project: ""
partition_admin_accounts: ""
google_audiences: ""
- key_ring: ""
- kms_key: ""
log_level: "INFO"
service_account_tail: ".iam.gserviceaccount.com"
+ spring_profiles_active: "gcp"
conf:
diff --git a/provider/partition-gcp/README.md b/provider/partition-gcp/README.md
index 95b635050b8acda2dd8a571573b7f220f317d7ac..ba368928f686e75dff37e03debd4d69d5cac2f1f 100644
--- a/provider/partition-gcp/README.md
+++ b/provider/partition-gcp/README.md
@@ -33,106 +33,6 @@ Pre-requisites
* Lombok 1.16 or later
* Maven
-## Mapper tuning mechanisms
-
-This service uses specific implementations of DestinationResolvers. A total of 2 resolvers have been implemented, which are divided into two groups:
-
-### for universal technologies:
-
-- for Postgres: osm/config/resolver/OsmPostgresDestinationResolver.java
-
-#### Their algorithms are as follows:
-
-- incoming Destination carries data-partition-id
-- resolver accesses the Partition service and gets PartitionInfo
-- from PartitionInfo resolver retrieves properties for the connection: URL, username, password etc.
-- resolver creates a data source, connects to the resource, remembers the datasource
-- resolver gives the datasource to the mapper in the Resolution object
-
-### for native Google Cloud technologies:
-
-- for Datastore: osm/config/resolver/OsmDatastoreDestinationResolver.java
-
-#### Their algorithms are similar,
-
-Except that they do not receive special properties from the Partition service for connection, because the location of
-the resources is unambiguously known - they are in the GCP project. And credentials are also not needed - access to data
-is made on behalf of the Google Identity SA under which the service itself is launched. Therefore, resolver takes only
-the value of the **projectId** property from PartitionInfo and uses it to connect to a resource in the corresponding GCP
-project.
-
-### Installation
-In order to run the service locally or remotely, you will need to have the following environment variables defined.
-
-| name | value | description | sensitive? | source |
-| --- | --- | --- | --- | --- |
-| `LOG_PREFIX` | `service` | Logging prefix | no | - |
-| `SERVER_SERVLET_CONTEXPATH` | `/api/partition/v1` | Servlet context path | no | - |
-| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
-| `GOOGLE_CLOUD_PROJECT` | ex `osdu-cicd-epam` | Google Cloud Project Id| no | output of infrastructure deployment |
-| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
-| `PARTITION_ADMIN_ACCOUNTS` | ex `admin@domen.iam.gserviceaccount.com,osdu-gcp-sa,workload-identity` | List of partition admin account emails, could be in full form like `admin@domen.iam.gserviceaccount.com` or in `starts with` pattern like `osdu-gcp-sa`| no | - |
-| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `KEY_RING` | ex `csqp` | A key ring holds keys in a specific Google Cloud location and permit us to manage access control on groups of keys | yes | https://cloud.google.com/kms/docs/resource-hierarchy#key_rings |
-| `KMS_KEY` | ex `partitionService` | A key exists on one key ring linked to a specific location. | yes | https://cloud.google.com/kms/docs/resource-hierarchy#key_rings |
-| `PARTITION_PROPERTY_KIND` | ex `PartitionProperty` | Kind name to store the properties. | no | - |
-| `PARTITION_NAMESPACE` | ex `partition` | Namespace for database. | no | - |
-| `osmDriver` | ex `postgres` or `datastore` | Osm driver mode that defines which storage will be used | no | - |
-| `osm.postgres.url` | ex `jdbc:postgresql://127.0.0.1:5432/postgres` | Postgres server URL | no | - |
-| `osm.postgres.username` | ex `postgres` | Postgres admin username | no | - |
-| `osm.postgres.password` | ex `postgres` | Postgres admin password | yes | - |
-| `ENVIRONMENT` | `gcp` or `anthos` | If `anthos` then authorization is disabled | no | - |
-| `SERVICE_ACCOUNT_TAIL` | `****` |By default Partition service while authenticating the request, verifies that the email in provided token belongs to a service account from a specific project by email tail `<GOOGLE_CLOUD_PROJECT> + .iam.gserviceaccount.com`, this behavior can be changed with this variable, you may specify which email tail exactly expected.| no | - |
-
-## Configuring mappers' Datasources
-
-When using non-Google-Cloud-native technologies, property sets must be defined on the Partition service as part of
-PartitionInfo for each Tenant.
-
-They are specific to each storage technology:
-
-#### for OSM - Postgres:
-
-**database structure**
-OSM works with data logically organized as "partition"->"namespace"->"kind"->"record"->"columns". The above sequence
-describes how it is named in Google Datastore, where "partition" maps to "GCP project".
-
-This is how **Postgres** OSM driver does. Notice, the above hierarchy has been kept, but Postgres uses alternative entities
-for it.
-
-| Datastore hierarchy level | | Postgres alternative used |
-|---------------------------|-----|----------------------------|
-| partition (GCP project) | == | Postgres server URL |
-| namespace | == | Schema |
-| kind | == | Table |
-| record | == | '<multiple table records>' |
-| columns | == | id, data (jsonb) |
-
-As we can see in the above table, Postgres uses different approach in storing business data in records. Not like
-Datastore, which segments data into multiple physical columns, Postgres organises them into the single JSONB "data"
-column. It allows provisioning new data registers easily not taking care about specifics of certain registers structure.
-In the current OSM version (as on December'21) the Postgres OSM driver is not able to create new tables in runtime.
-
-So this is a responsibility of DevOps / CICD to provision all required SQL tables (for all required data kinds) when on new
-environment or tenant provisioning when using Postgres. Detailed instructions (with examples) for creating new tables is
-in the **OSM module Postgres driver README.md** `org/opengroup/osdu/core/gcp/osm/translate/postgresql/README.md`
-
-As a quick shortcut, this example snippet can be used by DevOps DBA:
-
-* `exampleschema` equals to `PARTITION_NAMESPACE`
-* `ExampleKind` equals to `PARTITION_PROPERTY_KIND`
-
-```postgres-psql
---CREATE SCHEMA "<exampleschema>";
-CREATE TABLE <exampleschema>."<ExampleKind>"(
- id text COLLATE pg_catalog."default" NOT NULL,
- pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
- data jsonb NOT NULL,
- CONSTRAINT <ExampleKind>_id UNIQUE (id)
-);
-CREATE INDEX <ExampleKind>_datagin ON <exampleschema>."<ExampleKind>" USING GIN (data);
-```
-
### Run Locally
Check that maven is installed:
@@ -246,26 +146,6 @@ Partition Service is compatible with App Engine Flexible Environment and Cloud R
* To deploy into App Engine, please, use this documentation:
https://cloud.google.com/appengine/docs/flexible/java/quickstart
-#### Cloud KMS Setup
-
-Enable cloud KMS on master project.
-
-Create king ring and key in the ***master project***
-
-```bash
- gcloud services enable cloudkms.googleapis.com
- export KEYRING_NAME="csqp"
- export CRYPTOKEY_NAME="partionService"
- gcloud kms keyrings create $KEYRING_NAME --location global
- gcloud kms keys create $CRYPTOKEY_NAME --location global \
- --keyring $KEYRING_NAME \
- --purpose encryption
-```
-
-Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the used **service account** by Partition Service of the ***master project*** through IAM - Role tab.
-
-Add "Cloud KMS Encrypt/Decrypt" role to the used **service account** by Partition Service of the ***master project*** through IAM - Role tab.
-
## Licence
Copyright © Google LLC
Copyright © EPAM Systems
diff --git a/provider/partition-gcp/docs/anthos/README.md b/provider/partition-gcp/docs/anthos/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..096537e7409da9a4816b00b8b906a335dd645b7f
--- /dev/null
+++ b/provider/partition-gcp/docs/anthos/README.md
@@ -0,0 +1,47 @@
+## Service Configuration for Anthos
+
+## Environment variables:
+
+Define the following environment variables.
+
+Must have:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for Anthos environment | false | - |
+| `OSM_POSTGRES_URL` | ex `jdbc:postgresql://127.0.0.1:5432/postgres` | Postgres server URL | no | - |
+| `OSM_POSTGRES_USERNAME` | ex `postgres` | Postgres admin username | no | - |
+| `OSM_POSTGRES_PASSWORD` | ex `postgres` | Postgres admin password | yes | - |
+
+Defined in default application property file but possible to override:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `SERVER_SERVLET_CONTEXPATH` | `/api/partition/v1` | Servlet context path | no | - |
+| `PARTITION_PROPERTY_KIND` | ex `PartitionProperty` | Kind name to store the properties. | no | - |
+| `PARTITION_NAMESPACE` | ex `partition` | Namespace for database. | no | - |
+| `SERVICE_ACCOUNT_TAIL` | `****` |By default Partition service while authenticating the request, verifies that the email in provided token belongs to a service account from a specific project by email tail `<GOOGLE_CLOUD_PROJECT> + .iam.gserviceaccount.com`, this behavior can be changed with this variable, you may specify which email tail exactly expected.| no | - |
+
+These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+Usage of spring profiles is preferred.
+
+| `OSMDRIVER` | ex `postgres` or `datastore` | Osm driver mode that defines which storage will be used | no | - |
+| `ENVIRONMENT` | `gcp` or `anthos` | If `anthos` then authorization is disabled | no | - |
+
+
+#### for OSM - Postgres:
+
+### Schema configuration:
+
+```
+CREATE TABLE partition."PartitionProperty"(
+id text COLLATE pg_catalog."default" NOT NULL,
+pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+data jsonb NOT NULL,
+CONSTRAINT PartitionProperty_id UNIQUE (id)
+);
+CREATE INDEX PartitionProperty_dataGin ON partition."PartitionProperty" USING GIN (data);
+
+```
diff --git a/provider/partition-gcp/docs/gcp/README.md b/provider/partition-gcp/docs/gcp/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..331676b32307186fcb4a978b5ca4c047a3c1c63c
--- /dev/null
+++ b/provider/partition-gcp/docs/gcp/README.md
@@ -0,0 +1,38 @@
+## Service Configuration for GCP
+
+## Environment variables:
+
+Define the following environment variables.
+
+Must have:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - |
+| `PARTITION_ADMIN_ACCOUNTS` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service | yes | - |
+| `GOOGLE_CLOUD_PROJECT` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service | false | - |
+| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+
+Defined in default application property file but possible to override:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `SERVER_SERVLET_CONTEXPATH` | `/api/partition/v1` | Servlet context path | no | - |
+| `PARTITION_PROPERTY_KIND` | ex `PartitionProperty` | Kind name to store the properties. | no | - |
+| `PARTITION_NAMESPACE` | ex `partition` | Namespace for database. | no | - |
+| `SERVICE_ACCOUNT_TAIL` | `****` |By default Partition service while authenticating the request, verifies that the email in provided token belongs to a service account from a specific project by email tail `<GOOGLE_CLOUD_PROJECT> + .iam.gserviceaccount.com`, this behavior can be changed with this variable, you may specify which email tail exactly expected.| no | - |
+
+These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+Usage of spring profiles is preferred.
+
+| `OSMDRIVER` | ex `postgres` or `datastore` | Osm driver mode that defines which storage will be used | no | - |
+| `ENVIRONMENT` | `gcp` or `anthos` | If `anthos` then authorization is disabled | no | - |
+
+## Google cloud service account configuration :
+TBD
+
+| Required roles |
+| --- |
+| - |
\ No newline at end of file
diff --git a/provider/partition-gcp/pom.xml b/provider/partition-gcp/pom.xml
index 4c89909124ab20375fb9ef481cff41618b3df67c..b0c64239e7d149ae6c5dc2d29dda0674316fff42 100644
--- a/provider/partition-gcp/pom.xml
+++ b/provider/partition-gcp/pom.xml
@@ -28,12 +28,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.13.0</version>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>osm</artifactId>
- <version>0.13.0</version>
+ <version>0.14.0-rc2</version>
</dependency>
<dependency>
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
index d61b123c99cb61348aab54d642aeed74fc366742..edb106306fa934fad4aed5560fe044586f3a2db9 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
@@ -17,19 +17,19 @@
package org.opengroup.osdu.partition.provider.gcp;
-import org.opengroup.osdu.core.gcp.di.GcpPartitionClientFactory;
+import org.opengroup.osdu.core.di.GcpPartitionClientFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
@ComponentScan(basePackages = {"org.opengroup.osdu"}, excludeFilters =
- @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {GcpPartitionClientFactory.class})
+@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {GcpPartitionClientFactory.class})
)
@SpringBootApplication
public class PartitionGcpApplication {
- public static void main(String[] args) {
- SpringApplication.run(PartitionGcpApplication.class, args);
- }
+ public static void main(String[] args) {
+ SpringApplication.run(PartitionGcpApplication.class, args);
+ }
}
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/GroupCache.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/GroupCache.java
deleted file mode 100644
index 94d417131349b5b8d14f66f3af18e7deb928aabf..0000000000000000000000000000000000000000
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/GroupCache.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- Copyright 2002-2021 Google LLC
- Copyright 2002-2021 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.partition.provider.gcp.cache;
-
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.common.model.entitlements.Groups;
-import org.springframework.stereotype.Component;
-
-@Component
-public class GroupCache extends VmCache<String, Groups> {
-
- public GroupCache() {
- super(30, 1000);
- }
-}
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
index 6ee1a1d267848305dc41a7b53f6589ce8e9420af..755ceaba598d352540e4f8da0655ba49cc3077c2 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
@@ -22,7 +22,9 @@ import lombok.Setter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+@Primary
@Configuration
@ConditionalOnProperty(name = "osmDriver", havingValue = "postgres")
@ConfigurationProperties(prefix = "osm.postgres")
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
index 337d9b06a53ce19dc8c8dbed3f9e50be1d666566..1f5fa5cb98c06668dc606ae561025eb293a87a0b 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
@@ -22,18 +22,21 @@ import org.opengroup.osdu.core.gcp.osm.model.Destination;
import org.opengroup.osdu.core.gcp.osm.model.Kind;
import org.opengroup.osdu.core.gcp.osm.model.Namespace;
import org.opengroup.osdu.partition.provider.gcp.config.PropertiesConfiguration;
+import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
+@Primary
@Service
@RequiredArgsConstructor
public class OsmPartitionDestinationProvider {
+
private final PropertiesConfiguration config;
public Destination getDestination() {
- return Destination.builder()
- .partitionId(config.getGoogleCloudProject())
- .namespace(new Namespace(config.getPartitionNamespace()))
- .kind(new Kind(config.getPartitionPropertyKind()))
- .build();
+ return Destination.builder()
+ .partitionId(config.getGoogleCloudProject())
+ .namespace(new Namespace(config.getPartitionNamespace()))
+ .kind(new Kind(config.getPartitionPropertyKind()))
+ .build();
}
}
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
index faeb51096b563f9a574e65c08c2a841175e1b1c5..28db1017a972fb1c00478befd6d6fbc8c72a65d9 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
@@ -17,32 +17,36 @@
package org.opengroup.osdu.partition.provider.gcp.osm.config.resolver;
+import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
+
import com.google.api.gax.retrying.RetrySettings;
import com.google.cloud.TransportOptions;
import com.google.cloud.datastore.Datastore;
import com.google.cloud.datastore.DatastoreOptions;
import com.google.cloud.http.HttpTransportOptions;
+import java.io.IOException;
+import java.util.Map;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.gcp.osm.model.Destination;
import org.opengroup.osdu.core.gcp.osm.translate.datastore.DsDestinationResolution;
import org.opengroup.osdu.core.gcp.osm.translate.datastore.DsDestinationResolver;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.threeten.bp.Duration;
-import java.io.IOException;
-import java.util.Map;
-
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
+@Primary
@Component
@Scope(SCOPE_SINGLETON)
@ConditionalOnProperty(name = "osmDriver", havingValue = "datastore")
@RequiredArgsConstructor
public class OsmDatastoreDestinationResolver implements DsDestinationResolver {
- protected static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder().setMaxAttempts(6).setInitialRetryDelay(Duration.ofSeconds(10L)).setMaxRetryDelay(Duration.ofSeconds(32L)).setRetryDelayMultiplier(2.0D).setTotalTimeout(Duration.ofSeconds(50L)).setInitialRpcTimeout(Duration.ofSeconds(50L)).setRpcTimeoutMultiplier(1.0D).setMaxRpcTimeout(Duration.ofSeconds(50L)).build();
+ protected static final RetrySettings RETRY_SETTINGS =
+ RetrySettings.newBuilder().setMaxAttempts(6).setInitialRetryDelay(Duration.ofSeconds(10L)).setMaxRetryDelay(Duration.ofSeconds(32L))
+ .setRetryDelayMultiplier(2.0D).setTotalTimeout(Duration.ofSeconds(50L)).setInitialRpcTimeout(Duration.ofSeconds(50L)).setRpcTimeoutMultiplier(1.0D)
+ .setMaxRpcTimeout(Duration.ofSeconds(50L)).build();
protected static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder().setReadTimeout(30000).build();
private final Map<String, Datastore> datastoreCache;
@@ -58,20 +62,20 @@ public class OsmDatastoreDestinationResolver implements DsDestinationResolver {
Datastore datastore = datastoreCache.computeIfAbsent(projectId, key -> getDatastoreFor(destination, key, projectId));
return DsDestinationResolution.builder()
- .projectId(datastore.getOptions().getProjectId())
- .datastore(datastore)
- .build();
+ .projectId(datastore.getOptions().getProjectId())
+ .datastore(datastore)
+ .build();
}
private Datastore getDatastoreFor(Destination destination, String key, String projectId) {
return datastoreCache.computeIfAbsent(key, k ->
- DatastoreOptions.newBuilder()
- .setRetrySettings(RETRY_SETTINGS)
- .setTransportOptions(TRANSPORT_OPTIONS)
- .setProjectId(projectId)
- .setNamespace(destination.getNamespace().getName())
- .build()
- .getService());
+ DatastoreOptions.newBuilder()
+ .setRetrySettings(RETRY_SETTINGS)
+ .setTransportOptions(TRANSPORT_OPTIONS)
+ .setProjectId(projectId)
+ .setNamespace(destination.getNamespace().getName())
+ .build()
+ .getService());
}
@Override
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
index ac3464f8111cf2df968609fad4b9f0dc551216ed..41385987501aeadd8d1783b3762abb025b331009 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
@@ -17,7 +17,13 @@
package org.opengroup.osdu.partition.provider.gcp.osm.config.resolver;
+import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
+
import com.zaxxer.hikari.HikariDataSource;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.PreDestroy;
+import javax.sql.DataSource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.gcp.osm.model.Destination;
@@ -26,17 +32,11 @@ import org.opengroup.osdu.core.gcp.osm.translate.postgresql.PgDestinationResolve
import org.opengroup.osdu.partition.provider.gcp.config.PostgresOsmConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
-import javax.annotation.PreDestroy;
-import javax.sql.DataSource;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
-
-
+@Primary
@Component
@Scope(SCOPE_SINGLETON)
@ConditionalOnProperty(name = "osmDriver", havingValue = "postgres")
@@ -66,18 +66,18 @@ public class OsmPostgresDestinationResolver implements PgDestinationResolver {
}
return PgDestinationResolution.builder()
- .datasource(dataSource)
- .build();
+ .datasource(dataSource)
+ .build();
}
private DataSource buildDataSourceFromProperties(String partitionId) {
DataSource dataSource;
dataSource = DataSourceBuilder.create()
- .driverClassName(DRIVER_CLASS_NAME)
- .url(properties.getUrl())
- .username(properties.getUsername())
- .password(properties.getPassword())
- .build();
+ .driverClassName(DRIVER_CLASS_NAME)
+ .url(properties.getUrl())
+ .username(properties.getUsername())
+ .password(properties.getPassword())
+ .build();
HikariDataSource hikariDataSource = (HikariDataSource) dataSource;
hikariDataSource.setMaximumPoolSize(properties.getMaximumPoolSize());
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
index e808d77f3117fbdbc1c899f2ee18a06f0ee212df..3de5311794084efe2bf47d7a66dd4845ff30a714 100644
--- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
+++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
@@ -17,11 +17,17 @@
package org.opengroup.osdu.partition.provider.gcp.service;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
import lombok.RequiredArgsConstructor;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.partition.logging.AuditLogger;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
@@ -31,9 +37,6 @@ import org.opengroup.osdu.partition.provider.interfaces.IPartitionService;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
-import java.io.IOException;
-import java.util.*;
-
@Service
@RequiredArgsConstructor
public class PartitionServiceImpl implements IPartitionService {
@@ -43,8 +46,6 @@ public class PartitionServiceImpl implements IPartitionService {
private final OsmPartitionPropertyRepository partitionPropertyEntityRepository;
- private final IKmsClient kmsClient;
-
private final AuditLogger auditLogger;
private final ICache<String, PartitionInfo> partitionServiceCache;
@@ -65,7 +66,6 @@ public class PartitionServiceImpl implements IPartitionService {
for (Map.Entry<String, Property> entry : partitionInfo.getProperties().entrySet()) {
PartitionPropertyEntity entity = new PartitionPropertyEntity(partitionId,
entry.getKey(), entry.getValue());
- encryptPartitionPropertyEntityIfNeeded(entity);
partitionProperties.add(entity);
}
this.partitionPropertyEntityRepository.saveAll(partitionProperties);
@@ -78,18 +78,6 @@ public class PartitionServiceImpl implements IPartitionService {
return pi;
}
- private void encryptPartitionPropertyEntityIfNeeded(PartitionPropertyEntity entity) {
- if (entity.isSensitive()) {
- String propertyValue = entity.getValue().toString();
- try {
- entity.setValue(this.kmsClient.encryptString(propertyValue));
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, UNKNOWN_ERROR_REASON,
- e.getMessage());
- }
- }
- }
-
@Override
public PartitionInfo updatePartition(String partitionId, PartitionInfo partitionInfo) {
if (partitionInfo.getProperties().containsKey("id")) {
@@ -114,7 +102,6 @@ public class PartitionServiceImpl implements IPartitionService {
} else {
entity = new PartitionPropertyEntity(partitionId, entry.getKey(), entry.getValue());
}
- encryptPartitionPropertyEntityIfNeeded(entity);
partitionProperties.add(entity);
}
this.partitionPropertyEntityRepository.saveAll(partitionProperties);
@@ -131,9 +118,6 @@ public class PartitionServiceImpl implements IPartitionService {
if (Objects.isNull(pi)) {
pi = getEncryptedPartition(partitionId);
- for (Property property : pi.getProperties().values()) {
- decryptPartitionPropertyIfNeeded(property);
- }
if (Objects.nonNull(pi)) {
partitionServiceCache.put(partitionId, pi);
@@ -163,18 +147,6 @@ public class PartitionServiceImpl implements IPartitionService {
return partitionInfo;
}
- private void decryptPartitionPropertyIfNeeded(Property property) {
- if (property.isSensitive()) {
- String propertyValue = property.getValue().toString();
- try {
- property.setValue(this.kmsClient.decryptString(propertyValue));
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, UNKNOWN_ERROR_REASON,
- e.getMessage());
- }
- }
- }
-
@Override
public boolean deletePartition(String partitionId) {
if (!this.partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) {
diff --git a/provider/partition-gcp/src/main/resources/application-anthos.properties b/provider/partition-gcp/src/main/resources/application-anthos.properties
new file mode 100644
index 0000000000000000000000000000000000000000..6d9a9e8e70470a06dd4bdab10ada25aa0a6fcaeb
--- /dev/null
+++ b/provider/partition-gcp/src/main/resources/application-anthos.properties
@@ -0,0 +1,25 @@
+#
+# Copyright 2020-2022 Google LLC
+# Copyright 2020-2022 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+osmDriver=postgres
+environment=anthos
+osm.postgres.url=jdbc:postgresql://127.0.0.1:5432/postgres
+osm.postgres.username=postgres
+osm.postgres.password=postgres
+
+#stub value, not used on anthos, but will cause NPE if not set
+google-cloud-project=anthos
\ No newline at end of file
diff --git a/provider/partition-gcp/src/main/resources/application-gcp.properties b/provider/partition-gcp/src/main/resources/application-gcp.properties
new file mode 100644
index 0000000000000000000000000000000000000000..3828f94019adbec89eedf118850261f4df68b80a
--- /dev/null
+++ b/provider/partition-gcp/src/main/resources/application-gcp.properties
@@ -0,0 +1,22 @@
+#
+# Copyright 2020-2022 Google LLC
+# Copyright 2020-2022 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+osmDriver=datastore
+environment=gcp
+partition-admin-accounts=${PARTITION_ADMIN_ACCOUNTS}
+google-cloud-project=${GOOGLE_CLOUD_PROJECT}
+google-audiences=${GOOGLE_AUDIENCES}
diff --git a/provider/partition-gcp/src/main/resources/application-osm-postgres.properties b/provider/partition-gcp/src/main/resources/application-osm-postgres.properties
deleted file mode 100644
index b700407e265a30eab092e0e4118e8c5c40bcfb3f..0000000000000000000000000000000000000000
--- a/provider/partition-gcp/src/main/resources/application-osm-postgres.properties
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Copyright 2020-2021 Google LLC
-# Copyright 2020-2021 EPAM Systems, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-google-cloud-project=osdu-cicd-epam
-
-LOG_PREFIX=partition
-server.servlet.contextPath=/api/partition/v1
-server.port=8080
-springfox.documentation.swagger.v2.path=/api-docs
-authorize-api=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
-spring.cloud.gcp.datastore.namespace=${partition-namespace}
-#ACCEPT_HTTP=true
-environment=gcp
-
-cache-expiration=1
-cache-maxSize=1000
-
-key-ring=csqp
-kms-key=searchService
-
-KEY_RING=${key-ring}
-KMS_KEY=${kms-key}
-GOOGLE_CLOUD_PROJECT=${google-cloud-project}
-google-audiences=123.apps.googleusercontent.com
-partition-admin-accounts=osdu-gcp-sa
-partition-property-kind=PartitionProperty
-partition-namespace=partition
-
-#logging configuration
-logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
-
-management.endpoint.health.show-details=always
-management.health.diskspace.enabled=false
-
-osmDriver=postgres
-
-osm.postgres.url=jdbc:postgresql://127.0.0.1:5432/postgres
-osm.postgres.username=postgres
-osm.postgres.password=postgres
diff --git a/provider/partition-gcp/src/main/resources/application.properties b/provider/partition-gcp/src/main/resources/application.properties
index bce9e7f120286f42f4b2d68458f1d5b0cb8c7a20..bff52e44e0ecddc94e1a6a9736a48c0d2d2a3ccf 100644
--- a/provider/partition-gcp/src/main/resources/application.properties
+++ b/provider/partition-gcp/src/main/resources/application.properties
@@ -15,35 +15,24 @@
# limitations under the License.
#
-google-cloud-project=osdu-cicd-epam
-
-LOG_PREFIX=partition
+#server config
server.servlet.contextPath=/api/partition/v1
server.port=8080
springfox.documentation.swagger.v2.path=/api-docs
-authorize-api=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
-spring.cloud.gcp.datastore.namespace=${partition-namespace}
-#ACCEPT_HTTP=true
-environment=gcp
+management.endpoint.health.show-details=always
+management.health.diskspace.enabled=false
+#logging configuration
+logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
+LOG_PREFIX=partition
+
+#cache config
cache-expiration=1
cache-maxSize=1000
-key-ring=csqp
-kms-key=searchService
-
-KEY_RING=${key-ring}
-KMS_KEY=${kms-key}
-GOOGLE_CLOUD_PROJECT=${google-cloud-project}
-google-audiences=123.apps.googleusercontent.com
-partition-admin-accounts=osdu-gcp-sa
+#service config
partition-property-kind=PartitionProperty
partition-namespace=partition
-#logging configuration
-logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
-management.endpoint.health.show-details=always
-management.health.diskspace.enabled=false
-osmDriver=datastore
diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
index 3715aee7f107825629b35910cec622ce40036165..050af8c25924369c17433de19dad9c9a72389678 100644
--- a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
+++ b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
@@ -17,6 +17,18 @@
package org.opengroup.osdu.partition.provider.gcp.service;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -24,22 +36,12 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.partition.logging.AuditLogger;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
import org.opengroup.osdu.partition.provider.gcp.model.PartitionPropertyEntity;
import org.opengroup.osdu.partition.provider.gcp.osm.repository.OsmPartitionPropertyRepository;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.*;
-
@ExtendWith(MockitoExtension.class)
public class PartitionServiceImplCacheTest {
@@ -61,7 +63,6 @@ public class PartitionServiceImplCacheTest {
public void setup() {
partitionServiceImpl = new PartitionServiceImpl(
partitionPropertyEntityRepository,
- mock(IKmsClient.class),
auditLogger,
partitionServiceCache,
partitionListCache
diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
index 213aafa897b61e4dc4f462f64a1df3bd6609862b..1abda3af661c29f6c2e0f87991dce397c1a550d2 100644
--- a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
+++ b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
@@ -17,6 +17,16 @@
package org.opengroup.osdu.partition.provider.gcp.service;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -24,25 +34,12 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.partition.logging.AuditLogger;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
import org.opengroup.osdu.partition.provider.gcp.model.PartitionPropertyEntity;
import org.opengroup.osdu.partition.provider.gcp.osm.repository.OsmPartitionPropertyRepository;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
@ExtendWith(MockitoExtension.class)
public class PartitionServiceImplTest {
@@ -74,7 +71,6 @@ public class PartitionServiceImplTest {
public void setup() {
partitionServiceImpl = new PartitionServiceImpl(
partitionPropertyEntityRepository,
- mock(IKmsClient.class),
auditLogger,
partitionServiceCache,
partitionListCache