From b0575f7f198f4526b846e1f009503811bb790b08 Mon Sep 17 00:00:00 2001 From: "Rustam Lotsmanenko (EPAM)" <rustam_lotsmanenko@epam.com> Date: Mon, 18 Nov 2024 14:36:02 +0000 Subject: [PATCH] Cim gc group caching --- NOTICE | 2 +- provider/storage-gc/docs/gc/README.md | 4 ++ .../provider/gcp/web/cache/CacheConfig.java | 19 ++++++-- .../provider/gcp/web/cache/GroupCache.java | 46 ------------------- .../gcp/web/config/GcpAppServiceConfig.java | 5 ++ .../gcp/web/repository/ObmStorage.java | 4 ++ storage-core-plus/docs/baremetal/README.md | 4 ++ .../provider/gcp/web/cache/CacheConfig.java | 20 +++++--- .../provider/gcp/web/cache/GroupCache.java | 46 ------------------- .../gcp/web/config/GcpAppServiceConfig.java | 5 ++ .../gcp/web/repository/ObmStorage.java | 4 ++ .../TestRecordAccessAuthorization.java | 6 +-- 12 files changed, 57 insertions(+), 108 deletions(-) delete mode 100644 provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/GroupCache.java delete mode 100644 storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/GroupCache.java diff --git a/NOTICE b/NOTICE index 338fbb0fe..87ad3598f 100644 --- a/NOTICE +++ b/NOTICE @@ -54,7 +54,7 @@ The following software have components provided under the terms of this license: - Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core-ga/, http://hc.apache.org/httpcomponents-core/) - Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api) - Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core) -- Apache Log4j JUL Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul) +- Apache Log4j JUL Handler (from https://logging.apache.org/log4j/3.x/) - Apache Log4j SLF4J Binding (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl) - Apache Log4j to SLF4J Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-to-slf4j) - AssertJ Core (from https://assertj.github.io/doc/#assertj-core) diff --git a/provider/storage-gc/docs/gc/README.md b/provider/storage-gc/docs/gc/README.md index 8d67973d6..ff7a0fcfa 100644 --- a/provider/storage-gc/docs/gc/README.md +++ b/provider/storage-gc/docs/gc/README.md @@ -41,6 +41,10 @@ Defined in default application property file but possible to override: | `REDIS_STORAGE_PASSWORD` | ex `*****` | Redis storage host password | yes | | | `REDIS_STORAGE_WITH_SSL` | ex `true` or `false` | Redis storage host ssl config | no | | | `REDIS_STORAGE_EXPIRATION` | ex `30` | Redis storage cache expiration in seconds | no | | +| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for storage groups | no | | +| `REDIS_GROUP_PASSWORD` | ex `*****` | Redis storage groups host password | yes | | +| `REDIS_GROUP_WITH_SSL` | ex `true` or `false` | Redis storage groups host ssl config | no | | +| `REDIS_GROUP_EXPIRATION` | ex `30` | Redis storage groups cache expiration in seconds | no | | | `POLICY_API` | ex `http://localhost:8080/api/policy/v1/` | Police service endpoint | no | output of infrastructure deployment | | `POLICY_ID` | ex `storage` | policeId from ex `http://localhost:8080/api/policy/v1/policies`. Look at `POLICY_API` | no | - | | `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - | diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java index 4c391aa97..5115bbd5d 100644 --- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java +++ b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java @@ -35,11 +35,7 @@ public class CacheConfig { private final RedisCacheBuilder<String, String> legalRedisCacheBuilder; private final RedisCacheBuilder<String, Schema> schemaRedisCacheBuilder; - - @Bean - public ICache<String, Groups> groupCache() { - return new GroupCache(); - } + private final RedisCacheBuilder<String, Groups> groupsRedisCacheBuilder; @Bean("LegalTagCache") public ICache<String, String> legalTagCache(GcpAppServiceConfig gcpAppServiceConfig) { @@ -68,6 +64,19 @@ public class CacheConfig { ); } + @Bean + public RedisCache<String, Groups> groupsCache(GcpAppServiceConfig gcpAppServiceConfig){ + return groupsRedisCacheBuilder.buildRedisCache( + gcpAppServiceConfig.getRedisGroupHost(), + gcpAppServiceConfig.getRedisGroupPort(), + gcpAppServiceConfig.getRedisGroupPassword(), + gcpAppServiceConfig.getRedisGroupExpiration(), + gcpAppServiceConfig.getRedisGroupWithSsl(), + String.class, + Groups.class + ); + } + @Bean public ICache<String, PartitionInfo> partitionInfoCache() { return new VmCache<>(600, 2000); diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/GroupCache.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/GroupCache.java deleted file mode 100644 index 23776417b..000000000 --- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/GroupCache.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2020-2023 Google LLC - * Copyright 2020-2023 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.storage.provider.gcp.web.cache; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.core.common.model.entitlements.Groups; - -// Group cache is used in common part. According to the current Google Cloud architecture, we don't -// use cache. Thus, methods are empty. -public class GroupCache implements ICache<String, Groups> { - - @Override - public void put(String s, Groups o) { - // do nothing - } - - @Override - public Groups get(String s) { - return null; - } - - @Override - public void delete(String s) { - // do nothing - } - - @Override - public void clearAll() { - // do nothing - } -} diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java index 1400f11b9..40f80770e 100644 --- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java +++ b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java @@ -34,4 +34,9 @@ public class GcpAppServiceConfig { private Integer redisStorageExpiration = 60 * 60; private Boolean redisStorageWithSsl = false; + private String redisGroupHost; + private Integer redisGroupPort; + private String redisGroupPassword; + private Integer redisGroupExpiration = 30; + private Boolean redisGroupWithSsl = false; } diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java index cbb5b78a2..b664cf988 100644 --- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java +++ b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java @@ -383,6 +383,10 @@ public class ObmStorage implements ICloudStorage { } private void validateMetadata(RecordMetadata metadata) { + if (entitlementsService.isDataManager(headers)) { + return; + } + List<String> aclGroups = new ArrayList<>(); Collections.addAll(aclGroups, metadata.getAcl().getViewers()); diff --git a/storage-core-plus/docs/baremetal/README.md b/storage-core-plus/docs/baremetal/README.md index e29787d56..c424e6615 100644 --- a/storage-core-plus/docs/baremetal/README.md +++ b/storage-core-plus/docs/baremetal/README.md @@ -56,6 +56,10 @@ Defined in default application property file but possible to override: | `REDIS_STORAGE_PASSWORD` | ex `*****` | Redis storage host password | yes | | | `REDIS_STORAGE_WITH_SSL` | ex `true` or `false` | Redis storage host ssl config | no | | | `REDIS_STORAGE_EXPIRATION` | ex `30` | Redis storage cache expiration in seconds | no | | +| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for storage groups | no | | +| `REDIS_GROUP_PASSWORD` | ex `*****` | Redis storage groups host password | yes | | +| `REDIS_GROUP_WITH_SSL` | ex `true` or `false` | Redis storage groups host ssl config | no | | +| `REDIS_GROUP_EXPIRATION` | ex `30` | Redis storage groups cache expiration in seconds | no | | | `POLICY_API` | ex `http://localhost:8080/api/policy/v1/` | Police service endpoint | no | output of infrastructure deployment | | `POLICY_ID` | ex `search` | policeId from ex `http://localhost:8080/api/policy/v1/policies`. Look at `POLICY_API` | no | - | | `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - | diff --git a/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java b/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java index d4f189ad0..dfb438667 100644 --- a/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java +++ b/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/CacheConfig.java @@ -25,7 +25,6 @@ import org.opengroup.osdu.core.common.cache.VmCache; import org.opengroup.osdu.core.common.model.entitlements.Groups; import org.opengroup.osdu.core.common.model.storage.Schema; import org.opengroup.osdu.core.common.partition.PartitionInfo; - import org.opengroup.osdu.storage.provider.gcp.web.config.GcpAppServiceConfig; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -36,11 +35,7 @@ public class CacheConfig { private final RedisCacheBuilder<String, String> legalRedisCacheBuilder; private final RedisCacheBuilder<String, Schema> schemaRedisCacheBuilder; - - @Bean - public ICache<String, Groups> groupCache() { - return new GroupCache(); - } + private final RedisCacheBuilder<String, Groups> groupsRedisCacheBuilder; @Bean("LegalTagCache") public ICache<String, String> legalTagCache(GcpAppServiceConfig gcpAppServiceConfig) { @@ -69,6 +64,19 @@ public class CacheConfig { ); } + @Bean + public RedisCache<String, Groups> groupsCache(GcpAppServiceConfig gcpAppServiceConfig){ + return groupsRedisCacheBuilder.buildRedisCache( + gcpAppServiceConfig.getRedisGroupHost(), + gcpAppServiceConfig.getRedisGroupPort(), + gcpAppServiceConfig.getRedisGroupPassword(), + gcpAppServiceConfig.getRedisGroupExpiration(), + gcpAppServiceConfig.getRedisGroupWithSsl(), + String.class, + Groups.class + ); + } + @Bean public ICache<String, PartitionInfo> partitionInfoCache() { return new VmCache<>(600, 2000); diff --git a/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/GroupCache.java b/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/GroupCache.java deleted file mode 100644 index 23776417b..000000000 --- a/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/cache/GroupCache.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2020-2023 Google LLC - * Copyright 2020-2023 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.storage.provider.gcp.web.cache; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.core.common.model.entitlements.Groups; - -// Group cache is used in common part. According to the current Google Cloud architecture, we don't -// use cache. Thus, methods are empty. -public class GroupCache implements ICache<String, Groups> { - - @Override - public void put(String s, Groups o) { - // do nothing - } - - @Override - public Groups get(String s) { - return null; - } - - @Override - public void delete(String s) { - // do nothing - } - - @Override - public void clearAll() { - // do nothing - } -} diff --git a/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java b/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java index 1400f11b9..40f80770e 100644 --- a/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java +++ b/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/config/GcpAppServiceConfig.java @@ -34,4 +34,9 @@ public class GcpAppServiceConfig { private Integer redisStorageExpiration = 60 * 60; private Boolean redisStorageWithSsl = false; + private String redisGroupHost; + private Integer redisGroupPort; + private String redisGroupPassword; + private Integer redisGroupExpiration = 30; + private Boolean redisGroupWithSsl = false; } diff --git a/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java b/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java index 949be8079..79232c3aa 100644 --- a/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java +++ b/storage-core-plus/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java @@ -383,6 +383,10 @@ public class ObmStorage implements ICloudStorage { } private void validateMetadata(RecordMetadata metadata) { + if (entitlementsService.isDataManager(headers)) { + return; + } + List<String> aclGroups = new ArrayList<>(); Collections.addAll(aclGroups, metadata.getAcl().getViewers()); diff --git a/testing/storage-test-gc/src/test/java/org/opengroup/osdu/storage/records/TestRecordAccessAuthorization.java b/testing/storage-test-gc/src/test/java/org/opengroup/osdu/storage/records/TestRecordAccessAuthorization.java index 4bb4ecb0c..c20e7a424 100644 --- a/testing/storage-test-gc/src/test/java/org/opengroup/osdu/storage/records/TestRecordAccessAuthorization.java +++ b/testing/storage-test-gc/src/test/java/org/opengroup/osdu/storage/records/TestRecordAccessAuthorization.java @@ -67,14 +67,12 @@ public class TestRecordAccessAuthorization extends RecordAccessAuthorizationTest Map<String, String> headers = HeaderUtils.getHeaders(TenantUtils.getTenantName(), testUtils.getNoDataAccessToken()); - CloseableHttpResponse response = TestUtils.send("records", "PUT", headers, + CloseableHttpResponse response = TestUtils.send("records", "PUT", headers, RecordUtil.createDefaultJsonRecord(RECORD_ID, KIND, LEGAL_TAG), ""); assertEquals(HttpStatus.SC_UNAUTHORIZED, response.getCode()); JsonObject json = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); assertEquals(401, json.get("code").getAsInt()); - assertEquals("Error from compliance service", json.get("reason").getAsString()); - assertEquals("Legal response 401 {\"code\":401,\"reason\":\"Unauthorized\",\"message\":\"The user is not authorized to perform this action\"}", json.get("message").getAsString()); - } + } } } -- GitLab