From c8f51b2d2f8519d6939e7babd968e6463665aaa3 Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Fri, 10 Sep 2021 16:28:48 +0300 Subject: [PATCH 1/9] Partition service's (azure-provider) latency is more than 300 seconds +Cluster lock on cache rebuild in Azure. -Cache SPI. https://community.opengroup.org/osdu/platform/system/partition/-/issues/16 --- .../osdu/partition/api/PartitionApi.java | 2 +- .../interfaces/IPartitionServiceCache.java | 20 -- .../service/CachedPartitionServiceImpl.java | 111 ------- .../CachedPartitionServiceImplTest.java | 150 --------- .../PartitionServiceDummyListCacheImpl.java | 52 --- ...ionServiceDummyPartitionInfoCacheImpl.java | 51 --- .../aws/service/PartitionServiceImpl.java | 3 + provider/partition-azure/pom.xml | 12 + .../azure/cache/PartitionListCacheImpl.java | 38 --- .../cache/PartitionServiceCacheImpl.java | 37 --- .../provider/azure/di/RedisConfig.java | 28 ++ .../provider/azure/di/RetryConfig.java | 38 +++ .../azure/service/PartitionServiceImpl.java | 132 ++++++-- .../src/main/resources/application.properties | 9 + .../PartitionServiceImplCacheTest.java | 296 +++++++++++++++++ .../service/PartitionServiceImplTest.java | 90 ++++-- .../gcp/cache/PartitionListCacheImpl.java | 53 --- .../gcp/cache/PartitionServiceCacheImpl.java | 52 --- .../gcp/service/PartitionServiceImpl.java | 86 +++-- .../PartitionServiceImplCacheTest.java | 195 +++++++++++ .../gcp/service/PartitionServiceImplTest.java | 304 ++++++++++-------- .../ibm/cache/PartitionListCacheImpl.java | 45 --- .../ibm/cache/PartitionServiceCacheImpl.java | 42 --- .../ibm/service/PartitionServiceImpl.java | 104 ++++-- 24 files changed, 1052 insertions(+), 898 deletions(-) delete mode 100644 partition-core/src/main/java/org/opengroup/osdu/partition/provider/interfaces/IPartitionServiceCache.java delete mode 100644 partition-core/src/main/java/org/opengroup/osdu/partition/service/CachedPartitionServiceImpl.java delete mode 100644 partition-core/src/test/java/org/opengroup/osdu/partition/service/CachedPartitionServiceImplTest.java delete mode 100644 provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceDummyListCacheImpl.java delete mode 100644 provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceDummyPartitionInfoCacheImpl.java delete mode 100644 provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/cache/PartitionListCacheImpl.java delete mode 100644 provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/cache/PartitionServiceCacheImpl.java create mode 100644 provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/RetryConfig.java create mode 100644 provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImplCacheTest.java delete mode 100644 provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/PartitionListCacheImpl.java delete mode 100644 provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/PartitionServiceCacheImpl.java create mode 100644 provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java delete mode 100644 provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/cache/PartitionListCacheImpl.java delete mode 100644 provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/cache/PartitionServiceCacheImpl.java diff --git a/partition-core/src/main/java/org/opengroup/osdu/partition/api/PartitionApi.java b/partition-core/src/main/java/org/opengroup/osdu/partition/api/PartitionApi.java index 860086cf..0d55f778 100644 --- a/partition-core/src/main/java/org/opengroup/osdu/partition/api/PartitionApi.java +++ b/partition-core/src/main/java/org/opengroup/osdu/partition/api/PartitionApi.java @@ -39,7 +39,7 @@ import java.util.Map; public class PartitionApi { @Autowired - @Qualifier("cachedPartitionServiceImpl") + @Qualifier("partitionServiceImpl") private IPartitionService partitionService; @Autowired diff --git a/partition-core/src/main/java/org/opengroup/osdu/partition/provider/interfaces/IPartitionServiceCache.java b/partition-core/src/main/java/org/opengroup/osdu/partition/provider/interfaces/IPartitionServiceCache.java deleted file mode 100644 index 593d67c6..00000000 --- a/partition-core/src/main/java/org/opengroup/osdu/partition/provider/interfaces/IPartitionServiceCache.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017-2020, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.partition.provider.interfaces; - -import org.opengroup.osdu.core.common.cache.ICache; - -public interface IPartitionServiceCache extends ICache { -} diff --git a/partition-core/src/main/java/org/opengroup/osdu/partition/service/CachedPartitionServiceImpl.java b/partition-core/src/main/java/org/opengroup/osdu/partition/service/CachedPartitionServiceImpl.java deleted file mode 100644 index a0609023..00000000 --- a/partition-core/src/main/java/org/opengroup/osdu/partition/service/CachedPartitionServiceImpl.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2017-2020, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.partition.service; - -import org.apache.http.HttpStatus; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionService; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -import javax.inject.Inject; -import java.util.List; - -@Service -public class CachedPartitionServiceImpl implements IPartitionService { - - private static final String PARTITION_LIST_KEY = "getAllPartitions"; - - @Inject - @Qualifier("partitionServiceImpl") - private IPartitionService partitionService; - - @Inject - @Qualifier("partitionServiceCache") - private IPartitionServiceCache partitionServiceCache; - - @Inject - @Qualifier("partitionListCache") - private IPartitionServiceCache> partitionListCache; - - @Override - public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) { - if (partitionServiceCache.get(partitionId) != null) - throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist"); - PartitionInfo pi = partitionService.createPartition(partitionId, partitionInfo); - - if (pi != null) { - partitionServiceCache.put(partitionId, pi); - partitionListCache.clearAll(); - } - - return pi; - } - - @Override - public PartitionInfo updatePartition(String partitionId, PartitionInfo partitionInfo) { - PartitionInfo pi = partitionService.updatePartition(partitionId, partitionInfo); - - if(pi != null) { - partitionServiceCache.put(partitionId, pi); - } - - return pi; - } - - @Override - public PartitionInfo getPartition(String partitionId) { - PartitionInfo pi = (PartitionInfo) partitionServiceCache.get(partitionId); - - if (pi == null) { - pi = partitionService.getPartition(partitionId); - - if (pi != null) { - partitionServiceCache.put(partitionId, pi); - } - } - - return pi; - } - - @Override - public boolean deletePartition(String partitionId) { - if (partitionService.deletePartition(partitionId)) { - if (partitionServiceCache.get(partitionId) != null) { - partitionServiceCache.delete(partitionId); - } - partitionListCache.clearAll(); - return true; - } - - return false; - } - - @Override - public List getAllPartitions() { - List partitions = (List)partitionListCache.get(PARTITION_LIST_KEY); - - if (partitions == null) { - partitions = partitionService.getAllPartitions(); - - if (partitions != null) { - partitionListCache.put(PARTITION_LIST_KEY, partitions); - } - } - return partitions; - } -} diff --git a/partition-core/src/test/java/org/opengroup/osdu/partition/service/CachedPartitionServiceImplTest.java b/partition-core/src/test/java/org/opengroup/osdu/partition/service/CachedPartitionServiceImplTest.java deleted file mode 100644 index 7d268057..00000000 --- a/partition-core/src/test/java/org/opengroup/osdu/partition/service/CachedPartitionServiceImplTest.java +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2017-2020, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.partition.service; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; -import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionService; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; - -import java.util.ArrayList; -import java.util.List; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -@RunWith(MockitoJUnitRunner.class) -public class CachedPartitionServiceImplTest { - - @Mock - private IPartitionService partitionServiceImpl; - - @Mock - private IPartitionServiceCache partitionServiceCache; - - @Mock - private IPartitionServiceCache> partitionListCache; - - @InjectMocks - private CachedPartitionServiceImpl cachedPartitionServiceImpl; - - @Test - public void createPartitionSucceed() { - String partId = "key"; - - PartitionInfo newPi = PartitionInfo.builder().build(); - PartitionInfo retPi = PartitionInfo.builder().build(); - - when(partitionServiceImpl.createPartition(partId, newPi)).thenReturn(retPi); - cachedPartitionServiceImpl.createPartition(partId, newPi); - - verify(partitionServiceImpl, times(1)).createPartition(partId, newPi); - verify(partitionServiceCache, times(1)).put(partId, retPi); - verify(partitionListCache, times(1)).clearAll(); - } - - @Test - public void createPartitionFailed() { - String partId = "key"; - PartitionInfo newPi = PartitionInfo.builder().build(); - - when(partitionServiceCache.get(partId)).thenReturn(null); - when(partitionServiceImpl.createPartition(partId, newPi)).thenReturn(null); - - cachedPartitionServiceImpl.createPartition(partId, newPi); - - verify(partitionServiceImpl, times(1)).createPartition(partId, newPi); - verify(partitionServiceCache, times(0)).put(any(), any()); - verify(partitionListCache, times(0)).clearAll(); - verify(partitionServiceCache, times(1)).get(any()); - } - - @Test - public void updatePartitionSucceed() { - String partId = "key"; - - PartitionInfo newPi = PartitionInfo.builder().build(); - PartitionInfo retPi = PartitionInfo.builder().build(); - - when(partitionServiceImpl.updatePartition(partId, newPi)).thenReturn(retPi); - - cachedPartitionServiceImpl.updatePartition(partId, newPi); - - verify(partitionServiceImpl, times(1)).updatePartition(partId, newPi); - verify(partitionServiceCache, times(1)).put(partId, retPi); - } - - @Test - public void updatePartitionFailed() { - String partId = "key"; - PartitionInfo newPi = PartitionInfo.builder().build(); - - when(partitionServiceImpl.updatePartition(partId, newPi)).thenReturn(null); - - cachedPartitionServiceImpl.updatePartition(partId, newPi); - - verify(partitionServiceImpl, times(1)).updatePartition(partId, newPi); - verify(partitionServiceCache, times(0)).put(any(), any()); - verify(partitionServiceCache, times(0)).get(any()); - } - - @Test - public void getPartition() { - String partId = "key"; - - PartitionInfo retPi = PartitionInfo.builder().build(); - - when(partitionServiceImpl.getPartition(partId)).thenReturn(retPi); - - cachedPartitionServiceImpl.getPartition(partId); - - verify(partitionServiceCache, times(1)).get(partId); - verify(partitionServiceImpl, times(1)).getPartition(partId); - verify(partitionServiceCache, times(1)).put(partId, retPi); - } - - @Test - public void deletePartition() { - String partId = "key"; - PartitionInfo retPi = PartitionInfo.builder().build(); - - when(partitionServiceImpl.deletePartition(partId)).thenReturn(true); - when(partitionServiceCache.get(partId)).thenReturn(retPi); - - cachedPartitionServiceImpl.deletePartition(partId); - - verify(partitionServiceImpl, times(1)).deletePartition(partId); - verify(partitionServiceCache, times(1)).delete(partId); - verify(partitionServiceCache, times(1)).get(partId); - verify(partitionListCache, times(1)).clearAll(); - } - - @Test - public void getAllPartitions() { - List partitions = new ArrayList<>(); - - when(partitionServiceImpl.getAllPartitions()).thenReturn(partitions); - cachedPartitionServiceImpl.getAllPartitions(); - String partKey = "getAllPartitions"; - verify(partitionListCache, times(1)).get(partKey); - verify(partitionServiceImpl, times(1)).getAllPartitions(); - verify(partitionListCache, times(1)).put(partKey, partitions); - } - -} \ No newline at end of file diff --git a/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceDummyListCacheImpl.java b/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceDummyListCacheImpl.java deleted file mode 100644 index 6065c902..00000000 --- a/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceDummyListCacheImpl.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright © 2020 Amazon Web Services -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.partition.provider.aws.service; - -import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; -import java.util.List; - -/*** - * We don't want to use cache. Implement a dummy service to always return a cache miss. - */ -@Service -@Qualifier("partitionListCache") -public class PartitionServiceDummyListCacheImpl implements IPartitionServiceCache> { - public PartitionServiceDummyListCacheImpl() { - - } - - @Override - public void clearAll() { - return; - } - - @Override - public void delete(String arg0) { - return; - } - - @Override - public List get(String arg0) { - return null; - } - - @Override - public void put(String arg0, List arg1) { - return; - } -} diff --git a/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceDummyPartitionInfoCacheImpl.java b/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceDummyPartitionInfoCacheImpl.java deleted file mode 100644 index 615ddef1..00000000 --- a/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceDummyPartitionInfoCacheImpl.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright © 2020 Amazon Web Services -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.partition.provider.aws.service; - -import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -/*** - * We don't want to use cache. Implement a dummy service to always return a cache miss. - */ -@Service -@Qualifier("partitionServiceCache") -public class PartitionServiceDummyPartitionInfoCacheImpl implements IPartitionServiceCache { - public PartitionServiceDummyPartitionInfoCacheImpl() { - - } - - @Override - public void clearAll() { - return; - } - - @Override - public void delete(String arg0) { - return; - } - - @Override - public PartitionInfo get(String arg0) { - return null; - } - - @Override - public void put(String arg0, PartitionInfo arg1) { - return; - } -} diff --git a/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceImpl.java b/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceImpl.java index ce1433df..1a2e6a33 100644 --- a/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceImpl.java +++ b/provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/service/PartitionServiceImpl.java @@ -27,6 +27,9 @@ import org.opengroup.osdu.partition.provider.interfaces.IPartitionService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +/** + * AWS implementation doesn't use cache. + */ @Service public class PartitionServiceImpl implements IPartitionService { diff --git a/provider/partition-azure/pom.xml b/provider/partition-azure/pom.xml index 4d9f57fa..3a4e0b07 100644 --- a/provider/partition-azure/pom.xml +++ b/provider/partition-azure/pom.xml @@ -19,6 +19,8 @@ 2.25.0 2.0.2 2.4.6 + 3.15.3 + 1.7.0 @@ -99,6 +101,16 @@ org.projectlombok lombok + + org.redisson + redisson + ${redisson.version} + + + io.github.resilience4j + resilience4j-retry + ${resilience4j.version} + junit diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/cache/PartitionListCacheImpl.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/cache/PartitionListCacheImpl.java deleted file mode 100644 index 11a6a706..00000000 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/cache/PartitionListCacheImpl.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.opengroup.osdu.partition.provider.azure.cache; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; -import java.util.List; - -@Service -@Qualifier("partitionListCache") -public class PartitionListCacheImpl implements IPartitionServiceCache> { - - @Resource(name="partitionListCache") - private ICache> cache; - - @Override - public void put(String s, List o) { - this.cache.put(s, o); - } - - @Override - public List get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } - -} diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/cache/PartitionServiceCacheImpl.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/cache/PartitionServiceCacheImpl.java deleted file mode 100644 index 372432c0..00000000 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/cache/PartitionServiceCacheImpl.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opengroup.osdu.partition.provider.azure.cache; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; - -@Service -@Qualifier("partitionServiceCache") -public class PartitionServiceCacheImpl implements IPartitionServiceCache { - - @Resource(name="partitionServiceCache") - private ICache cache; - - @Override - public void put(String s, PartitionInfo o) { - this.cache.put(s, o); - } - - @Override - public PartitionInfo get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } -} diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/RedisConfig.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/RedisConfig.java index 75163aac..497e8548 100644 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/RedisConfig.java +++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/RedisConfig.java @@ -6,6 +6,9 @@ import com.lambdaworks.redis.SocketOptions; import org.opengroup.osdu.azure.KeyVaultFacade; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.partition.model.PartitionInfo; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; @@ -46,6 +49,9 @@ public class RedisConfig { @Value("${redis.connection.timeout}") private long timeout; + @Value("${spring.application.name}") + private String applicationName; + @Bean public RedisCache partitionServiceCache(@Named("REDIS_HOST") String host, @Named("REDIS_PASSWORD") String password) { ClientOptions clientOptions = ClientOptions.builder() @@ -62,6 +68,16 @@ public class RedisConfig { return new RedisCache(host, port, password, expiration, database, clientOptions, String.class, List.class); } + @Bean + public RedissonClient getRedissonClient(@Named("REDIS_HOST") String host, @Named("REDIS_PASSWORD") String password) { + Config config = new Config(); + config.useSingleServer().setAddress(String.format("rediss://%s:%d", host, port)) + .setPassword(password) + .setDatabase(database) + .setKeepAlive(true) + .setClientName(applicationName); + return Redisson.create(config); + } } @Configuration @@ -80,6 +96,9 @@ public class RedisConfig { @Value("${redis.connection.timeout}") private long timeout; + @Value("${spring.application.name}") + private String applicationName; + @Bean public RedisCache partitionServiceCache(@Named("REDIS_HOST") String host) { ClientOptions clientOptions = ClientOptions.builder() @@ -96,5 +115,14 @@ public class RedisConfig { return new RedisCache(host, port, expiration, database, clientOptions, String.class, List.class); } + @Bean + public RedissonClient getRedissonClient(@Named("REDIS_HOST") String host) { + Config config = new Config(); + config.useSingleServer().setAddress(String.format("redis://%s:%d", host, port)) + .setDatabase(database) + .setKeepAlive(true) + .setClientName(applicationName); + return Redisson.create(config); + } } } diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/RetryConfig.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/RetryConfig.java new file mode 100644 index 00000000..b6ac804f --- /dev/null +++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/RetryConfig.java @@ -0,0 +1,38 @@ +package org.opengroup.osdu.partition.provider.azure.di; + +import io.github.resilience4j.core.IntervalFunction; +import io.github.resilience4j.retry.Retry; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Objects; + +@Configuration +public class RetryConfig { + + @Value("${spring.application.name}") + private String applicationName; + + @Value("${cache.retry.max}") + private int maxRetry; + + @Value("${cache.retry.interval}") + private int retryInterval; + + @Value("${cache.retry.random.factor}") + private float randomFactor; + + /** + Retry client with internal of retryInterval +/- random(randomFactor*retryInterval) for maximum maxRetry times. + This client is used for the concurrent cache rebuild thread which do not acquire the lock to wait and get the rebuilt cache result + */ + @Bean + public Retry getRetryClient() { + return Retry.of(applicationName, io.github.resilience4j.retry.RetryConfig.custom() + .maxAttempts(maxRetry) + .intervalFunction(IntervalFunction.ofRandomized(retryInterval, randomFactor)) + .retryOnResult(Objects::isNull) + .build()); + } +} diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java index ec96e47a..2f405b45 100644 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java +++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java @@ -14,42 +14,79 @@ package org.opengroup.osdu.partition.provider.azure.service; +import io.github.resilience4j.retry.Retry; import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.model.Property; import org.opengroup.osdu.partition.provider.azure.persistence.PartitionTableStore; import org.opengroup.osdu.partition.provider.interfaces.IPartitionService; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import java.util.ArrayList; +import javax.inject.Inject; import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; @Service public class PartitionServiceImpl implements IPartitionService { - private final String PARTITION_NOT_FOUND = "partition not found"; + static final String PARTITION_LIST_KEY = "getAllPartitions"; + static final String PARTITION_NOT_FOUND = "partition not found"; + static final String LOCK_KEY_FORMAT = "lock-ProviderCachedPartitionService-%s"; @Autowired private PartitionTableStore tableStore; + @Autowired + private JaxRsDpsLog log; + + @Autowired + private RedissonClient redissonClient; + + @Autowired + private Retry retry; + + @Inject + @Qualifier("partitionServiceCache") + private ICache partitionServiceCache; + + @Inject + @Qualifier("partitionListCache") + private ICache> partitionListCache; + + @Value("${redisson.lock.acquisition.timeout}") + private int redissonLockAcquisitionTimeOut; + + @Value("${redisson.lock.expiration}") + private int redissonLockExpiration; + @Override public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) { - if (this.tableStore.partitionExists(partitionId)) { + if (partitionServiceCache.get(partitionId) != null || tableStore.partitionExists(partitionId)) { throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist"); } + tableStore.addPartition(partitionId, partitionInfo); - this.tableStore.addPartition(partitionId, partitionInfo); + if (partitionInfo != null) { + partitionServiceCache.put(partitionId, partitionInfo); + partitionListCache.clearAll(); + } return partitionInfo; } @Override public PartitionInfo updatePartition(String partitionId, PartitionInfo partitionInfo) { - if (!this.tableStore.partitionExists(partitionId)) { + if (!tableStore.partitionExists(partitionId)) { throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); } @@ -57,35 +94,88 @@ public class PartitionServiceImpl implements IPartitionService { throw new AppException(HttpStatus.SC_BAD_REQUEST, "can not update id", "the field id can not be updated"); } - this.tableStore.addPartition(partitionId, partitionInfo); - return PartitionInfo.builder().properties(this.tableStore.getPartition(partitionId)).build(); + tableStore.addPartition(partitionId, partitionInfo); + PartitionInfo updatedPartitionInfo = Optional.ofNullable(tableStore.getPartition(partitionId)) + .map(map -> PartitionInfo.builder().properties(map).build()) + .orElse(null); + + if(updatedPartitionInfo != null) { + partitionServiceCache.put(partitionId, updatedPartitionInfo); + } + + return updatedPartitionInfo; } @Override public PartitionInfo getPartition(String partitionId) { - Map out = new HashMap<>(); - out.putAll(this.tableStore.getPartition(partitionId)); - - if (out.isEmpty()) { - throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); - } - - return PartitionInfo.builder().properties(out).build(); + return Optional.ofNullable(partitionServiceCache.get(partitionId)) + .orElseGet(() -> lockCacheEntryAndRebuild( + partitionId, + partitionServiceCache, + () -> Optional.of(new HashMap<>(tableStore.getPartition(partitionId))) + .filter(out -> !out.isEmpty()) + .map(out -> PartitionInfo.builder().properties(out).build()) + .orElseThrow(() -> new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId))) + )); } @Override public boolean deletePartition(String partitionId) { - if (!this.tableStore.partitionExists(partitionId)) { + if (!tableStore.partitionExists(partitionId)) { throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); } - this.tableStore.deletePartition(partitionId); + tableStore.deletePartition(partitionId); + + if (partitionServiceCache.get(partitionId) != null) { + partitionServiceCache.delete(partitionId); + } + partitionListCache.clearAll(); return true; } @Override public List getAllPartitions() { - return this.tableStore.getAllPartitions(); + return Optional.ofNullable(partitionListCache.get(PARTITION_LIST_KEY)) + .orElseGet(() -> lockCacheEntryAndRebuild( + PARTITION_LIST_KEY, + partitionListCache, + tableStore::getAllPartitions + )); + } + + private T lockCacheEntryAndRebuild(String key, ICache cache, Supplier supplier) { + String lockKey = String.format(LOCK_KEY_FORMAT, key); + RLock cacheEntryLock = redissonClient.getLock(lockKey); + boolean locked = false; + try { + locked = cacheEntryLock.tryLock(redissonLockAcquisitionTimeOut, redissonLockExpiration, TimeUnit.MILLISECONDS); + if (locked) { + T data = supplier.get(); + cache.put(key, data); + return data; + } else { + T data = Retry.decorateSupplier(retry, () -> cache.get(key)).get(); + if (data != null) { + return data; + } + } + } catch (InterruptedException ex) { + log.error(String.format("InterruptedException caught when lock the cache key %s: %s", key, ex)); + Thread.currentThread().interrupt(); + } finally { + if (locked) { + try { + cacheEntryLock.unlock(); + } catch (Exception ex) { + log.warning(String.format("unlock exception: %s", ex)); + } + } + } + throw new AppException( + org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE.value(), + org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE.getReasonPhrase(), + "Failed to get the partition(s)."); } -} \ No newline at end of file +} diff --git a/provider/partition-azure/src/main/resources/application.properties b/provider/partition-azure/src/main/resources/application.properties index cb27e04f..ff13ec44 100644 --- a/provider/partition-azure/src/main/resources/application.properties +++ b/provider/partition-azure/src/main/resources/application.properties @@ -1,4 +1,6 @@ LOG_PREFIX=partition +spring.application.name=partition + server.servlet.contextPath=/api/partition/v1 logging.level.org.springframework.web=${partition_spring_logging_level:INFO} JAVA_OPTS=-Dserver.port=80 @@ -42,6 +44,13 @@ redis.ssl.enabled=true redis.connection.timeout=15 redis.database=${REDIS_DATABASE} +# Cache +redisson.lock.acquisition.timeout=10 +redisson.lock.expiration=5000 +cache.retry.max=15 +cache.retry.interval=200 +cache.retry.random.factor=0.1 + # health check management.health.azure-key-vault.enabled=false management.server.port=8081 diff --git a/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImplCacheTest.java b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImplCacheTest.java new file mode 100644 index 00000000..c2e21cde --- /dev/null +++ b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImplCacheTest.java @@ -0,0 +1,296 @@ +// Copyright 2017-2020, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.partition.provider.azure.service; + +import io.github.resilience4j.retry.Retry; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.partition.model.PartitionInfo; +import org.opengroup.osdu.partition.model.Property; +import org.opengroup.osdu.partition.provider.azure.persistence.PartitionTableStore; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class PartitionServiceImplCacheTest { + + @Mock + private PartitionTableStore tableStore; + + @Mock + private Retry retry; + + @Mock(name = "partitionServiceCache") + private ICache partitionServiceCache; + + @Mock(name = "partitionListCache") + private ICache> partitionListCache; + + @Mock + private RedissonClient redissonClient; + + @InjectMocks + private PartitionServiceImpl partitionService; + + private final PartitionInfo partitionInfo = new PartitionInfo(); + + private final static String PARTITION_ID = "my-tenant"; + private final Map properties = new HashMap<>(); + + @SneakyThrows + @BeforeEach + public void setup() { + properties.put("id", Property.builder().value(PARTITION_ID).build()); + properties.put("storageAccount", Property.builder().value("storage-account").sensitive(true).build()); + properties.put("complianceRuleSet", Property.builder().value("compliance-rule-set").build()); + partitionInfo.setProperties(properties); + + RLock rLock = mock(RLock.class); + doReturn(true).when(rLock).tryLock(anyLong(), anyLong(), any(TimeUnit.class)); + doReturn(rLock).when(redissonClient).getLock(anyString()); + + Retry.Context context = mock(Retry.Context.class); + doReturn(true).when(context).onResult(any()); + doReturn(context).when(retry).context(); + } + + @Nested + class CacheTest { + private final String TEST_PARTITION_ID = "test_partition_id"; + private final Map TEST_PARTITION_PROPERTIES = new HashMap() {{ + put("1", mock(Property.class)); + }}; + private final PartitionInfo TEST_PARTITION = PartitionInfo.builder().properties(TEST_PARTITION_PROPERTIES).build(); + private final List TEST_PARTITION_LIST = new ArrayList<>(); + + @BeforeEach + public void setup() { + doReturn(TEST_PARTITION_PROPERTIES).when(tableStore).getPartition(TEST_PARTITION_ID); + doReturn(null).when(partitionListCache).get(PartitionServiceImpl.PARTITION_LIST_KEY); + } + + @Test + public void getPartitionUnlockedTest() { + doReturn(getRLock(false)).when(redissonClient) + .getLock(matches( + getLockKey(TEST_PARTITION_ID) + )); + + PartitionInfo partition = partitionService.getPartition(TEST_PARTITION_ID); + + assertEquals(TEST_PARTITION_PROPERTIES, partition.getProperties()); + verify(partitionServiceCache, times(1)).put(eq(TEST_PARTITION_ID), eq(TEST_PARTITION)); + } + + @Test + public void getPartitionLockedTest() { + doReturn(getRLock(true)).when(redissonClient) + .getLock(matches( + getLockKey(TEST_PARTITION_ID) + )); + doReturn(null, TEST_PARTITION).when(partitionServiceCache).get(eq(TEST_PARTITION_ID)); + doReturn(getRetryContext(TEST_PARTITION)).when(retry).context(); + + PartitionInfo partition = partitionService.getPartition(TEST_PARTITION_ID); + + assertEquals(TEST_PARTITION, partition); + } + + @Test + public void getPartitionLockedNoDataTest() { + assertThrows(AppException.class, () -> { + doReturn(getRLock(true)).when(redissonClient) + .getLock(matches( + getLockKey(TEST_PARTITION_ID) + )); + doReturn(getRetryContext(null)).when(retry).context(); + + partitionService.getPartition(TEST_PARTITION_ID); + }); + } + + @Test + public void getPartitionListUnlockedTest() { + doReturn(getRLock(false)).when(redissonClient) + .getLock(matches( + getLockKey(PartitionServiceImpl.PARTITION_LIST_KEY) + )); + + List partitions = partitionService.getAllPartitions(); + + assertEquals(TEST_PARTITION_LIST, partitions); + verify(partitionListCache, times(1)).put(eq(PartitionServiceImpl.PARTITION_LIST_KEY), eq(TEST_PARTITION_LIST)); + } + + private String getLockKey(String cacheKey) { + return String.format(PartitionServiceImpl.LOCK_KEY_FORMAT, cacheKey); + } + + @Test + public void getPartitionListLockedTest() { + doReturn(getRLock(true)).when(redissonClient) + .getLock(matches( + String.format(PartitionServiceImpl.LOCK_KEY_FORMAT, PartitionServiceImpl.PARTITION_LIST_KEY) + )); + doReturn(null, TEST_PARTITION_LIST).when(partitionListCache).get(eq(PartitionServiceImpl.PARTITION_LIST_KEY)); + doReturn(getRetryContext(TEST_PARTITION_LIST)).when(retry).context(); + + List partitions = partitionService.getAllPartitions(); + + assertEquals(TEST_PARTITION_LIST, partitions); + } + + @Test + public void getPartitionListLockedNoDataTest() { + assertThrows(AppException.class, () -> { + doReturn(getRLock(true)).when(redissonClient) + .getLock(matches( + String.format(PartitionServiceImpl.LOCK_KEY_FORMAT, PartitionServiceImpl.PARTITION_LIST_KEY) + )); + doReturn(getRetryContext(null)).when(retry).context(); + + partitionService.getAllPartitions(); + }); + } + + @SneakyThrows + private RLock getRLock(boolean locked) { + RLock rLock = mock(RLock.class); + doReturn(!locked).when(rLock).tryLock(anyLong(), anyLong(), any(TimeUnit.class)); + return rLock; + } + + private Retry.Context getRetryContext(Object expectedData) { + @SuppressWarnings("unchecked") + Retry.Context context = mock(Retry.Context.class); + doReturn(false).when(context).onResult(eq(expectedData)); + return context; + } + + @Test + public void createPartitionSucceed() { + String partId = "key"; + + PartitionInfo newPi = PartitionInfo.builder().build(); + PartitionInfo retPi = PartitionInfo.builder().build(); + + PartitionInfo partition = partitionService.createPartition(partId, newPi); + + assertEquals(newPi, partition); + verify(tableStore, times(1)).addPartition(partId, newPi); + verify(partitionServiceCache, times(1)).put(partId, retPi); + verify(partitionListCache, times(1)).clearAll(); + } + + @Test + public void updatePartitionSucceed() { + String partId = "key"; + + PartitionInfo newPi = PartitionInfo.builder().build(); + Map retPiProps = new HashMap<>(); + PartitionInfo retPi = PartitionInfo.builder().properties(retPiProps).build(); + + when(tableStore.partitionExists(partId)).thenReturn(true); + when(tableStore.getPartition(partId)).thenReturn(retPiProps); + + partitionService.updatePartition(partId, newPi); + + verify(tableStore, times(1)).addPartition(partId, newPi); + verify(partitionServiceCache, times(1)).put(partId, retPi); + } + + @Test + public void updatePartitionFailed() { + String partId = "key"; + PartitionInfo newPi = PartitionInfo.builder().build(); + + when(tableStore.partitionExists(partId)).thenReturn(true); + when(tableStore.getPartition(partId)).thenReturn(null); + + partitionService.updatePartition(partId, newPi); + + verify(tableStore, times(1)).addPartition(partId, newPi); + verify(partitionServiceCache, times(0)).put(any(), any()); + verify(partitionServiceCache, times(0)).get(any()); + } + + @Test + public void getPartition() { + String partId = "key"; + + Map retPiProps = new HashMap() {{ + put("1", mock(Property.class)); + }}; + PartitionInfo retPi = PartitionInfo.builder().properties(retPiProps).build(); + + when(tableStore.getPartition(partId)).thenReturn(retPiProps); + + partitionService.getPartition(partId); + + verify(partitionServiceCache, times(1)).get(partId); + verify(tableStore, times(1)).getPartition(partId); + verify(partitionServiceCache, times(1)).put(partId, retPi); + } + + @Test + public void deletePartition() { + String partId = "key"; + PartitionInfo retPi = PartitionInfo.builder().build(); + + when(tableStore.partitionExists(partId)).thenReturn(true); + when(partitionServiceCache.get(partId)).thenReturn(retPi); + + partitionService.deletePartition(partId); + + verify(tableStore, times(1)).deletePartition(partId); + verify(partitionServiceCache, times(1)).delete(partId); + verify(partitionServiceCache, times(1)).get(partId); + verify(partitionListCache, times(1)).clearAll(); + } + + @Test + public void getAllPartitions() { + List partitions = new ArrayList<>(); + + when(tableStore.getAllPartitions()).thenReturn(partitions); + partitionService.getAllPartitions(); + String partKey = "getAllPartitions"; + verify(partitionListCache, times(1)).get(partKey); + verify(tableStore, times(1)).getAllPartitions(); + verify(partitionListCache, times(1)).put(partKey, partitions); + } + } +} \ No newline at end of file diff --git a/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImplTest.java b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImplTest.java index 8643ebee..4e19f0ca 100644 --- a/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImplTest.java +++ b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImplTest.java @@ -14,52 +14,83 @@ package org.opengroup.osdu.partition.provider.azure.service; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import io.github.resilience4j.retry.Retry; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.opengroup.osdu.core.common.cache.ICache; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.partition.model.PartitionInfo; import org.opengroup.osdu.partition.model.Property; import org.opengroup.osdu.partition.provider.azure.persistence.PartitionTableStore; -import org.powermock.modules.junit4.PowerMockRunner; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; -import static org.mockito.Mockito.when; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; -@RunWith(PowerMockRunner.class) +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) public class PartitionServiceImplTest { @Mock private PartitionTableStore tableStore; + + @Mock + private Retry retry; + + @Mock(name = "partitionServiceCache") + private ICache partitionServiceCache; + + @Mock(name = "partitionListCache") + private ICache> partitionListCache; + + @Mock + private RedissonClient redissonClient; + @InjectMocks - private PartitionServiceImpl sut; + private PartitionServiceImpl partitionService; private final PartitionInfo partitionInfo = new PartitionInfo(); private final static String PARTITION_ID = "my-tenant"; private final Map properties = new HashMap<>(); - @Before + @SneakyThrows + @BeforeEach public void setup() { properties.put("id", Property.builder().value(PARTITION_ID).build()); properties.put("storageAccount", Property.builder().value("storage-account").sensitive(true).build()); properties.put("complianceRuleSet", Property.builder().value("compliance-rule-set").build()); partitionInfo.setProperties(properties); + + RLock rLock = mock(RLock.class); + doReturn(true).when(rLock).tryLock(anyLong(), anyLong(), any(TimeUnit.class)); + doReturn(rLock).when(redissonClient).getLock(anyString()); + + Retry.Context context = mock(Retry.Context.class); + doReturn(true).when(context).onResult(any()); + doReturn(context).when(retry).context(); } @Test public void should_ThrowConflictError_when_createPartition_whenPartitionExists() { - when(this.tableStore.partitionExists(PARTITION_ID)).thenReturn(true); + when(tableStore.partitionExists(PARTITION_ID)).thenReturn(true); try { - sut.createPartition(PARTITION_ID, this.partitionInfo); + partitionService.createPartition(PARTITION_ID, partitionInfo); } catch (AppException e) { assertEquals(409, e.getError().getCode()); assertTrue(e.getError().getReason().equalsIgnoreCase("partition exist")); @@ -69,9 +100,9 @@ public class PartitionServiceImplTest { @Test public void should_returnPartitionInfo_when_createPartition_whenPartitionDoesntExist() { - when(this.tableStore.partitionExists(PARTITION_ID)).thenReturn(false); + when(tableStore.partitionExists(PARTITION_ID)).thenReturn(false); - PartitionInfo partInfo = sut.createPartition(PARTITION_ID, this.partitionInfo); + PartitionInfo partInfo = partitionService.createPartition(PARTITION_ID, partitionInfo); assertEquals(3, partInfo.getProperties().size()); assertTrue(partInfo.getProperties().containsKey("id")); assertTrue(partInfo.getProperties().containsKey("complianceRuleSet")); @@ -80,10 +111,10 @@ public class PartitionServiceImplTest { @Test public void should_ThrowNotFoundError_when_updatePartition_whenPartitionDoesnsExist() { - when(this.tableStore.partitionExists(PARTITION_ID)).thenReturn(false); + when(tableStore.partitionExists(PARTITION_ID)).thenReturn(false); try { - sut.updatePartition(PARTITION_ID, this.partitionInfo); + partitionService.updatePartition(PARTITION_ID, partitionInfo); } catch (AppException e) { assertEquals(404, e.getError().getCode()); assertTrue(e.getError().getReason().equalsIgnoreCase("partition not found")); @@ -93,10 +124,10 @@ public class PartitionServiceImplTest { @Test public void should_ThrowBadRequestError_when_updatePartition_whenUpdatingPartitionId() { - when(this.tableStore.partitionExists(PARTITION_ID)).thenReturn(true); + when(tableStore.partitionExists(PARTITION_ID)).thenReturn(true); try { - sut.updatePartition(PARTITION_ID, this.partitionInfo); + partitionService.updatePartition(PARTITION_ID, partitionInfo); } catch (AppException e) { assertEquals(400, e.getError().getCode()); assertTrue(e.getError().getReason().equalsIgnoreCase("can not update id")); @@ -106,9 +137,9 @@ public class PartitionServiceImplTest { @Test public void should_returnPartition_when_partitionExists() { - when(this.tableStore.getPartition(PARTITION_ID)).thenReturn(properties); + when(tableStore.getPartition(PARTITION_ID)).thenReturn(properties); - PartitionInfo partitionInfo = this.sut.getPartition(PARTITION_ID); + PartitionInfo partitionInfo = partitionService.getPartition(PARTITION_ID); assertTrue(partitionInfo.getProperties().containsKey("storageAccount")); assertTrue(partitionInfo.getProperties().containsKey("complianceRuleSet")); assertTrue(partitionInfo.getProperties().containsKey("id")); @@ -116,10 +147,10 @@ public class PartitionServiceImplTest { @Test public void should_throwNotFoundException_when_partitionDoesntExist() { - when(this.tableStore.getPartition(PARTITION_ID)).thenReturn(new HashMap<>()); + when(tableStore.getPartition(PARTITION_ID)).thenReturn(new HashMap<>()); try { - sut.getPartition(PARTITION_ID); + partitionService.getPartition(PARTITION_ID); } catch (AppException e) { assertEquals(404, e.getError().getCode()); assertTrue(e.getError().getReason().equalsIgnoreCase("partition not found")); @@ -129,33 +160,32 @@ public class PartitionServiceImplTest { @Test public void should_returnTrue_when_successfullyDeletingSecretes() { - when(this.tableStore.partitionExists(PARTITION_ID)).thenReturn(true); + when(tableStore.partitionExists(PARTITION_ID)).thenReturn(true); - assertTrue(this.sut.deletePartition(PARTITION_ID)); + assertTrue(partitionService.deletePartition(PARTITION_ID)); } @Test public void should_throwException_when_deletingNonExistentPartition() { - when(this.tableStore.partitionExists(PARTITION_ID)).thenReturn(false); + when(tableStore.partitionExists(PARTITION_ID)).thenReturn(false); try { - this.sut.deletePartition("test-partition"); + partitionService.deletePartition("test-partition"); } catch (AppException ae) { assertEquals(404, ae.getError().getCode()); assertEquals("test-partition partition not found", ae.getError().getMessage()); } } - @Test(expected = AppException.class) + @Test public void should_throwException_when_deletingInvalidPartition() { - - this.sut.deletePartition(null); + assertThrows(AppException.class, () -> partitionService.deletePartition(null)); } @Test public void should_returnEmptyList_when_no_partitions() { - when(this.tableStore.getAllPartitions()).thenReturn(new ArrayList<>()); - List partitions = sut.getAllPartitions(); + when(tableStore.getAllPartitions()).thenReturn(new ArrayList<>()); + List partitions = partitionService.getAllPartitions(); assertNotNull(partitions); assertTrue(partitions.isEmpty()); diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/PartitionListCacheImpl.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/PartitionListCacheImpl.java deleted file mode 100644 index 366bb47a..00000000 --- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/PartitionListCacheImpl.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2002-2021 Google LLC - Copyright 2002-2021 EPAM Systems, Inc - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package org.opengroup.osdu.partition.provider.gcp.cache; - -import java.util.List; -import lombok.RequiredArgsConstructor; -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -@Service -@Qualifier("partitionListCache") -@RequiredArgsConstructor -public class PartitionListCacheImpl implements IPartitionServiceCache> { - - private final ICache> cache; - - @Override - public void put(String s, List o) { - this.cache.put(s, o); - } - - @Override - public List get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } -} diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/PartitionServiceCacheImpl.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/PartitionServiceCacheImpl.java deleted file mode 100644 index af35fb94..00000000 --- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/PartitionServiceCacheImpl.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright 2002-2021 Google LLC - Copyright 2002-2021 EPAM Systems, Inc - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package org.opengroup.osdu.partition.provider.gcp.cache; - -import lombok.RequiredArgsConstructor; -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -@Service -@Qualifier("partitionServiceCache") -@RequiredArgsConstructor -public class PartitionServiceCacheImpl implements IPartitionServiceCache { - - private final ICache cache; - - @Override - public void put(String s, PartitionInfo o) { - this.cache.put(s, o); - } - - @Override - public PartitionInfo get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } -} diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java index 1c82f49c..39a7ad06 100644 --- a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java +++ b/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java @@ -17,15 +17,9 @@ package org.opengroup.osdu.partition.provider.gcp.service; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; import lombok.RequiredArgsConstructor; import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.cache.ICache; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; import org.opengroup.osdu.partition.logging.AuditLogger; @@ -37,12 +31,16 @@ import org.opengroup.osdu.partition.provider.interfaces.IPartitionService; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.util.*; + @Service @RequiredArgsConstructor public class PartitionServiceImpl implements IPartitionService { - private static final String UNKNOWN_ERROR_REASON = "unknown error"; - + static final String UNKNOWN_ERROR_REASON = "unknown error"; + static final String PARTITION_LIST_KEY = "getAllPartitions"; private final PartitionPropertyEntityRepository partitionPropertyEntityRepository; @@ -50,17 +48,24 @@ public class PartitionServiceImpl implements IPartitionService { private final AuditLogger auditLogger; + private final ICache partitionServiceCache; + + private final ICache> partitionListCache; + @Override public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) { + if (partitionServiceCache.get(partitionId) != null) + throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist"); + if (this.partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) { this.auditLogger.createPartitionFailure(Collections.singletonList(partitionId)); throw new AppException(HttpStatus.SC_CONFLICT, UNKNOWN_ERROR_REASON, - "Partition already exists."); + "Partition already exists."); } List partitionProperties = new ArrayList<>(); for (Map.Entry entry : partitionInfo.getProperties().entrySet()) { PartitionPropertyEntity entity = new PartitionPropertyEntity(partitionId, - entry.getKey(), entry.getValue()); + entry.getKey(), entry.getValue()); encryptPartitionPropertyEntityIfNeeded(entity); partitionProperties.add(entity); } @@ -68,7 +73,13 @@ public class PartitionServiceImpl implements IPartitionService { repository.saveAll(partitionProperties); return true; }); - return getPartition(partitionId); + PartitionInfo pi = getPartition(partitionId); + + if (pi != null) { + partitionListCache.clearAll(); + } + + return pi; } private void encryptPartitionPropertyEntityIfNeeded(PartitionPropertyEntity entity) { @@ -88,19 +99,19 @@ public class PartitionServiceImpl implements IPartitionService { if (partitionInfo.getProperties().containsKey("id")) { this.auditLogger.updatePartitionSecretFailure(Collections.singletonList(partitionId)); throw new AppException(HttpStatus.SC_BAD_REQUEST, "can not update id", - "the field id can not be updated"); + "the field id can not be updated"); } if (!this.partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) { this.auditLogger.updatePartitionSecretFailure(Collections.singletonList(partitionId)); throw new AppException(HttpStatus.SC_NOT_FOUND, UNKNOWN_ERROR_REASON, - "An attempt to update not existing partition."); + "An attempt to update not existing partition."); } List partitionProperties = new ArrayList<>(); for (Map.Entry entry : partitionInfo.getProperties().entrySet()) { PartitionPropertyEntity entity = this.partitionPropertyEntityRepository - .findByPartitionIdAndName(partitionId, entry.getKey()); + .findByPartitionIdAndName(partitionId, entry.getKey()); if (Objects.nonNull(entity)) { entity.setSensitive(entry.getValue().isSensitive()); entity.setValue(entry.getValue().getValue()); @@ -114,26 +125,37 @@ public class PartitionServiceImpl implements IPartitionService { repository.saveAll(partitionProperties); return true; }); + return getPartition(partitionId); } @Override public PartitionInfo getPartition(String partitionId) { - PartitionInfo partitionInfo = getEncryptedPartition(partitionId); - for (Property property : partitionInfo.getProperties().values()) { - decryptPartitionPropertyIfNeeded(property); + PartitionInfo pi = partitionServiceCache.get(partitionId); + + if (pi == null) { + pi = getEncryptedPartition(partitionId); + for (Property property : pi.getProperties().values()) { + decryptPartitionPropertyIfNeeded(property); + } + + if (pi != null) { + partitionServiceCache.put(partitionId, pi); + } } - return partitionInfo; + + return pi; } private PartitionInfo getEncryptedPartition(String partitionId) { - if (!this.partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) { + Optional> partitionPropertyEntitiesOptional = partitionPropertyEntityRepository + .findByPartitionId(partitionId); + if (!partitionPropertyEntitiesOptional.isPresent()) { this.auditLogger.readPartitionFailure(Collections.singletonList(partitionId)); throw new AppException(HttpStatus.SC_NOT_FOUND, UNKNOWN_ERROR_REASON, "Partition does not exist."); } - List partitionPropertiesList = this.partitionPropertyEntityRepository - .findByPartitionId(partitionId).get(); + List partitionPropertiesList = partitionPropertyEntitiesOptional.get(); PartitionInfo partitionInfo = new PartitionInfo(); Map partitionInfoProperties = new HashMap<>(); for (PartitionPropertyEntity entity : partitionPropertiesList) { @@ -163,16 +185,30 @@ public class PartitionServiceImpl implements IPartitionService { if (!this.partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) { this.auditLogger.deletePartitionFailure(Collections.singletonList(partitionId)); throw new AppException(HttpStatus.SC_NOT_FOUND, UNKNOWN_ERROR_REASON, - "An attempt to delete not existing partition."); + "An attempt to delete not existing partition."); } this.partitionPropertyEntityRepository.deleteByPartitionId(partitionId); + + if (partitionServiceCache.get(partitionId) != null) { + partitionServiceCache.delete(partitionId); + } + partitionListCache.clearAll(); return true; } @Transactional @Override public List getAllPartitions() { - List allPartitions = this.partitionPropertyEntityRepository.getAllPartitions(); - return (allPartitions.isEmpty() ? null : allPartitions); + List partitions = partitionListCache.get(PARTITION_LIST_KEY); + + if (partitions == null) { + List allPartitions = this.partitionPropertyEntityRepository.getAllPartitions(); + partitions = (allPartitions.isEmpty() ? null : allPartitions); + + if (partitions != null) { + partitionListCache.put(PARTITION_LIST_KEY, partitions); + } + } + return partitions; } } diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java new file mode 100644 index 00000000..d1ce6f0a --- /dev/null +++ b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java @@ -0,0 +1,195 @@ +/* + Copyright 2002-2021 Google LLC + Copyright 2002-2021 EPAM Systems, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package org.opengroup.osdu.partition.provider.gcp.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; +import org.opengroup.osdu.partition.logging.AuditLogger; +import org.opengroup.osdu.partition.model.PartitionInfo; +import org.opengroup.osdu.partition.model.Property; +import org.opengroup.osdu.partition.provider.gcp.model.PartitionPropertyEntity; +import org.opengroup.osdu.partition.provider.gcp.repository.PartitionPropertyEntityRepository; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class PartitionServiceImplCacheTest { + + @Mock + private ICache partitionServiceCache; + + @Mock + private ICache> partitionListCache; + + @Mock + private PartitionPropertyEntityRepository partitionPropertyEntityRepository; + + @Mock + private AuditLogger auditLogger; + + private PartitionServiceImpl partitionServiceImpl; + + @BeforeEach + public void setup() { + partitionServiceImpl = new PartitionServiceImpl( + partitionPropertyEntityRepository, + mock(IKmsClient.class), + auditLogger, + partitionServiceCache, + partitionListCache + ); + } + + private List partitionInfoToEntity(String partitionId, PartitionInfo partitionInfo) { + return partitionInfo.getProperties().entrySet().stream() + .map(entry -> new PartitionPropertyEntity(partitionId, entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } + + @Test + public void createPartitionSucceed() { + String partId = "key"; + + PartitionInfo newPi = PartitionInfo.builder().build(); + PartitionInfo retPi = PartitionInfo.builder().build(); + String propKey = "123987123498"; + retPi.getProperties().put(propKey, new Property()); + + doReturn(Optional.empty(), Optional.of(partitionInfoToEntity(partId, retPi))) + .when(partitionPropertyEntityRepository).findByPartitionId(partId); + partitionServiceImpl.createPartition(partId, newPi); + + verify(partitionPropertyEntityRepository, times(1)).performTransaction(any()); + verify(partitionServiceCache, times(1)) + .put(any(), argThat(argument -> argument.getProperties().containsKey(propKey))); + verify(partitionListCache).clearAll(); + } + + @Test + public void createPartitionFailed() { + String partId = "key"; + PartitionInfo newPi = PartitionInfo.builder().build(); + + when(partitionServiceCache.get(partId)).thenReturn(null); + doReturn(Optional.empty()) + .when(partitionPropertyEntityRepository).findByPartitionId(partId); + + assertThrows(AppException.class, () -> partitionServiceImpl.createPartition(partId, newPi)); + + verify(partitionServiceCache, times(0)).put(any(), any()); + verify(partitionListCache, times(0)).clearAll(); + verify(partitionServiceCache, times(2)).get(any()); + } + + @Test + public void updatePartitionSucceed() { + String partId = "key"; + + PartitionInfo newPi = PartitionInfo.builder().build(); + PartitionInfo retPi = PartitionInfo.builder().build(); + String propKey = "123987123498"; + retPi.getProperties().put(propKey, new Property()); + + doReturn(Optional.of(partitionInfoToEntity(partId, retPi))) + .when(partitionPropertyEntityRepository).findByPartitionId(partId); + + partitionServiceImpl.updatePartition(partId, newPi); + + verify(partitionPropertyEntityRepository, times(1)).performTransaction(any()); + verify(partitionServiceCache, times(1)) + .put(any(), argThat(argument -> argument.getProperties().containsKey(propKey))); + } + + @Test + public void updatePartitionFailed() { + String partId = "key"; + PartitionInfo newPi = PartitionInfo.builder().build(); + + doReturn(Optional.empty()) + .when(partitionPropertyEntityRepository).findByPartitionId(partId); + + assertThrows(AppException.class, () -> partitionServiceImpl.updatePartition(partId, newPi)); + + verify(partitionServiceCache, times(0)).put(any(), any()); + verify(partitionServiceCache, times(0)).get(any()); + } + + + @Test + public void getPartition() { + String partId = "key"; + + PartitionInfo retPi = PartitionInfo.builder().build(); + String propKey = "123987123498"; + retPi.getProperties().put(propKey, new Property()); + + doReturn(Optional.of(partitionInfoToEntity(partId, retPi))) + .when(partitionPropertyEntityRepository).findByPartitionId(partId); + + partitionServiceImpl.getPartition(partId); + + verify(partitionServiceCache, times(1)).get(partId); + verify(partitionPropertyEntityRepository, times(1)).findByPartitionId(partId); + verify(partitionServiceCache, times(1)).put(partId, retPi); + } + + @Test + public void deletePartition() { + String partId = "key"; + PartitionInfo retPi = PartitionInfo.builder().build(); + + doReturn(Optional.of(partitionInfoToEntity(partId, retPi))) + .when(partitionPropertyEntityRepository).findByPartitionId(partId); + when(partitionServiceCache.get(partId)).thenReturn(retPi); + + partitionServiceImpl.deletePartition(partId); + + verify(partitionPropertyEntityRepository, times(1)).deleteByPartitionId(partId); + verify(partitionServiceCache, times(1)).delete(partId); + verify(partitionServiceCache, times(1)).get(partId); + verify(partitionListCache, times(1)).clearAll(); + } + + @Test + public void getAllPartitions() { + List partitions = new ArrayList<>(); + partitions.add("1"); + + doReturn(partitions) + .when(partitionPropertyEntityRepository).getAllPartitions(); + + partitionServiceImpl.getAllPartitions(); + + verify(partitionListCache, times(1)).get(PartitionServiceImpl.PARTITION_LIST_KEY); + verify(partitionPropertyEntityRepository, times(1)).getAllPartitions(); + verify(partitionListCache, times(1)).put(PartitionServiceImpl.PARTITION_LIST_KEY, partitions); + } +} \ No newline at end of file diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java index fadb1f15..00134a99 100644 --- a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java +++ b/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java @@ -17,161 +17,179 @@ package org.opengroup.osdu.partition.provider.gcp.service; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opengroup.osdu.core.common.cache.ICache; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; import org.opengroup.osdu.partition.logging.AuditLogger; import org.opengroup.osdu.partition.model.PartitionInfo; import org.opengroup.osdu.partition.model.Property; import org.opengroup.osdu.partition.provider.gcp.model.PartitionPropertyEntity; import org.opengroup.osdu.partition.provider.gcp.repository.PartitionPropertyEntityRepository; -@RunWith(MockitoJUnitRunner.class) -public class PartitionServiceImplTest { - - private static final String PARTITION_ID = "newPartition"; - private static final boolean SENSITIVE = false; - private static final String NAME = "new-key"; - private static final String VALUE = "new-value"; - - - @Mock - private PartitionPropertyEntityRepository partitionPropertyEntityRepository; - - @Mock - private AuditLogger auditLogger; - - @InjectMocks - private PartitionServiceImpl partitionServiceImpl; - - private PartitionInfo expectedPartitionInfo; - private PartitionPropertyEntity partitionPropertyEntity; - private Optional> partitionPropertyEntityList; - private Optional> emptyList; - - @Before - public void setup() { - this.expectedPartitionInfo = new PartitionInfo(); - - Property property = new Property(); - property.setSensitive(SENSITIVE); - property.setValue(VALUE); - - Map properties = new HashMap<>(); - properties.put(NAME, property); - - this.expectedPartitionInfo.setProperties(properties); - - partitionPropertyEntity = new PartitionPropertyEntity(); - partitionPropertyEntity.setPartitionId(PARTITION_ID); - partitionPropertyEntity.setName(NAME); - partitionPropertyEntity.setSensitive(SENSITIVE); - partitionPropertyEntity.setValue(VALUE); - - List entities = new ArrayList<>(); - entities.add(partitionPropertyEntity); - this.partitionPropertyEntityList = Optional.of(entities); - - this.emptyList = Optional.empty(); - } - - @Test - public void should_createPartition_when_partitionDoesNotExist() { - when(this.partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) - .thenReturn(this.emptyList, this.partitionPropertyEntityList); +import java.util.*; - PartitionInfo actualPartitionInfo = this.partitionServiceImpl - .createPartition(PARTITION_ID, this.expectedPartitionInfo); +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; - assertEquals(this.expectedPartitionInfo, actualPartitionInfo); - } - - @Test(expected = AppException.class) - public void should_throwAnException_when_createPartitionWhichAlreadyExists() { - when(this.partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) - .thenReturn(this.emptyList); - - this.partitionServiceImpl.createPartition(PARTITION_ID, new PartitionInfo()); - } - - @Test - public void should_updatePartition_when_partitionExists() { - when(this.partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) - .thenReturn(this.partitionPropertyEntityList); - when(this.partitionPropertyEntityRepository.findByPartitionIdAndName(PARTITION_ID, NAME)) - .thenReturn(null); - - PartitionInfo actualPartitionInfo = this.partitionServiceImpl - .updatePartition(PARTITION_ID, this.expectedPartitionInfo); - - assertEquals(this.expectedPartitionInfo, actualPartitionInfo); - } - - @Test(expected = AppException.class) - public void should_throwAnException_when_updatePartitionWhichDoesNotExist() { - when(this.partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) - .thenReturn(this.emptyList); - this.partitionServiceImpl.createPartition(PARTITION_ID, new PartitionInfo()); - } - - @Test - public void should_getPartition_when_partitionExists() { - when(this.partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) - .thenReturn(this.partitionPropertyEntityList); - - PartitionInfo actualPartitionInfo = this.partitionServiceImpl.getPartition(PARTITION_ID); - - assertEquals(this.expectedPartitionInfo, actualPartitionInfo); - } - - @Test(expected = AppException.class) - public void should_throwAnException_when_getPartitionWhichDoesNotExist() { - when(this.partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) - .thenReturn(this.emptyList); - this.partitionServiceImpl.getPartition(PARTITION_ID); - } - - @Test - public void should_deletePartition_when_partitionExists() { - when(this.partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) - .thenReturn(this.partitionPropertyEntityList); - doNothing().when(this.partitionPropertyEntityRepository).deleteByPartitionId(PARTITION_ID); - - boolean expected = true; - boolean actual = partitionServiceImpl.deletePartition(PARTITION_ID); - - assertEquals(expected, actual); - } +@ExtendWith(MockitoExtension.class) +public class PartitionServiceImplTest { - @Test(expected = AppException.class) - public void should_throwAnException_when_deletePartitionWhichDoesNotExist() { - when(this.partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) - .thenReturn(this.emptyList); - this.partitionServiceImpl.deletePartition(PARTITION_ID); - } - - @Test - public void should_getAllPartitions() { - List expectedPartitions = new ArrayList<>(); - expectedPartitions.add(PARTITION_ID); + private static final String PARTITION_ID = "newPartition"; + private static final boolean SENSITIVE = false; + private static final String NAME = "new-key"; + private static final String VALUE = "new-value"; + + @Mock + private ICache partitionServiceCache; - when(this.partitionPropertyEntityRepository.getAllPartitions()).thenReturn(expectedPartitions); + @Mock + private ICache> partitionListCache; - List actualPartitions = this.partitionServiceImpl.getAllPartitions(); + @Mock + private PartitionPropertyEntityRepository partitionPropertyEntityRepository; - assertEquals(expectedPartitions, actualPartitions); - } + @Mock + private AuditLogger auditLogger; + + private PartitionServiceImpl partitionServiceImpl; + + private PartitionInfo expectedPartitionInfo; + private PartitionPropertyEntity partitionPropertyEntity; + private Optional> partitionPropertyEntityList; + private Optional> emptyList; + + @BeforeEach + public void setup() { + partitionServiceImpl = new PartitionServiceImpl( + partitionPropertyEntityRepository, + mock(IKmsClient.class), + auditLogger, + partitionServiceCache, + partitionListCache + ); + + expectedPartitionInfo = new PartitionInfo(); + + Property property = new Property(); + property.setSensitive(SENSITIVE); + property.setValue(VALUE); + + Map properties = new HashMap<>(); + properties.put(NAME, property); + + expectedPartitionInfo.setProperties(properties); + + partitionPropertyEntity = new PartitionPropertyEntity(); + partitionPropertyEntity.setPartitionId(PARTITION_ID); + partitionPropertyEntity.setName(NAME); + partitionPropertyEntity.setSensitive(SENSITIVE); + partitionPropertyEntity.setValue(VALUE); + + List entities = new ArrayList<>(); + entities.add(partitionPropertyEntity); + partitionPropertyEntityList = Optional.of(entities); + + emptyList = Optional.empty(); + } + + @Test + public void should_createPartition_when_partitionDoesNotExist() { + when(partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) + .thenReturn(emptyList, partitionPropertyEntityList); + + PartitionInfo actualPartitionInfo = partitionServiceImpl + .createPartition(PARTITION_ID, expectedPartitionInfo); + + assertEquals(expectedPartitionInfo, actualPartitionInfo); + } + + @Test + public void should_throwAnException_when_createPartitionWhichAlreadyExists() { + assertThrows(AppException.class, () -> { + when(partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) + .thenReturn(emptyList); + + partitionServiceImpl.createPartition(PARTITION_ID, new PartitionInfo()); + }); + } + + @Test + public void should_updatePartition_when_partitionExists() { + when(partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) + .thenReturn(partitionPropertyEntityList); + when(partitionPropertyEntityRepository.findByPartitionIdAndName(PARTITION_ID, NAME)) + .thenReturn(null); + + PartitionInfo actualPartitionInfo = partitionServiceImpl + .updatePartition(PARTITION_ID, expectedPartitionInfo); + + assertEquals(expectedPartitionInfo, actualPartitionInfo); + } + + @Test + public void should_throwAnException_when_updatePartitionWhichDoesNotExist() { + assertThrows(AppException.class, () -> { + when(partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) + .thenReturn(emptyList); + partitionServiceImpl.createPartition(PARTITION_ID, new PartitionInfo()); + }); + } + + @Test + public void should_getPartition_when_partitionExists() { + when(partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) + .thenReturn(partitionPropertyEntityList); + + PartitionInfo actualPartitionInfo = partitionServiceImpl.getPartition(PARTITION_ID); + + assertEquals(expectedPartitionInfo, actualPartitionInfo); + } + + @Test + public void should_throwAnException_when_getPartitionWhichDoesNotExist() { + assertThrows(AppException.class, () -> { + when(partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) + .thenReturn(emptyList); + partitionServiceImpl.getPartition(PARTITION_ID); + }); + } + + @Test + public void should_deletePartition_when_partitionExists() { + when(partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) + .thenReturn(partitionPropertyEntityList); + doNothing().when(partitionPropertyEntityRepository).deleteByPartitionId(PARTITION_ID); + + boolean expected = true; + boolean actual = partitionServiceImpl.deletePartition(PARTITION_ID); + + assertEquals(expected, actual); + } + + @Test + public void should_throwAnException_when_deletePartitionWhichDoesNotExist() { + assertThrows(AppException.class, () -> { + when(partitionPropertyEntityRepository.findByPartitionId(PARTITION_ID)) + .thenReturn(emptyList); + partitionServiceImpl.deletePartition(PARTITION_ID); + }); + } + + @Test + public void should_getAllPartitions() { + List expectedPartitions = new ArrayList<>(); + expectedPartitions.add(PARTITION_ID); + + when(partitionPropertyEntityRepository.getAllPartitions()).thenReturn(expectedPartitions); + + List actualPartitions = partitionServiceImpl.getAllPartitions(); + + assertEquals(expectedPartitions, actualPartitions); + } } \ No newline at end of file diff --git a/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/cache/PartitionListCacheImpl.java b/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/cache/PartitionListCacheImpl.java deleted file mode 100644 index a1dc1232..00000000 --- a/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/cache/PartitionListCacheImpl.java +++ /dev/null @@ -1,45 +0,0 @@ -/* Licensed Materials - Property of IBM */ -/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/ - -package org.opengroup.osdu.partition.provider.ibm.cache; - -import java.util.List; - -import javax.annotation.Resource; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -@Service -@Qualifier("partitionListCache") -public class PartitionListCacheImpl implements IPartitionServiceCache> { - - - @Autowired - @Qualifier("partitionListCache") - private ICache> cache; - - @Override - public void put(String s, List o) { - this.cache.put(s, o); - } - - @Override - public List get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } - -} diff --git a/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/cache/PartitionServiceCacheImpl.java b/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/cache/PartitionServiceCacheImpl.java deleted file mode 100644 index 4afd7cd1..00000000 --- a/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/cache/PartitionServiceCacheImpl.java +++ /dev/null @@ -1,42 +0,0 @@ -/* Licensed Materials - Property of IBM */ -/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/ - -package org.opengroup.osdu.partition.provider.ibm.cache; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; - -@Service -@Qualifier("partitionServiceCache") -public class PartitionServiceCacheImpl implements IPartitionServiceCache { - - @Autowired - @Qualifier("partitionServiceCache") - private ICache cache; - - @Override - public void put(String s, PartitionInfo o) { - this.cache.put(s, o); - } - - @Override - public PartitionInfo get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } -} diff --git a/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/service/PartitionServiceImpl.java b/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/service/PartitionServiceImpl.java index 247f4840..37e7a071 100644 --- a/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/service/PartitionServiceImpl.java +++ b/provider/partition-ibm/src/main/java/org/opengroup/osdu/partition/provider/ibm/service/PartitionServiceImpl.java @@ -10,6 +10,7 @@ import java.util.List; import javax.annotation.PostConstruct; import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.cache.ICache; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.ibm.auth.ServiceCredentials; @@ -18,6 +19,7 @@ import org.opengroup.osdu.partition.model.PartitionInfo; import org.opengroup.osdu.partition.provider.ibm.model.PartitionDoc; import org.opengroup.osdu.partition.provider.interfaces.IPartitionService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -33,10 +35,19 @@ import lombok.extern.slf4j.Slf4j; public class PartitionServiceImpl implements IPartitionService { private static final String PARTITION_DATABASE = "partition"; + private static final String PARTITION_LIST_KEY = "getAllPartitions"; @Autowired private JaxRsDpsLog logger; + @Autowired + @Qualifier("partitionServiceCache") + private ICache partitionServiceCache; + + @Autowired + @Qualifier("partitionListCache") + private ICache> partitionListCache; + Database db; private IBMCloudantClientFactory cloudantFactory; @@ -69,10 +80,15 @@ public class PartitionServiceImpl implements IPartitionService { @Override public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) { + if (partitionServiceCache.get(partitionId) != null) { + throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist"); + } + + PartitionInfo pi; PartitionDoc partitionDoc = new PartitionDoc(partitionId, partitionInfo); try { db.save(partitionDoc); - return partitionInfo; + pi = partitionInfo; } catch (DocumentConflictException e) { log.error("Partition already exists"); throw new AppException(e.getStatusCode(), "Conflict", "partition already exists", e); @@ -81,11 +97,18 @@ public class PartitionServiceImpl implements IPartitionService { e.printStackTrace(); throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage(), "Partition creation failed", e); } - + + if (pi != null) { + partitionServiceCache.put(partitionId, pi); + partitionListCache.clearAll(); + } + + return pi; } @Override public PartitionInfo updatePartition(String partitionId, PartitionInfo partitionInfo) { + PartitionInfo pi; if (partitionInfo.getProperties().containsKey("id")) { throw new AppException(HttpStatus.SC_BAD_REQUEST, "can not update id", "the field id can not be updated"); } @@ -93,7 +116,7 @@ public class PartitionServiceImpl implements IPartitionService { PartitionDoc partitionDoc = db.find(PartitionDoc.class, partitionId); partitionDoc.getPartitionInfo().getProperties().putAll(partitionInfo.getProperties()); Response update = db.update(partitionDoc); - return partitionDoc.getPartitionInfo(); + pi = partitionDoc.getPartitionInfo(); } catch (NoDocumentException e) { log.error(String.format("%s partition does not exists", partitionId)); e.printStackTrace(); @@ -110,23 +133,38 @@ public class PartitionServiceImpl implements IPartitionService { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Partition update failed", e.getMessage(), e); } + if(pi != null) { + partitionServiceCache.put(partitionId, pi); + } + + return pi; } @Override public PartitionInfo getPartition(String partitionId) { - PartitionDoc partitionDoc = null; - try { - partitionDoc = db.find(PartitionDoc.class, partitionId); - } catch (NoDocumentException e) { - log.error(String.format("%s partition does not exists", partitionId)); - e.printStackTrace(); - throw new AppException(e.getStatusCode(), e.getReason(), String.format("%s partition does not exists", partitionId), e); - } catch (Exception e) { - log.error("Partition could not found"); - e.printStackTrace(); - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "unknown error", "Partition could not found", e ); + PartitionInfo pi = partitionServiceCache.get(partitionId); + + if (pi == null) { + PartitionDoc partitionDoc = null; + try { + partitionDoc = db.find(PartitionDoc.class, partitionId); + } catch (NoDocumentException e) { + log.error(String.format("%s partition does not exists", partitionId)); + e.printStackTrace(); + throw new AppException(e.getStatusCode(), e.getReason(), String.format("%s partition does not exists", partitionId), e); + } catch (Exception e) { + log.error("Partition could not found"); + e.printStackTrace(); + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "unknown error", "Partition could not found", e ); + } + pi = partitionDoc.getPartitionInfo(); + + if (pi != null) { + partitionServiceCache.put(partitionId, pi); + } } - return partitionDoc.getPartitionInfo(); + + return pi; } @Override @@ -143,26 +181,38 @@ public class PartitionServiceImpl implements IPartitionService { log.error("Deletion Failed. Unexpected error"); e.printStackTrace(); } + if(deleteStatus.getStatusCode() == 200) { + if (partitionServiceCache.get(partitionId) != null) { + partitionServiceCache.delete(partitionId); + } + partitionListCache.clearAll(); return true; - } + } + return false; - } @Override public List getAllPartitions() { - List partitionList = null; - try { - partitionList = db.getAllDocsRequestBuilder().includeDocs(true).build().getResponse().getDocIds(); - } catch (IOException e) { - log.error("Partitions could not found. IOException occurred", e); - e.printStackTrace(); - } catch (Exception e) { - log.error("Partition could not found.", e); - e.printStackTrace(); + List partitions = partitionListCache.get(PARTITION_LIST_KEY); + + if (partitions == null) { + try { + partitions = db.getAllDocsRequestBuilder().includeDocs(true).build().getResponse().getDocIds(); + } catch (IOException e) { + log.error("Partitions could not found. IOException occurred", e); + e.printStackTrace(); + } catch (Exception e) { + log.error("Partition could not found.", e); + e.printStackTrace(); + } + + if (partitions != null) { + partitionListCache.put(PARTITION_LIST_KEY, partitions); + } } - return partitionList; + return partitions; } } -- GitLab From 4adf1d851e8709471337fa0aaebaadf11f860a33 Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Tue, 14 Sep 2021 17:38:55 +0300 Subject: [PATCH 2/9] Updating FOSSA NOTICE --- NOTICE | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/NOTICE b/NOTICE index 8e4ead69..8d76a606 100644 --- a/NOTICE +++ b/NOTICE @@ -111,6 +111,8 @@ The following software have components provided under the terms of this license: - Identity and Access Management (IAM) API v1-rev20210226-1.31.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam) - J2ObjC Annotations (from https://github.com/google/j2objc/) - JBoss Logging 3 (from http://www.jboss.org) +- JBoss Marshalling API (from https://repo1.maven.org/maven2/org/jboss/marshalling/jboss-marshalling) +- JBoss Marshalling River (from https://repo1.maven.org/maven2/org/jboss/marshalling/jboss-marshalling-river) - JCIP Annotations under Apache License (from http://stephenc.github.com/jcip-annotations) - JDOM (from http://www.jdom.org) - JSON Small and Fast Parser (from https://repo1.maven.org/maven2/net/minidev/json-smart) @@ -118,6 +120,7 @@ The following software have components provided under the terms of this license: - JSON Web Token support for the JVM (from https://repo1.maven.org/maven2/io/jsonwebtoken/jjwt) - JSON library from Android SDK (from http://developer.android.com/sdk) - JSONassert (from https://github.com/skyscreamer/JSONassert) +- JSR107 API and SPI (from https://github.com/jsr107/jsr107spec) - Jackson (from http://jackson.codehaus.org) - Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2) - Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary) @@ -125,7 +128,6 @@ The following software have components provided under the terms of this license: - Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8) - Jackson extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson) - Jackson module: Afterburner (from https://github.com/FasterXML/jackson-modules-base) -- Jackson module: JAXB Annotations (from https://github.com/FasterXML/jackson-modules-base) - Jackson-annotations (from http://github.com/FasterXML/jackson) - Jackson-annotations (from http://github.com/FasterXML/jackson) - Jackson-core (from https://github.com/FasterXML/jackson-core) @@ -133,6 +135,7 @@ The following software have components provided under the terms of this license: - Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding) - Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text) - Jackson-datatype-JODA (from http://wiki.fasterxml.com/JacksonModuleJoda) +- Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations) - Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names) - Jakarta Bean Validation API (from https://beanvalidation.org) - Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el) @@ -224,15 +227,23 @@ The following software have components provided under the terms of this license: - Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j) - QpidJMS Client (from ) - Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty) -- Retrofit (from https://github.com/square/retrofit) +- Redisson (from http://redisson.org) +- Retrofit (from https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit) +- RxJava (from https://github.com/ReactiveX/RxJava) - Servlet Specification 2.5 API (from http://jetty.mortbay.org) - SnakeYAML (from http://www.snakeyaml.org) - Spring AOP (from https://github.com/spring-projects/spring-framework) - Spring Beans (from https://github.com/spring-projects/spring-framework) -- Spring Boot Log4j 2 Starter (from http://projects.spring.io/spring-boot/) +- Spring Boot Actuator (from http://projects.spring.io/spring-boot/) +- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/) +- Spring Boot Log4j 2 Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-log4j2) - Spring Boot Security Starter (from http://projects.spring.io/spring-boot/) - Spring Boot Security Starter (from http://projects.spring.io/spring-boot/) -- Spring Boot Tomcat Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat) +- Spring Boot Starter (from http://projects.spring.io/spring-boot/) +- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/) +- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/) +- Spring Boot Tomcat Starter (from http://projects.spring.io/spring-boot/) +- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/) - Spring Cloud GCP Autoconfigure Module (from https://repo1.maven.org/maven2/org/springframework/cloud/spring-cloud-gcp-autoconfigure) - Spring Cloud GCP Core Module (from https://repo1.maven.org/maven2/org/springframework/cloud/spring-cloud-gcp-core) - Spring Cloud GCP Datastore Module (from https://repo1.maven.org/maven2/org/springframework/cloud/spring-cloud-gcp-data-datastore) @@ -248,11 +259,14 @@ The following software have components provided under the terms of this license: - Spring Messaging (from https://github.com/spring-projects/spring-framework) - Spring Plugin - Metadata Extension (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-metadata) - Spring Plugin Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core) +- Spring Security - Core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core) - Spring TestContext Framework (from https://github.com/spring-projects/spring-framework) - Spring Transaction (from https://github.com/spring-projects/spring-framework) - Spring Web (from https://github.com/spring-projects/spring-framework) - Spring Web MVC (from https://github.com/spring-projects/spring-framework) - Spring WebFlux (from https://github.com/spring-projects/spring-framework) +- Vavr (from http://vavr.io) +- Vavr Match (from http://vavr.io) - Woodstox (from https://github.com/FasterXML/woodstox) - Xerces2-j (from https://xerces.apache.org/xerces2-j/) - Zipkin Core Library (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin) @@ -309,25 +323,20 @@ The following software have components provided under the terms of this license: - proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1) - proto-google-common-protos (from https://github.com/googleapis/java-iam/proto-google-common-protos) - proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1) +- resilience4j (from https://resilience4j.readme.io) +- resilience4j (from https://resilience4j.readme.io) - rxjava (from https://github.com/ReactiveX/RxJava) - spring-boot (from https://spring.io/projects/spring-boot) -- spring-boot-actuator (from https://spring.io/projects/spring-boot) - spring-boot-actuator-autoconfigure (from https://spring.io/projects/spring-boot) - spring-boot-autoconfigure (from https://spring.io/projects/spring-boot) -- spring-boot-starter (from https://spring.io/projects/spring-boot) -- spring-boot-starter-actuator (from https://spring.io/projects/spring-boot) - spring-boot-starter-json (from https://spring.io/projects/spring-boot) - spring-boot-starter-logging (from https://spring.io/projects/spring-boot) - spring-boot-starter-reactor-netty (from https://spring.io/projects/spring-boot) -- spring-boot-starter-test (from https://spring.io/projects/spring-boot) -- spring-boot-starter-test (from https://spring.io/projects/spring-boot) - spring-boot-starter-validation (from https://spring.io/projects/spring-boot) -- spring-boot-starter-web (from https://spring.io/projects/spring-boot) - spring-boot-starter-webflux (from https://spring.io/projects/spring-boot) - spring-boot-test (from https://spring.io/projects/spring-boot) - spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot) - spring-security-config (from https://spring.io/spring-security) -- spring-security-core (from https://spring.io/spring-security) - spring-security-oauth2-client (from https://spring.io/spring-security) - spring-security-oauth2-client (from https://spring.io/spring-security) - spring-security-oauth2-core (from https://spring.io/spring-security) @@ -364,6 +373,8 @@ The following software have components provided under the terms of this license: - Hamcrest (from http://hamcrest.org/JavaHamcrest/) - Hamcrest Core (from http://hamcrest.org/) - HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/) +- Jodd BeanUtil (from http://jodd.org) +- Jodd Core (from http://jodd.org) - Plexus Common Utilities (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) - Reflections (from http://code.google.com/p/reflections/) - Stax2 API (from http://github.com/FasterXML/stax2-api) @@ -388,6 +399,8 @@ The following software have components provided under the terms of this license: - HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/) - Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) - Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api) +- Jodd BeanUtil (from http://jodd.org) +- Jodd Core (from http://jodd.org) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -396,6 +409,7 @@ The following software have components provided under the terms of this license: - Plexus Common Utilities (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils) - Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) - Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util) +- Redisson (from http://redisson.org) - Reflections (from http://code.google.com/p/reflections/) - SnakeYAML (from http://www.snakeyaml.org) - Spring Core (from https://github.com/spring-projects/spring-framework) @@ -623,6 +637,7 @@ The following software have components provided under the terms of this license: - Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions) - JUL to SLF4J bridge (from http://www.slf4j.org) - Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) +- Java JWT (from http://www.jwt.io) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -650,17 +665,16 @@ The following software have components provided under the terms of this license: - SLF4J API Module (from http://www.slf4j.org) - Spongy Castle (from http://rtyley.github.io/spongycastle/) - Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos) +- Spring Security - Core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core) - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) - azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/) - documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/) -- java jwt (from https://github.com/auth0/java-jwt) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - mockito-core (from https://github.com/mockito/mockito) - mockito-core (from https://github.com/mockito/mockito) - mockito-junit-jupiter (from https://github.com/mockito/mockito) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java) -- spring-security-core (from https://spring.io/spring-security) ======================================================================== MPL-1.1 @@ -771,6 +785,8 @@ The following software have components provided under the terms of this license: - JUnit Platform Engine API (from https://junit.org/junit5/) - Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) - Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api) +- Jodd BeanUtil (from http://jodd.org) +- Jodd Core (from http://jodd.org) - Spongy Castle (from http://rtyley.github.io/spongycastle/) - xml-apis (from ) -- GitLab From 486f105ddd66190c0bd282c41ce62b3261775e7e Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Fri, 17 Sep 2021 14:19:25 +0300 Subject: [PATCH 3/9] bugfix/16 Changes after review. --- .../provider/azure/service/PartitionServiceImpl.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java index 2f405b45..a1ab1da6 100644 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java +++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java @@ -71,7 +71,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) { - if (partitionServiceCache.get(partitionId) != null || tableStore.partitionExists(partitionId)) { + if (getPartition(partitionId) != null) { throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist"); } tableStore.addPartition(partitionId, partitionInfo); @@ -86,7 +86,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public PartitionInfo updatePartition(String partitionId, PartitionInfo partitionInfo) { - if (!tableStore.partitionExists(partitionId)) { + if (getPartition(partitionId) == null) { throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); } @@ -121,7 +121,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public boolean deletePartition(String partitionId) { - if (!tableStore.partitionExists(partitionId)) { + if (getPartition(partitionId) == null) { throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); } @@ -162,14 +162,14 @@ public class PartitionServiceImpl implements IPartitionService { } } } catch (InterruptedException ex) { - log.error(String.format("InterruptedException caught when lock the cache key %s: %s", key, ex)); Thread.currentThread().interrupt(); + log.error(String.format("InterruptedException on cache rebuild for key=%s.", key), ex); } finally { if (locked) { try { cacheEntryLock.unlock(); } catch (Exception ex) { - log.warning(String.format("unlock exception: %s", ex)); + log.error(String.format("Exception when unlock on cache rebuild for key=%s", key), ex); } } } -- GitLab From 8b04a4ad60805af0279803d2b0208b308a95a655 Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Fri, 17 Sep 2021 15:05:31 +0300 Subject: [PATCH 4/9] bugfix/16 Changes after review. --- .../cache/PartitionListCacheImpl.java | 53 ----------- .../cache/PartitionServiceCacheImpl.java | 52 ----------- .../service/PartitionServiceImpl.java | 91 +++++++++++++++---- 3 files changed, 71 insertions(+), 125 deletions(-) delete mode 100644 provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/cache/PartitionListCacheImpl.java delete mode 100644 provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/cache/PartitionServiceCacheImpl.java diff --git a/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/cache/PartitionListCacheImpl.java b/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/cache/PartitionListCacheImpl.java deleted file mode 100644 index 83853f6c..00000000 --- a/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/cache/PartitionListCacheImpl.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2002-2021 Google LLC - Copyright 2002-2021 EPAM Systems, Inc - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package org.opengroup.osdu.partition.provider.reference.cache; - -import java.util.List; -import lombok.RequiredArgsConstructor; -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -@Service -@Qualifier("partitionListCache") -@RequiredArgsConstructor -public class PartitionListCacheImpl implements IPartitionServiceCache> { - - private final ICache> cache; - - @Override - public void put(String s, List o) { - this.cache.put(s, o); - } - - @Override - public List get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } -} diff --git a/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/cache/PartitionServiceCacheImpl.java b/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/cache/PartitionServiceCacheImpl.java deleted file mode 100644 index 55a0c33e..00000000 --- a/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/cache/PartitionServiceCacheImpl.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright 2002-2021 Google LLC - Copyright 2002-2021 EPAM Systems, Inc - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package org.opengroup.osdu.partition.provider.reference.cache; - -import lombok.RequiredArgsConstructor; -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.partition.model.PartitionInfo; -import org.opengroup.osdu.partition.provider.interfaces.IPartitionServiceCache; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -@Service -@Qualifier("partitionServiceCache") -@RequiredArgsConstructor -public class PartitionServiceCacheImpl implements IPartitionServiceCache { - - private final ICache cache; - - @Override - public void put(String s, PartitionInfo o) { - this.cache.put(s, o); - } - - @Override - public PartitionInfo get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } -} diff --git a/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/service/PartitionServiceImpl.java b/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/service/PartitionServiceImpl.java index 32bf7283..661ac968 100644 --- a/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/service/PartitionServiceImpl.java +++ b/provider/partition-reference/src/main/java/org/opengroup/osdu/partition/provider/reference/service/PartitionServiceImpl.java @@ -17,11 +17,9 @@ package org.opengroup.osdu.partition.provider.reference.service; -import java.util.Collections; -import java.util.List; -import java.util.Optional; import lombok.RequiredArgsConstructor; import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.cache.ICache; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; import org.opengroup.osdu.partition.logging.AuditLogger; @@ -31,28 +29,48 @@ import org.opengroup.osdu.partition.provider.interfaces.IPartitionService; import org.opengroup.osdu.partition.provider.reference.repository.PartitionPropertyEntityRepository; import org.springframework.stereotype.Service; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + @Service @RequiredArgsConstructor public class PartitionServiceImpl implements IPartitionService { private static final String UNKNOWN_ERROR_REASON = "unknown error"; + private static final String PARTITION_LIST_KEY = "getAllPartitions"; + private final PartitionPropertyEntityRepository partitionPropertyEntityRepository; private final IKmsClient kmsClient; private final AuditLogger auditLogger; + private final ICache partitionServiceCache; + + private final ICache> partitionListCache; + @Override public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) { + if (partitionServiceCache.get(partitionId) != null) + throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist"); + if (partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) { throw new AppException(HttpStatus.SC_CONFLICT, UNKNOWN_ERROR_REASON, - "Partition already exists."); + "Partition already exists."); } partitionInfo.getProperties() - .forEach((key, property) -> encryptPartitionPropertyEntityIfNeeded(property)); + .forEach((key, property) -> encryptPartitionPropertyEntityIfNeeded(property)); partitionPropertyEntityRepository.createPartition(partitionId, partitionInfo); - return getPartition(partitionId); + PartitionInfo pi = getPartition(partitionId); + + if (pi != null) { + partitionServiceCache.put(partitionId, pi); + partitionListCache.clearAll(); + } + + return pi; } @Override @@ -60,16 +78,16 @@ public class PartitionServiceImpl implements IPartitionService { if (partitionInfo.getProperties().containsKey("id")) { this.auditLogger.updatePartitionSecretFailure(Collections.singletonList(partitionId)); throw new AppException(HttpStatus.SC_BAD_REQUEST, "can not update id", - "the field id can not be updated"); + "the field id can not be updated"); } if (!partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) { this.auditLogger.updatePartitionSecretFailure(Collections.singletonList(partitionId)); throw new AppException(HttpStatus.SC_NOT_FOUND, UNKNOWN_ERROR_REASON, - "An attempt to update not existing partition."); + "An attempt to update not existing partition."); } partitionInfo.getProperties().forEach((key, value) -> { Optional property = this.partitionPropertyEntityRepository - .findByPartitionIdAndName(partitionId, key); + .findByPartitionIdAndName(partitionId, key); if (property.isPresent()) { property.get().setSensitive(value.isSensitive()); property.get().setValue(value.getValue()); @@ -78,19 +96,35 @@ public class PartitionServiceImpl implements IPartitionService { encryptPartitionPropertyEntityIfNeeded(value); }); partitionPropertyEntityRepository.updatePartition(partitionId, partitionInfo); - return getPartition(partitionId); + PartitionInfo pi = getPartition(partitionId); + + if(pi != null) { + partitionServiceCache.put(partitionId, pi); + } + + return pi; } @Override public PartitionInfo getPartition(String partitionId) { - Optional result = partitionPropertyEntityRepository.findByPartitionId( - partitionId); - if (result.isPresent()) { - result.get().getProperties() - .forEach((key, property) -> decryptPartitionPropertyIfNeeded(property)); - return result.get(); + PartitionInfo pi = (PartitionInfo) partitionServiceCache.get(partitionId); + + if (pi == null) { + Optional result = partitionPropertyEntityRepository.findByPartitionId( + partitionId); + if (result.isPresent()) { + result.get().getProperties() + .forEach((key, property) -> decryptPartitionPropertyIfNeeded(property)); + return result.get(); + } + pi = new PartitionInfo(); + + if (pi != null) { + partitionServiceCache.put(partitionId, pi); + } } - return new PartitionInfo(); + + return pi; } @Override @@ -98,14 +132,31 @@ public class PartitionServiceImpl implements IPartitionService { if (!partitionPropertyEntityRepository.findByPartitionId(partitionId).isPresent()) { this.auditLogger.deletePartitionFailure(Collections.singletonList(partitionId)); throw new AppException(HttpStatus.SC_NOT_FOUND, UNKNOWN_ERROR_REASON, - "An attempt to delete not existing partition."); + "An attempt to delete not existing partition."); } - return partitionPropertyEntityRepository.isDeletedPartitionInfoByPartitionId(partitionId); + if (partitionPropertyEntityRepository.isDeletedPartitionInfoByPartitionId(partitionId)) { + if (partitionServiceCache.get(partitionId) != null) { + partitionServiceCache.delete(partitionId); + } + partitionListCache.clearAll(); + return true; + } + + return false; } @Override public List getAllPartitions() { - return partitionPropertyEntityRepository.getAllPartitions(); + List partitions = (List)partitionListCache.get(PARTITION_LIST_KEY); + + if (partitions == null) { + partitions = partitionPropertyEntityRepository.getAllPartitions(); + + if (partitions != null) { + partitionListCache.put(PARTITION_LIST_KEY, partitions); + } + } + return partitions; } private void encryptPartitionPropertyEntityIfNeeded(Property property) { -- GitLab From 76ffc0bb4473ad1d1c5f7c39276fbb0bf5cf75f8 Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Fri, 17 Sep 2021 17:13:18 +0300 Subject: [PATCH 5/9] Revert "bugfix/16" This reverts commit 486f105ddd66190c0bd282c41ce62b3261775e7e. --- .../provider/azure/service/PartitionServiceImpl.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java index a1ab1da6..2f405b45 100644 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java +++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java @@ -71,7 +71,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) { - if (getPartition(partitionId) != null) { + if (partitionServiceCache.get(partitionId) != null || tableStore.partitionExists(partitionId)) { throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist"); } tableStore.addPartition(partitionId, partitionInfo); @@ -86,7 +86,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public PartitionInfo updatePartition(String partitionId, PartitionInfo partitionInfo) { - if (getPartition(partitionId) == null) { + if (!tableStore.partitionExists(partitionId)) { throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); } @@ -121,7 +121,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public boolean deletePartition(String partitionId) { - if (getPartition(partitionId) == null) { + if (!tableStore.partitionExists(partitionId)) { throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); } @@ -162,14 +162,14 @@ public class PartitionServiceImpl implements IPartitionService { } } } catch (InterruptedException ex) { + log.error(String.format("InterruptedException caught when lock the cache key %s: %s", key, ex)); Thread.currentThread().interrupt(); - log.error(String.format("InterruptedException on cache rebuild for key=%s.", key), ex); } finally { if (locked) { try { cacheEntryLock.unlock(); } catch (Exception ex) { - log.error(String.format("Exception when unlock on cache rebuild for key=%s", key), ex); + log.warning(String.format("unlock exception: %s", ex)); } } } -- GitLab From 6739d6c5f66899bf41d89b3c235f5b480e1f7d24 Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Fri, 17 Sep 2021 17:16:31 +0300 Subject: [PATCH 6/9] bugfix/16 Changes after review. --- .../provider/azure/service/PartitionServiceImpl.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java index 2f405b45..585e588d 100644 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java +++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java @@ -71,7 +71,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) { - if (partitionServiceCache.get(partitionId) != null || tableStore.partitionExists(partitionId)) { + if (isPartitionExists(partitionId)) { throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist"); } tableStore.addPartition(partitionId, partitionInfo); @@ -86,7 +86,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public PartitionInfo updatePartition(String partitionId, PartitionInfo partitionInfo) { - if (!tableStore.partitionExists(partitionId)) { + if (!isPartitionExists(partitionId)) { throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); } @@ -121,7 +121,7 @@ public class PartitionServiceImpl implements IPartitionService { @Override public boolean deletePartition(String partitionId) { - if (!tableStore.partitionExists(partitionId)) { + if (!isPartitionExists(partitionId)) { throw new AppException(HttpStatus.SC_NOT_FOUND, PARTITION_NOT_FOUND, String.format("%s partition not found", partitionId)); } @@ -178,4 +178,8 @@ public class PartitionServiceImpl implements IPartitionService { org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE.getReasonPhrase(), "Failed to get the partition(s)."); } + + private boolean isPartitionExists(String partitionId) { + return partitionServiceCache.get(partitionId) != null || tableStore.partitionExists(partitionId); + } } -- GitLab From 3c46e3ee85004caa3cc8d98c046f31503b72966b Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Fri, 17 Sep 2021 18:00:09 +0300 Subject: [PATCH 7/9] Updating FOSSA NOTICE --- NOTICE | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NOTICE b/NOTICE index 8d76a606..10b7821a 100644 --- a/NOTICE +++ b/NOTICE @@ -72,7 +72,6 @@ The following software have components provided under the terms of this license: - Commons Digester (from http://commons.apache.org/digester/) - Commons Lang (from http://commons.apache.org/lang/) - Converter: Jackson (from https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson) -- Core Reactor components (from https://github.com/reactor/reactor) - Core functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty) - Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default) - Doxia :: APT Module (from http://maven.apache.org/doxia/doxia/doxia-modules/doxia-module-apt/) @@ -82,7 +81,7 @@ The following software have components provided under the terms of this license: - Doxia :: XDoc Module (from http://maven.apache.org/doxia/doxia/doxia-modules/doxia-module-xdoc/) - Doxia :: XHTML Module (from http://maven.apache.org/doxia/doxia/doxia-modules/doxia-module-xhtml/) - Doxia Sitetools :: Decoration Model (from http://maven.apache.org/doxia/doxia-sitetools/doxia-decoration-model/) -- Doxia Sitetools :: Site Renderer Component (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/) +- Doxia Sitetools :: Site Renderer (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-site-renderer) - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) - Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) - Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine) @@ -212,6 +211,7 @@ The following software have components provided under the terms of this license: - Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type) - Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) - Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags) +- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor-core) - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) - Objenesis (from http://objenesis.org) - OkHttp Logging Interceptor (from https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor) @@ -226,7 +226,7 @@ The following software have components provided under the terms of this license: - Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf) - Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j) - QpidJMS Client (from ) -- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty) +- Reactor Netty with all modules (from https://github.com/reactor/reactor-netty) - Redisson (from http://redisson.org) - Retrofit (from https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit) - RxJava (from https://github.com/ReactiveX/RxJava) @@ -637,7 +637,6 @@ The following software have components provided under the terms of this license: - Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions) - JUL to SLF4J bridge (from http://www.slf4j.org) - Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) -- Java JWT (from http://www.jwt.io) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -669,6 +668,7 @@ The following software have components provided under the terms of this license: - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) - azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/) - documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/) +- java jwt (from https://github.com/auth0/java-jwt) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - mockito-core (from https://github.com/mockito/mockito) - mockito-core (from https://github.com/mockito/mockito) -- GitLab From 40a5a59f720eabb34d238316fcac03da4b031bef Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Mon, 20 Sep 2021 15:24:43 +0300 Subject: [PATCH 8/9] Updating FOSSA NOTICE --- NOTICE | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/NOTICE b/NOTICE index 10b7821a..4461e780 100644 --- a/NOTICE +++ b/NOTICE @@ -259,7 +259,6 @@ The following software have components provided under the terms of this license: - Spring Messaging (from https://github.com/spring-projects/spring-framework) - Spring Plugin - Metadata Extension (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-metadata) - Spring Plugin Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core) -- Spring Security - Core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core) - Spring TestContext Framework (from https://github.com/spring-projects/spring-framework) - Spring Transaction (from https://github.com/spring-projects/spring-framework) - Spring Web (from https://github.com/spring-projects/spring-framework) @@ -296,7 +295,7 @@ The following software have components provided under the terms of this license: - java-cloudant (from https://cloudant.com) - javatuples (from http://www.javatuples.org) - javax.inject (from http://code.google.com/p/atinject/) -- lettuce (from http://github.com/mp911de/lettuce/wiki) +- lettuce (from https://github.com/lettuce-io/lettuce-core/wiki) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer) - mockito-core (from https://github.com/mockito/mockito) @@ -337,15 +336,16 @@ The following software have components provided under the terms of this license: - spring-boot-test (from https://spring.io/projects/spring-boot) - spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot) - spring-security-config (from https://spring.io/spring-security) +- spring-security-core (from https://spring.io/projects/spring-security) - spring-security-oauth2-client (from https://spring.io/spring-security) - spring-security-oauth2-client (from https://spring.io/spring-security) - spring-security-oauth2-core (from https://spring.io/spring-security) - spring-security-oauth2-jose (from https://spring.io/spring-security) - spring-security-oauth2-jose (from https://spring.io/spring-security) - spring-security-oauth2-resource-server (from https://spring.io/spring-security) -- spring-security-test (from https://spring.io/spring-security) -- spring-security-test (from https://spring.io/spring-security) -- spring-security-web (from https://spring.io/spring-security) +- spring-security-test (from https://spring.io/projects/spring-security) +- spring-security-test (from https://spring.io/projects/spring-security) +- spring-security-web (from https://spring.io/projects/spring-security) - springfox-core (from https://github.com/springfox/springfox) - springfox-schema (from https://github.com/springfox/springfox) - springfox-spi (from https://github.com/springfox/springfox) @@ -664,7 +664,6 @@ The following software have components provided under the terms of this license: - SLF4J API Module (from http://www.slf4j.org) - Spongy Castle (from http://rtyley.github.io/spongycastle/) - Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos) -- Spring Security - Core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core) - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) - azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/) - documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/) @@ -675,6 +674,7 @@ The following software have components provided under the terms of this license: - mockito-junit-jupiter (from https://github.com/mockito/mockito) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java) +- spring-security-core (from https://spring.io/projects/spring-security) ======================================================================== MPL-1.1 -- GitLab From 5c4fa6dc69b5744daf09a8382b081d1279df22f4 Mon Sep 17 00:00:00 2001 From: DGerashchenko Date: Tue, 28 Sep 2021 13:30:12 +0300 Subject: [PATCH 9/9] TableStorage timeouts. --- .../azure/di/TableStorageBootstrapConfig.java | 14 +++++++++++++- .../azure/service/PartitionServiceImpl.java | 4 ++-- .../src/main/resources/application.properties | 5 +++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/TableStorageBootstrapConfig.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/TableStorageBootstrapConfig.java index a0e09618..81c8fc2a 100644 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/TableStorageBootstrapConfig.java +++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/TableStorageBootstrapConfig.java @@ -15,12 +15,14 @@ package org.opengroup.osdu.partition.provider.azure.di; import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.RetryLinearRetry; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.table.CloudTable; import com.microsoft.azure.storage.table.CloudTableClient; import org.apache.http.HttpStatus; import org.opengroup.osdu.common.Validators; import org.opengroup.osdu.core.common.model.http.AppException; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; @@ -34,6 +36,13 @@ public class TableStorageBootstrapConfig { private final static String CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net"; + @Value("${azure.table-storage.retry.delta-backoff}") + private int deltaBackoff; + @Value("${azure.table-storage.retry.max-attempts}") + private int maxAttempts; + @Value("${azure.table-storage.maximum-execution-time}") + private int maximumExecutionTime; + @Bean @Lazy public CloudTableClient getCloudTableClient( @@ -45,7 +54,10 @@ public class TableStorageBootstrapConfig { final String storageConnectionString = String.format(CONNECTION_STRING, storageAccountName, storageAccountKey); CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); - return storageAccount.createCloudTableClient(); + CloudTableClient cloudTableClient = storageAccount.createCloudTableClient(); + cloudTableClient.getDefaultRequestOptions().setRetryPolicyFactory(new RetryLinearRetry(deltaBackoff, maxAttempts)); + cloudTableClient.getDefaultRequestOptions().setMaximumExecutionTimeInMs(maximumExecutionTime); + return cloudTableClient; } catch (URISyntaxException | InvalidKeyException e) { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error creating cloud table storage client", e.getMessage(), e); } diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java index 585e588d..a4b64429 100644 --- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java +++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/service/PartitionServiceImpl.java @@ -162,14 +162,14 @@ public class PartitionServiceImpl implements IPartitionService { } } } catch (InterruptedException ex) { - log.error(String.format("InterruptedException caught when lock the cache key %s: %s", key, ex)); + log.error(String.format("InterruptedException for key: %s.", key), ex); Thread.currentThread().interrupt(); } finally { if (locked) { try { cacheEntryLock.unlock(); } catch (Exception ex) { - log.warning(String.format("unlock exception: %s", ex)); + log.error(String.format("Unlock exception for key: %s.", key), ex); } } } diff --git a/provider/partition-azure/src/main/resources/application.properties b/provider/partition-azure/src/main/resources/application.properties index ff13ec44..afb2ae2e 100644 --- a/provider/partition-azure/src/main/resources/application.properties +++ b/provider/partition-azure/src/main/resources/application.properties @@ -30,6 +30,11 @@ azure.keyvault.url=${KEYVAULT_URI} # Azure App Insights configuration azure.application-insights.instrumentation-key=${appinsights_key} +#Azure TableStorage +azure.table-storage.retry.delta-backoff=2000 +azure.table-storage.retry.max-attempts=3 +azure.table-storage.maximum-execution-time=60000 + # Cache configuration, provider [vm or redis] cache.provider=redis -- GitLab