From e97153e74ed12a176cffb83263387ed15f4fa0e4 Mon Sep 17 00:00:00 2001 From: Jagan Gottimukkula <jgottimukkula@slb.com> Date: Fri, 11 Oct 2019 21:17:08 -0700 Subject: [PATCH] Fixed cache for GCP & Azure --- .../indexer/azure/cache/AttributesCache.java | 55 +++++++++++++++++ .../azure/cache/ElasticCredentialsCache.java | 39 ++++++++++++ .../osdu/indexer/azure/cache/IndexCache.java | 37 ++++++++++++ .../osdu/indexer/azure/cache/JwtCache.java | 39 ++++++++++++ .../osdu/indexer/azure/cache/KindsCache.java | 39 ++++++++++++ .../osdu/indexer/azure/cache/SchemaCache.java | 37 ++++++++++++ .../util/ServiceAccountJwtClientImpl.java | 6 +- .../service/ElasticSettingServiceTest.java | 4 +- .../service/IndexerSchemaServiceTest.java | 4 +- .../osdu/indexer/cache/AttributesCache.java | 60 +++++++++++++++++++ .../cache/ElasticCredentialsCache.java | 45 ++++++++++++++ .../osdu/indexer/cache/IndexCache.java | 43 +++++++++++++ .../osdu/indexer/cache/JwtCache.java | 45 ++++++++++++++ .../osdu/indexer/cache/KindsCache.java | 47 +++++++++++++++ .../osdu/indexer/cache/SchemaCache.java | 43 +++++++++++++ .../osdu/indexer/middleware/IndexFilter.java | 2 +- .../util/ServiceAccountJwtGcpClientImpl.java | 6 +- .../service/ElasticSettingServiceTest.java | 4 +- .../service/IndexerSchemaServiceTest.java | 4 +- .../ServiceAccountJwtGcpClientImplTest.java | 4 +- indexer-service-root/pom.xml | 2 +- .../osdu/indexer/api/RecordIndexerApi.java | 4 +- .../osdu/indexer/cache/SchemaCache.java | 30 ---------- .../provider/interfaces/ISchemaCache.java | 6 ++ .../service/ElasticSettingServiceImpl.java | 6 +- .../service/IndexSchemaServiceImpl.java | 12 ++-- 26 files changed, 563 insertions(+), 60 deletions(-) create mode 100644 indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/AttributesCache.java create mode 100644 indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java create mode 100644 indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexCache.java create mode 100644 indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtCache.java create mode 100644 indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/KindsCache.java create mode 100644 indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaCache.java create mode 100644 indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java create mode 100644 indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java create mode 100644 indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java create mode 100644 indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java create mode 100644 indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java create mode 100644 indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java delete mode 100644 indexer-service-root/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java create mode 100644 indexer-service-root/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/ISchemaCache.java diff --git a/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/AttributesCache.java b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/AttributesCache.java new file mode 100644 index 000000000..e6c987b68 --- /dev/null +++ b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/AttributesCache.java @@ -0,0 +1,55 @@ +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.azure.cache; + +import org.opengroup.osdu.core.cache.VmCache; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IAttributesCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Set; + +@Component +public class AttributesCache implements IAttributesCache<String,Set> { + + private VmCache<String, Set> cache; + + public AttributesCache(@Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION, + @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) { + cache = new VmCache<>(Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, + Integer.parseInt(MAX_CACHE_VALUE_SIZE)); + } + + @Override + public void put(String key, Set value) { + this.cache.put(key, value); + } + + @Override + public Set get(String key) { + return this.cache.get(key); + } + + @Override + public void delete(String key) { + this.cache.delete(key); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } + +} \ No newline at end of file diff --git a/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java new file mode 100644 index 000000000..9e8655545 --- /dev/null +++ b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java @@ -0,0 +1,39 @@ +package org.opengroup.osdu.indexer.azure.cache; + +import org.opengroup.osdu.core.cache.VmCache; +import org.opengroup.osdu.is.core.model.ClusterSettings; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IElasticCredentialsCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings> { + + private VmCache<String, ClusterSettings> cache; + + public ElasticCredentialsCache(@Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION, + @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) { + cache = new VmCache<>(Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60, + Integer.parseInt(MAX_CACHE_VALUE_SIZE)); + } + + @Override + public void put(String s, ClusterSettings o) { + this.cache.put(s,o); + } + + @Override + public ClusterSettings get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexCache.java b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexCache.java new file mode 100644 index 000000000..cefbe0e85 --- /dev/null +++ b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexCache.java @@ -0,0 +1,37 @@ +package org.opengroup.osdu.indexer.azure.cache; + +import org.opengroup.osdu.core.cache.VmCache; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IIndexCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class IndexCache implements IIndexCache<String, Boolean> { + private VmCache<String, Boolean> cache; + + public IndexCache(@Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION, + @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) { + cache = new VmCache<>(Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, + Integer.parseInt(MAX_CACHE_VALUE_SIZE)); + } + + @Override + public void put(String s, Boolean o) { + this.cache.put(s, o); + } + + @Override + public Boolean get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtCache.java b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtCache.java new file mode 100644 index 000000000..2fb2b0bfb --- /dev/null +++ b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtCache.java @@ -0,0 +1,39 @@ +package org.opengroup.osdu.indexer.azure.cache; + +import org.opengroup.osdu.core.cache.VmCache; +import org.opengroup.osdu.is.core.model.IdToken; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IJwtCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class JwtCache implements IJwtCache<String, IdToken> { + private VmCache<String, IdToken> cache; + + // Azure service account id_token can be requested only for 1 hr + private final static int EXPIRED_AFTER = 59; + + public JwtCache(@Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) { + cache = new VmCache<>(EXPIRED_AFTER * 60, Integer.parseInt(MAX_CACHE_VALUE_SIZE)); + } + + @Override + public void put(String s, IdToken o) { + this.cache.put(s, o); + } + + @Override + public IdToken get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/KindsCache.java b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/KindsCache.java new file mode 100644 index 000000000..a8ec2f8e3 --- /dev/null +++ b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/KindsCache.java @@ -0,0 +1,39 @@ +package org.opengroup.osdu.indexer.azure.cache; + +import org.opengroup.osdu.core.cache.VmCache; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IKindsCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Set; + +@Component +public class KindsCache implements IKindsCache<String, Set> { + private VmCache<String, Set> cache; + + public KindsCache(@Value("${KINDS_CACHE_EXPIRATION}") final String KINDS_CACHE_EXPIRATION, + @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) { + cache = new VmCache<>(Integer.parseInt(KINDS_CACHE_EXPIRATION) * 60, + Integer.parseInt(MAX_CACHE_VALUE_SIZE)); + } + + @Override + public void put(String s, Set o) { + this.cache.put(s, o); + } + + @Override + public Set get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaCache.java b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaCache.java new file mode 100644 index 000000000..f662ccec3 --- /dev/null +++ b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaCache.java @@ -0,0 +1,37 @@ +package org.opengroup.osdu.indexer.azure.cache; + +import org.opengroup.osdu.core.cache.VmCache; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class SchemaCache implements ISchemaCache<String, String> { + private VmCache<String, String> cache; + + public SchemaCache(@Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION, + @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) { + cache = new VmCache<>(Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, + Integer.parseInt(MAX_CACHE_VALUE_SIZE)); + } + + @Override + public void put(String s, String o) { + this.cache.put(s, o); + } + + @Override + public String get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java index 46f015ac2..f2ca8ff9d 100644 --- a/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java +++ b/indexer-service-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java @@ -25,9 +25,9 @@ import org.opengroup.osdu.core.multitenancy.ITenantFactory; import org.opengroup.osdu.core.multitenancy.TenantInfo; import org.opengroup.osdu.indexer.azure.model.AADConfiguration; -import org.opengroup.osdu.is.core.cache.JwtCache; import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; import org.opengroup.osdu.is.core.model.IdToken; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IJwtCache; import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo; import org.opengroup.osdu.is.core.provider.interfaces.util.IServiceAccountJwtClient; import org.opengroup.osdu.is.core.util.AppException; @@ -52,7 +52,7 @@ public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient { @Autowired private DpsHeaders dpsHeaders; @Autowired - private JwtCache cacheService; + private IJwtCache cacheService; @Autowired private JaxRsDpsLog log; @@ -69,7 +69,7 @@ public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient { String ACCESS_TOKEN = ""; try { - IdToken cachedToken = this.cacheService.get(tenant.getServiceAccount()); + IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); this.headersInfoAzure.getHeaders().put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); diff --git a/indexer-service-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java b/indexer-service-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java index bfd5649fe..ce69140ee 100644 --- a/indexer-service-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java +++ b/indexer-service-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java @@ -21,9 +21,9 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.opengroup.osdu.core.multitenancy.TenantInfo; import org.opengroup.osdu.indexer.service.ElasticSettingServiceImpl; -import org.opengroup.osdu.is.core.cache.ElasticCredentialsCache; import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; import org.opengroup.osdu.is.core.model.ClusterSettings; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IElasticCredentialsCache; import org.opengroup.osdu.is.core.provider.interfaces.persistence.ElasticRepository; import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo; import org.opengroup.osdu.is.core.service.TenantInfoService; @@ -42,7 +42,7 @@ public class ElasticSettingServiceTest { @Mock private ElasticRepository elasticRepository; @Mock - private ElasticCredentialsCache elasticCredentialCache; + private IElasticCredentialsCache elasticCredentialCache; @Mock private TenantInfo tenantInfo; @InjectMocks diff --git a/indexer-service-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerSchemaServiceTest.java b/indexer-service-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerSchemaServiceTest.java index d22277933..d006594c6 100644 --- a/indexer-service-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerSchemaServiceTest.java +++ b/indexer-service-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerSchemaServiceTest.java @@ -23,9 +23,9 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.indexer.cache.SchemaCache; import org.opengroup.osdu.indexer.model.IndexSchema; import org.opengroup.osdu.indexer.model.OperationType; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.opengroup.osdu.indexer.service.IndexSchemaServiceImpl; import org.opengroup.osdu.indexer.service.IndexerMappingService; import org.opengroup.osdu.indexer.service.StorageService; @@ -72,7 +72,7 @@ public class IndexerSchemaServiceTest { @Mock private IndicesService indicesService; @Mock - private SchemaCache schemaCache; + private ISchemaCache schemaCache; @InjectMocks private IndexSchemaServiceImpl sut; diff --git a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java new file mode 100644 index 000000000..bebf28bdf --- /dev/null +++ b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java @@ -0,0 +1,60 @@ +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.cache.RedisCache; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IAttributesCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import java.util.Set; + +@Component +public class AttributesCache implements IAttributesCache<String,Set>, AutoCloseable { + + private RedisCache<String, Set> cache; + + public AttributesCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) { + + cache = new RedisCache(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); + } + + @Override + public void put(String key, Set value) { + this.cache.put(key, value); + } + + @Override + public Set get(String key) { + return this.cache.get(key); + } + + @Override + public void delete(String key) { + this.cache.delete(key); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } + + @Override + public void close() { + this.cache.close(); + } +} \ No newline at end of file diff --git a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java new file mode 100644 index 000000000..9f517b8ee --- /dev/null +++ b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java @@ -0,0 +1,45 @@ +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.cache.RedisCache; +import org.opengroup.osdu.is.core.model.ClusterSettings; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IElasticCredentialsCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable { + + private RedisCache<String, ClusterSettings> cache; + + public ElasticCredentialsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60, String.class, ClusterSettings.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, ClusterSettings o) { + this.cache.put(s,o); + } + + @Override + public ClusterSettings get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java new file mode 100644 index 000000000..dbe09db28 --- /dev/null +++ b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java @@ -0,0 +1,43 @@ +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.cache.RedisCache; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IIndexCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class IndexCache implements IIndexCache<String, Boolean>, AutoCloseable { + private RedisCache<String, Boolean> cache; + + public IndexCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, Boolean o) { + this.cache.put(s, o); + } + + @Override + public Boolean get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java new file mode 100644 index 000000000..29b3b5e49 --- /dev/null +++ b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java @@ -0,0 +1,45 @@ +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.cache.RedisCache; +import org.opengroup.osdu.is.core.model.IdToken; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IJwtCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class JwtCache implements IJwtCache<String, IdToken>, AutoCloseable { + RedisCache<String, IdToken> cache; + + // google service account id_token can be requested only for 1 hr + private final static int EXPIRED_AFTER = 59; + + public JwtCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + EXPIRED_AFTER * 60, String.class, IdToken.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, IdToken o) { + this.cache.put(s, o); + } + + @Override + public IdToken get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java new file mode 100644 index 000000000..eacd8a8a7 --- /dev/null +++ b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java @@ -0,0 +1,47 @@ +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.cache.RedisCache; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IKindsCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Set; + +@Component +public class KindsCache implements IKindsCache<String, Set>, AutoCloseable { + private RedisCache<String, Set> cache; + + public KindsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${KINDS_CACHE_EXPIRATION}") final String KINDS_CACHE_EXPIRATION, + @Value("${KINDS_REDIS_DATABASE}") final String KINDS_REDIS_DATABASE) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(KINDS_CACHE_EXPIRATION) * 60, + Integer.parseInt(KINDS_REDIS_DATABASE), String.class, Set.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, Set o) { + this.cache.put(s, o); + } + + @Override + public Set get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java new file mode 100644 index 000000000..396f70ae3 --- /dev/null +++ b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java @@ -0,0 +1,43 @@ +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.cache.RedisCache; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class SchemaCache implements ISchemaCache<String, String>, AutoCloseable { + private RedisCache<String, String> cache; + + public SchemaCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, String o) { + this.cache.put(s, o); + } + + @Override + public String get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java index 1c4ea79c4..0498b70ce 100644 --- a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java +++ b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java @@ -46,7 +46,7 @@ public class IndexFilter implements Filter { String uri = httpRequest.getRequestURI().toLowerCase(); if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri.contains(PATH_TASK_HANDLERS)) { -// checkWorkerApiAccess(requestInfo); + checkWorkerApiAccess(requestInfo); } if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri.contains(PATH_CRON_HANDLERS)) { diff --git a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java index c78aac046..8dcf8d1ba 100644 --- a/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java +++ b/indexer-service-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java @@ -41,9 +41,9 @@ import org.apache.http.util.EntityUtils; import org.opengroup.osdu.core.api.DpsHeaders; import org.opengroup.osdu.core.multitenancy.ITenantFactory; import org.opengroup.osdu.core.multitenancy.TenantInfo; -import org.opengroup.osdu.is.core.cache.JwtCache; import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; import org.opengroup.osdu.is.core.model.IdToken; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IJwtCache; import org.opengroup.osdu.is.core.provider.interfaces.util.IServiceAccountJwtClient; import org.opengroup.osdu.is.core.util.AppException; import org.springframework.beans.factory.annotation.Autowired; @@ -72,7 +72,7 @@ public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient @Autowired private HeadersInfoGcpImpl headersInfoGcp; @Autowired - private JwtCache cacheService; + private IJwtCache cacheService; @Autowired private JaxRsDpsLog log; @Autowired @@ -93,7 +93,7 @@ public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient } try { - IdToken cachedToken = this.cacheService.get(tenant.getServiceAccount()); + IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); this.headersInfoGcp.getHeaders().put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); // Add the user to DpsHeaders directly this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); diff --git a/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java index 6e669933e..363ef0f24 100644 --- a/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java +++ b/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java @@ -21,9 +21,9 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.opengroup.osdu.core.multitenancy.TenantInfo; -import org.opengroup.osdu.is.core.cache.ElasticCredentialsCache; import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; import org.opengroup.osdu.is.core.model.ClusterSettings; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IElasticCredentialsCache; import org.opengroup.osdu.is.core.provider.interfaces.persistence.ElasticRepository; import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo; import org.opengroup.osdu.is.core.service.TenantInfoService; @@ -43,7 +43,7 @@ public class ElasticSettingServiceTest { @Mock private ElasticRepository elasticRepository; @Mock - private ElasticCredentialsCache elasticCredentialCache; + private IElasticCredentialsCache elasticCredentialCache; @Mock private TenantInfo tenantInfo; @InjectMocks diff --git a/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java index b5b190e34..d58a9663d 100644 --- a/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java +++ b/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java @@ -22,9 +22,9 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.indexer.cache.SchemaCache; import org.opengroup.osdu.indexer.model.IndexSchema; import org.opengroup.osdu.indexer.model.OperationType; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.opengroup.osdu.is.core.httpclient.RequestStatus; import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; import org.opengroup.osdu.is.core.service.IndicesService; @@ -66,7 +66,7 @@ public class IndexerSchemaServiceTest { @Mock private IndicesService indicesService; @Mock - private SchemaCache schemaCache; + private ISchemaCache schemaCache; @InjectMocks private IndexSchemaServiceImpl sut; diff --git a/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java b/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java index c28738344..fe8e4a6cf 100644 --- a/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java +++ b/indexer-service-gcp/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java @@ -35,10 +35,10 @@ import org.mockito.Spy; import org.opengroup.osdu.indexer.service.TenantInfoServiceImpl; import org.opengroup.osdu.core.api.DpsHeaders; import org.opengroup.osdu.core.multitenancy.TenantInfo; -import org.opengroup.osdu.is.core.cache.JwtCache; import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; import org.opengroup.osdu.is.core.model.DeploymentEnvironment; import org.opengroup.osdu.is.core.model.IdToken; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IJwtCache; import org.opengroup.osdu.is.core.util.AppException; import org.opengroup.osdu.is.core.util.Config; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -84,7 +84,7 @@ public class ServiceAccountJwtGcpClientImplTest { @Mock private TenantInfoServiceImpl tenantInfoService; @Mock - private JwtCache cacheService; + private IJwtCache cacheService; @Mock private HeadersInfoGcpImpl headersInfoGcp; @InjectMocks @Spy diff --git a/indexer-service-root/pom.xml b/indexer-service-root/pom.xml index 6fd66de46..ef524baa0 100644 --- a/indexer-service-root/pom.xml +++ b/indexer-service-root/pom.xml @@ -68,7 +68,7 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>indexer-search-core-lib</artifactId> - <version>1.0.4</version> + <version>1.0.7</version> </dependency> <dependency> diff --git a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java index 473275737..505bd5504 100644 --- a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java +++ b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java @@ -71,9 +71,7 @@ public class RecordIndexerApi { Type listType = new TypeToken<List<RecordInfo>>() { }.getType(); List<RecordInfo> recordInfos = new Gson().fromJson(recordChangedMessages.getData(), listType); - if (recordInfos == null) { - log.info("RECORD INFO IS NULL"); - } + if (recordInfos.size() == 0) { log.info("none of record-change message can be deserialized"); return new ResponseEntity(HttpStatus.OK); diff --git a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java deleted file mode 100644 index 302d6cf58..000000000 --- a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.cache; - -import org.opengroup.osdu.core.cache.RedisCache; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Component -public class SchemaCache extends RedisCache<String, String> { - - public SchemaCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION) { - super(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class); - } -} \ No newline at end of file diff --git a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/ISchemaCache.java b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/ISchemaCache.java new file mode 100644 index 000000000..c8b499feb --- /dev/null +++ b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/ISchemaCache.java @@ -0,0 +1,6 @@ +package org.opengroup.osdu.indexer.provider.interfaces; + +import org.opengroup.osdu.core.cache.ICache; + +public interface ISchemaCache <String,V> extends ICache<String, V> { +} diff --git a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java index 89947eb0b..b07fd8209 100644 --- a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java +++ b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java @@ -16,9 +16,9 @@ package org.opengroup.osdu.indexer.service; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.multitenancy.TenantInfo; -import org.opengroup.osdu.is.core.cache.ElasticCredentialsCache; import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; import org.opengroup.osdu.is.core.model.ClusterSettings; +import org.opengroup.osdu.is.core.provider.interfaces.cache.IElasticCredentialsCache; import org.opengroup.osdu.is.core.provider.interfaces.persistence.ElasticRepository; import org.opengroup.osdu.is.core.service.ElasticSettingService; import org.opengroup.osdu.is.core.service.TenantInfoService; @@ -35,7 +35,7 @@ public class ElasticSettingServiceImpl implements ElasticSettingService { @Autowired private ElasticRepository elasticRepository; @Autowired - private ElasticCredentialsCache elasticCredentialCache; + private IElasticCredentialsCache elasticCredentialCache; @Autowired private JaxRsDpsLog log; @@ -48,7 +48,7 @@ public class ElasticSettingServiceImpl implements ElasticSettingService { TenantInfo tenantInfo = this.tenantInfoServiceProvider.getTenantInfo(); String cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName()); - ClusterSettings clusterInfo = this.elasticCredentialCache.get(cacheKey); + ClusterSettings clusterInfo = (ClusterSettings) this.elasticCredentialCache.get(cacheKey); if (clusterInfo != null) { return clusterInfo; } diff --git a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java index 206ceb763..6225cb714 100644 --- a/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java +++ b/indexer-service-root/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java @@ -19,10 +19,10 @@ import com.google.gson.Gson; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.client.RestHighLevelClient; import org.opengroup.osdu.indexer.model.IndexSchema; -import org.opengroup.osdu.indexer.cache.SchemaCache; import org.opengroup.osdu.indexer.model.OperationType; import org.opengroup.osdu.indexer.model.Schema; import org.opengroup.osdu.indexer.model.StorageType; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.opengroup.osdu.indexer.util.TypeMapper; import org.opengroup.osdu.is.core.httpclient.RequestStatus; import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; @@ -60,7 +60,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { @Autowired private IndicesService indicesService; @Autowired - private SchemaCache schemaCache; + private ISchemaCache schemaCache; public void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException { try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { @@ -120,7 +120,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { public IndexSchema getIndexerInputSchema(String kind) throws AppException { try { - String schema = this.schemaCache.get(kind); + String schema = (String) this.schemaCache.get(kind); if (Strings.isNullOrEmpty(schema)) { // get from storage schema = this.storageService.getStorageSchema(kind); @@ -139,7 +139,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { } } else { // search flattened schema in memcache - String flattenedSchema = this.schemaCache.get(kind + FLATTENED_SCHEMA); + String flattenedSchema = (String) this.schemaCache.get(kind + FLATTENED_SCHEMA); if (Strings.isNullOrEmpty(flattenedSchema)) { Schema basicSchema = Schema.builder().kind(kind).build(); return normalizeSchema(gson.toJson(basicSchema)); @@ -154,10 +154,10 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { } private void invalidateCache(String kind) { - String schema = this.schemaCache.get(kind); + String schema = (String) this.schemaCache.get(kind); if (!Strings.isNullOrEmpty(schema)) this.schemaCache.delete(kind); - String flattenSchema = this.schemaCache.get(kind + FLATTENED_SCHEMA); + String flattenSchema = (String) this.schemaCache.get(kind + FLATTENED_SCHEMA); if (!Strings.isNullOrEmpty(flattenSchema)) this.schemaCache.delete(kind + FLATTENED_SCHEMA); } -- GitLab