From d3db85cebf888d171c07a2b97e10c2a3d990a701 Mon Sep 17 00:00:00 2001
From: NThakur4 <nthakur4@slb.com>
Date: Mon, 13 Sep 2021 18:54:46 -0500
Subject: [PATCH] redis cache for azure
---
provider/indexer-aws/pom.xml | 2 +-
provider/indexer-azure/README.md | 1 +
.../indexer/azure/cache/IndexRedisCache.java | 37 +++++++++
.../{IndexCache.java => IndexVmCache.java} | 8 +-
.../indexer/azure/cache/JwtRedisCache.java | 37 +++++++++
.../cache/{JwtCache.java => JwtVmCache.java} | 6 +-
.../indexer/azure/cache/SchemaRedisCache.java | 36 +++++++++
.../{SchemaCache.java => SchemaVmCache.java} | 8 +-
.../osdu/indexer/azure/di/RedisConfig.java | 76 +++++++++++++++++++
.../src/main/resources/application.properties | 7 +-
provider/indexer-reference/pom.xml | 2 +-
testing/indexer-test-azure/pom.xml | 2 +-
testing/indexer-test-core/pom.xml | 2 +-
testing/indexer-test-ibm/pom.xml | 2 +-
.../step_definitions/index/record/Steps.java | 2 +-
15 files changed, 213 insertions(+), 15 deletions(-)
create mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexRedisCache.java
rename provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/{IndexCache.java => IndexVmCache.java} (77%)
create mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtRedisCache.java
rename provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/{JwtCache.java => JwtVmCache.java} (83%)
create mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaRedisCache.java
rename provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/{SchemaCache.java => SchemaVmCache.java} (77%)
create mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/RedisConfig.java
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index 060335dfa..f96f8a3d1 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -47,7 +47,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.11.0</version>
</dependency>
<!-- AWS managed packages -->
diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md
index 8f12c5810..bc6a63ec3 100644
--- a/provider/indexer-azure/README.md
+++ b/provider/indexer-azure/README.md
@@ -39,6 +39,7 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
+| `runtime.env.local` | false (change this to `true` when running locally) | Var to check if app is running locally | no | - |
| `server.servlet.contextPath` | `/api/indexer/v2/` | Servlet context path | no | - |
| `schema_service_url` | ex `https://schema.azurewebsites.net` | Endpoint of schema service | no | output of infrastructure deployments |
| `SCHEMA_HOST` | `${schema_service_url}/schema` | Endpoint of schema API | no | - |
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexRedisCache.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexRedisCache.java
new file mode 100644
index 000000000..8ac5b0f10
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexRedisCache.java
@@ -0,0 +1,37 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+package org.opengroup.osdu.indexer.azure.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Named;
+
+@Component
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "false", matchIfMissing = true)
+public class IndexRedisCache extends RedisCache<String, Boolean> implements IIndexCache<String, Boolean> {
+
+ public IndexRedisCache(final @Named("REDIS_HOST") String host,
+ final @Named("REDIS_PORT") int port,
+ final @Named("REDIS_PASSWORD") String password,
+ final @Named("INDEX_REDIS_TTL") int timeout,
+ @Value("${redis.database}") final int database) {
+ super(host, port, password, timeout, database, String.class, Boolean.class);
+ }
+}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexCache.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexVmCache.java
similarity index 77%
rename from provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexCache.java
rename to provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexVmCache.java
index 8ca5500c2..d242c05b0 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexCache.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/IndexVmCache.java
@@ -17,14 +17,16 @@ package org.opengroup.osdu.indexer.azure.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Component
-public class IndexCache implements IIndexCache<String, Boolean> {
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "true")
+public class IndexVmCache implements IIndexCache<String, Boolean> {
private VmCache<String, Boolean> cache;
- public IndexCache(@Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION,
- @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ public IndexVmCache(@Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
cache = new VmCache<>(Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60,
Integer.parseInt(MAX_CACHE_VALUE_SIZE));
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtRedisCache.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtRedisCache.java
new file mode 100644
index 000000000..1c1a3571f
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtRedisCache.java
@@ -0,0 +1,37 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.IdToken;
+import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Named;
+
+@Component
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "false", matchIfMissing = true)
+public class JwtRedisCache extends RedisCache<String, IdToken> implements IJwtCache<String, IdToken> {
+
+ public JwtRedisCache(final @Named("REDIS_HOST") String host,
+ final @Named("REDIS_PORT") int port,
+ final @Named("REDIS_PASSWORD") String password,
+ final @Named("JWT_REDIS_TTL") int timeout,
+ @Value("${redis.database}") final int database) {
+ super(host, port, password, timeout, database, String.class, IdToken.class);
+ }
+}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtCache.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtVmCache.java
similarity index 83%
rename from provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtCache.java
rename to provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtVmCache.java
index 07e4f0403..429100969 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtCache.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/JwtVmCache.java
@@ -18,16 +18,18 @@ import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.common.model.search.IdToken;
import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Component
-public class JwtCache implements IJwtCache<String, IdToken> {
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "true")
+public class JwtVmCache implements IJwtCache<String, IdToken> {
private VmCache<String, IdToken> cache;
// Azure service account id_token can be requested only for 1 hr
private final static int EXPIRED_AFTER = 59;
- public JwtCache(@Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ public JwtVmCache(@Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
cache = new VmCache<>(EXPIRED_AFTER * 60, Integer.parseInt(MAX_CACHE_VALUE_SIZE));
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaRedisCache.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaRedisCache.java
new file mode 100644
index 000000000..b5f78c4dd
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaRedisCache.java
@@ -0,0 +1,36 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Named;
+
+@Component
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "false", matchIfMissing = true)
+public class SchemaRedisCache extends RedisCache<String, String> implements ISchemaCache<String, String> {
+
+ public SchemaRedisCache(final @Named("REDIS_HOST") String host,
+ final @Named("REDIS_PORT") int port,
+ final @Named("REDIS_PASSWORD") String password,
+ final @Named("SCHEMA_REDIS_TTL") int timeout,
+ @Value("${redis.database}") final int database) {
+ super(host, port, password, timeout, database, String.class, String.class);
+ }
+}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaCache.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaVmCache.java
similarity index 77%
rename from provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaCache.java
rename to provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaVmCache.java
index 97a5f1810..ba19d519d 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaCache.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/SchemaVmCache.java
@@ -17,14 +17,16 @@ package org.opengroup.osdu.indexer.azure.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Component
-public class SchemaCache implements ISchemaCache<String, String> {
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "true")
+public class SchemaVmCache implements ISchemaCache<String, String> {
private VmCache<String, String> cache;
- public SchemaCache(@Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION,
- @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ public SchemaVmCache(@Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
cache = new VmCache<>(Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60,
Integer.parseInt(MAX_CACHE_VALUE_SIZE));
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/RedisConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/RedisConfig.java
new file mode 100644
index 000000000..6b98eb7ce
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/RedisConfig.java
@@ -0,0 +1,76 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.di;
+
+import com.azure.security.keyvault.secrets.SecretClient;
+import org.opengroup.osdu.azure.KeyVaultFacade;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import javax.inject.Named;
+
+@Configuration
+public class RedisConfig {
+
+ @Value("${redis.port:6380}")
+ private int port;
+
+ @Value("${redis.index.ttl:3600}")
+ public int indexRedisTtl;
+
+ // Azure service account id_token can be requested only for 1 hr
+ @Value("${redis.jwt.ttl:3540}")
+ public int jwtTtl;
+
+ @Value("${redis.schema.ttl:3600}")
+ public int schemaTtl;
+
+ @Bean
+ @Named("REDIS_PORT")
+ public int getRedisPort() {
+ return port;
+ }
+
+ @Bean
+ @Named("INDEX_REDIS_TTL")
+ public int getIndexRedisTtl() {
+ return indexRedisTtl;
+ }
+
+ @Bean
+ @Named("JWT_REDIS_TTL")
+ public int getJwtRedisTtl() {
+ return jwtTtl;
+ }
+
+ @Bean
+ @Named("SCHEMA_REDIS_TTL")
+ public int getSchemaRedisTtl() {
+ return schemaTtl;
+ }
+
+ @Bean
+ @Named("REDIS_HOST")
+ public String redisHost(SecretClient kv) {
+ return KeyVaultFacade.getSecretWithValidation(kv, "redis-hostname");
+ }
+
+ @Bean
+ @Named("REDIS_PASSWORD")
+ public String redisPassword(SecretClient kv) {
+ return KeyVaultFacade.getSecretWithValidation(kv, "redis-password");
+ }
+}
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index 59231e1ae..0d27fa488 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+runtime.env.local=false
+
LOG_PREFIX=indexer
LOG_LEVEL=DEBUG
@@ -85,4 +87,7 @@ logging.ignore.servlet.paths=/swagger-ui.html
tenantFactoryImpl.required=true
# Disable keyVault for actuator health check
-management.health.azure-key-vault.enabled=false
\ No newline at end of file
+management.health.azure-key-vault.enabled=false
+
+#Redis
+redis.database=${REDIS_DATABASE}
\ No newline at end of file
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
index 466615c5d..26d200f11 100644
--- a/provider/indexer-reference/pom.xml
+++ b/provider/indexer-reference/pom.xml
@@ -48,7 +48,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.12.0-rc1</version>
</dependency>
<dependency>
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index ffa914cff..e7f45c911 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -36,7 +36,7 @@
<cucumber.version>1.2.5</cucumber.version>
<junit.jupiter.version>5.6.0</junit.jupiter.version>
<elasticsearch.version>7.8.1</elasticsearch.version>
- <os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
+ <os-core-common.version>0.12.0-rc1</os-core-common.version>
</properties>
<dependencies>
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index bbe26ce1b..808fab5f5 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -17,7 +17,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
- <os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
+ <os-core-common.version>0.12.0-rc1</os-core-common.version>
</properties>
<dependencies>
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index 36aabbea4..b0ad772a3 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -44,7 +44,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.3.8</version>
+ <version>0.12.0-rc1</version>
</dependency>
<dependency>
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 3ec2f0033..4aa2b505c 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -138,7 +138,7 @@ public class Steps extends SchemaServiceRecordSteps {
HttpRequest rq = HttpRequest.post(recordChangeMessage).url(url).headers(dpsHeaders.getHeaders()).build();
HttpResponse result = httpClient.send(rq);
- if(result.hasException().equals(false) && result.getResponseCode() == 500) {
+ if(result.hasException() == false && result.getResponseCode() == 500) {
assertTrue(result.getResponseCode() == 500);
} else {
AppError error = gson.fromJson(result.getBody(), AppError.class);
--
GitLab