diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java index a87c9d4577a573359f5d34682be962078b11bc67..d2c3e68f06d2da43a8988aa470b58d2002f36888 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,14 +27,15 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.FilterType; -@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) +@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class, + ManagementWebSecurityAutoConfiguration.class}) @Configuration @ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)}) + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)}) public class IndexerAnthosApplication { - public static void main(String[] args) { - SpringApplication.run(IndexerAnthosApplication.class, args); - } + public static void main(String[] args) { + SpringApplication.run(IndexerAnthosApplication.class, args); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java index ccc931148ed7a9826fb8cc7854c85be5f898d796..ec6996fe06ba02a1675486c9c22d341389736d32 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,9 +21,10 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; public class ServletInitializer extends SpringBootServletInitializer { - @Override - protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { - return application.sources(IndexerAnthosApplication.class); - } + + @Override + protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { + return application.sources(IndexerAnthosApplication.class); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java index bdfa51cd624d4afedfa08547fdda56a06c1f7051..b60b4d14405fbadaaa0e25977b81e08fb66fe905 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,47 +17,49 @@ package org.opengroup.osdu.indexer.cache; +import java.util.Set; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.Set; @Component -public class AttributesCache implements IAttributesCache<String,Set>, AutoCloseable { - - private RedisCache<String, Set> cache; - - public AttributesCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) { - - cache = new RedisCache(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); - } - - @Override - public void put(String key, Set value) { - this.cache.put(key, value); - } - - @Override - public Set get(String key) { - return this.cache.get(key); - } - - @Override - public void delete(String key) { - this.cache.delete(key); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } - - @Override - public void close() { - this.cache.close(); - } +public class AttributesCache implements IAttributesCache<String, Set>, AutoCloseable { + + private RedisCache<String, Set> cache; + + @Autowired + public AttributesCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getIndexCacheExpiration() * 60, + String.class, + Boolean.class); + } + + @Override + public void put(String key, Set value) { + this.cache.put(key, value); + } + + @Override + public Set get(String key) { + return this.cache.get(key); + } + + @Override + public void delete(String key) { + this.cache.delete(key); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } + + @Override + public void close() { + this.cache.close(); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java index d17e26f7b832658226f9cd32b11426b4e54fe1fc..6e832b44671a8ac95b65556076bf15122c6990a3 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,14 +19,19 @@ package org.opengroup.osdu.indexer.cache; import com.google.auth.oauth2.AccessToken; import org.opengroup.osdu.core.common.cache.RedisCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class DatastoreCredentialCache extends RedisCache<String, AccessToken> { - // Datastore credentials are only valid for 1hr, release the key 2 minutes before the expiration - public DatastoreCredentialCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) { - super(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), 58 * 60, String.class, AccessToken.class); - } + @Autowired + public DatastoreCredentialCache(IndexerConfigurationProperties indexerConfigurationProperties) { + super(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + 58 * 60, + String.class, + AccessToken.class); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java index 7a5d5f6f32d8a76eb6172b30047d893a718277d4..24804bc9000a8f10298b5e30a033ed99886c90de 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,43 +20,47 @@ package org.opengroup.osdu.indexer.cache; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component -public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable { - - private RedisCache<String, ClusterSettings> cache; - - public ElasticCredentialsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60, String.class, ClusterSettings.class); - } - - @Override - public void close() throws Exception { - this.cache.close(); - } - - @Override - public void put(String s, ClusterSettings o) { - this.cache.put(s,o); - } - - @Override - public ClusterSettings get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } +public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, + AutoCloseable { + + private RedisCache<String, ClusterSettings> cache; + + @Autowired + public ElasticCredentialsCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getIndexCacheExpiration() * 60, + String.class, + ClusterSettings.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, ClusterSettings o) { + this.cache.put(s, o); + } + + @Override + public ClusterSettings get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java index d9b7197fb040234eac6627e926e6e2b88a69fcc3..de447717623d63ded8d1d1c46efc42817d618183 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,42 +19,46 @@ package org.opengroup.osdu.indexer.cache; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class IndexCache implements IIndexCache<String, Boolean>, AutoCloseable { - private RedisCache<String, Boolean> cache; - - public IndexCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); - } - - @Override - public void close() throws Exception { - this.cache.close(); - } - - @Override - public void put(String s, Boolean o) { - this.cache.put(s, o); - } - - @Override - public Boolean get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } + + private RedisCache<String, Boolean> cache; + + @Autowired + public IndexCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getIndexCacheExpiration() * 60, + String.class, + Boolean.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, Boolean o) { + this.cache.put(s, o); + } + + @Override + public Boolean get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java index 2b442b622e4e3dca4266d93dbe5ea7dea8def433..9a2a02d753dfbbe506a10bab3ecb2d6083876bef 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,43 +20,47 @@ package org.opengroup.osdu.indexer.cache; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.model.search.IdToken; import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.stereotype.Component; @Component public class JwtCache implements IJwtCache<String, IdToken>, AutoCloseable { - RedisCache<String, IdToken> cache; - - // google service account id_token can be requested only for 1 hr - private final static int EXPIRED_AFTER = 59; - - public JwtCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - EXPIRED_AFTER * 60, String.class, IdToken.class); - } - - @Override - public void close() throws Exception { - this.cache.close(); - } - - @Override - public void put(String s, IdToken o) { - this.cache.put(s, o); - } - - @Override - public IdToken get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } + + RedisCache<String, IdToken> cache; + + // google service account id_token can be requested only for 1 hr + private final static int EXPIRED_AFTER = 59; + + public JwtCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + EXPIRED_AFTER * 60, + String.class, + IdToken.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, IdToken o) { + this.cache.put(s, o); + } + + @Override + public IdToken get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java index d70210a0c570e66dfa97c0e5ac3629f02c8c488c..afc2a8c39299158c3a50e1e9f5d1f5b67e5a6095 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,48 +17,50 @@ package org.opengroup.osdu.indexer.cache; +import java.util.Set; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.provider.interfaces.IKindsCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.Set; - @Component public class KindsCache implements IKindsCache<String, Set>, AutoCloseable { - private RedisCache<String, Set> cache; - public KindsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${KINDS_CACHE_EXPIRATION}") final String KINDS_CACHE_EXPIRATION, - @Value("${KINDS_REDIS_DATABASE}") final String KINDS_REDIS_DATABASE) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(KINDS_CACHE_EXPIRATION) * 60, - Integer.parseInt(KINDS_REDIS_DATABASE), String.class, Set.class); - } + private RedisCache<String, Set> cache; + + @Autowired + public KindsCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getKindsCacheExpiration() * 60, + indexerConfigurationProperties.getKindsRedisDatabase(), + String.class, + Set.class); + } - @Override - public void close() throws Exception { - this.cache.close(); - } + @Override + public void close() throws Exception { + this.cache.close(); + } - @Override - public void put(String s, Set o) { - this.cache.put(s, o); - } + @Override + public void put(String s, Set o) { + this.cache.put(s, o); + } - @Override - public Set get(String s) { - return this.cache.get(s); - } + @Override + public Set get(String s) { + return this.cache.get(s); + } - @Override - public void delete(String s) { - this.cache.delete(s); - } + @Override + public void delete(String s) { + this.cache.delete(s); + } - @Override - public void clearAll() { - this.cache.clearAll(); - } + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java index 1fc8b411471a71cd670e9f22d3981c2855a85dcc..3d84e22959501decedf8dd08564bc0869bee8d50 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,43 +18,47 @@ package org.opengroup.osdu.indexer.cache; import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class SchemaCache implements ISchemaCache<String, String>, AutoCloseable { - private RedisCache<String, String> cache; - - public SchemaCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class); - } - - @Override - public void close() throws Exception { - this.cache.close(); - } - - @Override - public void put(String s, String o) { - this.cache.put(s, o); - } - - @Override - public String get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } + + private RedisCache<String, String> cache; + + @Autowired + public SchemaCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getSchemaCacheExpiration() * 60, + String.class, + String.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, String o) { + this.cache.put(s, o); + } + + @Override + public String get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..1ba4425a4f2f09ff361c00cbcf4ec554811daf34 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class EntitlementsConfigProperties { + + private String AuthorizeApi; +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..080f76dfc25465b8539e8c0789e81941bd026e07 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class MongoDBConfigProperties { + + private String mongoDbUrl; + private String mongoDbUser; + private String mongoDbPassword; + private String mongoDbName; +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..55efbf1a75763e976d71a840335e60b14c8a5ad4 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class RabbitMqConfigProperties { + + private String mbRabbitMqUri; +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java index 8460aa5f0da3f175cffb3d54e1b0bdf9cdc2b33d..45131176cd1ff35eb2dc411b0073b245c0d3b8e8 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,8 @@ package org.opengroup.osdu.indexer.di; import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig; import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory; import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.EntitlementsConfigProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.AbstractFactoryBean; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @@ -31,24 +32,24 @@ import org.springframework.web.context.annotation.RequestScope; @Lazy public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> { - @Value("${AUTHORIZE_API}") - private String AUTHORIZE_API; - - @Value("${AUTHORIZE_API_KEY:}") - private String AUTHORIZE_API_KEY; - - @Override - protected IEntitlementsFactory createInstance() throws Exception { - - return new EntitlementsFactory(EntitlementsAPIConfig - .builder() - .rootUrl(AUTHORIZE_API) - .apiKey(AUTHORIZE_API_KEY) - .build()); - } - - @Override - public Class<?> getObjectType() { - return IEntitlementsFactory.class; - } + private EntitlementsConfigProperties entitlementsConfigProperties; + + @Override + protected IEntitlementsFactory createInstance() throws Exception { + return new EntitlementsFactory(EntitlementsAPIConfig + .builder() + .rootUrl(entitlementsConfigProperties.getAuthorizeApi()) + .build()); + } + + @Override + public Class<?> getObjectType() { + return IEntitlementsFactory.class; + } + + @Autowired + public void setEntitlementsConfigProperties( + EntitlementsConfigProperties entitlementsConfigProperties) { + this.entitlementsConfigProperties = entitlementsConfigProperties; + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java index 20236aaf68467a6441e23ba129331eb3f8907550..3103acc8e88eca93f51397fc921499de1fc11c35 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java @@ -27,56 +27,61 @@ import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.concurrent.TimeoutException; import javax.annotation.PostConstruct; +import org.opengroup.osdu.indexer.config.RabbitMqConfigProperties; import org.opengroup.osdu.indexer.messagebus.IMessageFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class RabbitMQFactoryImpl implements IMessageFactory { - private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class); + private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class); + private final RabbitMqConfigProperties rabbitMqConfigProperties; - @Value("${mb.rabbitmq.uri}") - private String uri; + private Channel channel; - private Channel channel; + @Autowired + public RabbitMQFactoryImpl(RabbitMqConfigProperties rabbitMqConfigProperties) { + this.rabbitMqConfigProperties = rabbitMqConfigProperties; + } - @PostConstruct - private void init() { - ConnectionFactory factory = new ConnectionFactory(); - try { - LOG.debug("RabbitMQ Channel " + uri); - factory.setUri(uri); - factory.setAutomaticRecoveryEnabled(true); - Connection conn = factory.newConnection(); - this.channel = conn.createChannel(); - LOG.debug("RabbitMQ Channel was created."); - for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { - channel.queueDeclare(queue, true, false, false, null); - LOG.debug("Queue [" + queue + "] was declared."); - } - } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { - LOG.error(e.getMessage(), e); - } + @PostConstruct + private void init() { + ConnectionFactory factory = new ConnectionFactory(); + try { + String uri = rabbitMqConfigProperties.getMbRabbitMqUri(); + LOG.debug(String.format("RabbitMQ Channel = %s", uri)); + factory.setUri(uri); + factory.setAutomaticRecoveryEnabled(true); + Connection conn = factory.newConnection(); + this.channel = conn.createChannel(); + LOG.debug("RabbitMQ Channel was created."); + for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { + channel.queueDeclare(queue, true, false, false, null); + LOG.debug(String.format("Queue [%s] was declared.", queue)); + } + } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { + LOG.error(e.getMessage(), e); + } - } + } - @Override - public void sendMessage(String msg) { - this.sendMessage("records", msg); - } + @Override + public void sendMessage(String msg) { + this.sendMessage("records", msg); + } - @Override - public void sendMessage(String queueName, String msg) { - String queueNameWithPrefix = queueName; - try { - channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); - LOG.info(" [x] Sent '" + msg + "' to queue [" + queueNameWithPrefix + "]"); - } catch (IOException e) { - LOG.error("Unable to publish message to [" + queueNameWithPrefix + "]"); - LOG.error(e.getMessage(), e); - } - } + @Override + public void sendMessage(String queueName, String msg) { + String queueNameWithPrefix = queueName; + try { + channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); + LOG.info(String.format("[x] Sent '%s' to queue [%s]", msg, queueNameWithPrefix)); + } catch (IOException e) { + LOG.error(String.format("Unable to publish message to [%s]", queueNameWithPrefix)); + LOG.error(e.getMessage(), e); + } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java index 8aad1a1a1edadc6cb355859a4e21aaea0520748e..b60b6e590e5ec671110e74fa49949391c219ce23 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,58 +40,63 @@ import org.springframework.stereotype.Component; @Component public class TenantFactoryImpl implements ITenantFactory { - private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); - - public static final String MAIN_DATABASE = "main"; - public static final String TENANT_INFO = "tenantinfo"; - - @Autowired - private MongoDdmsClient mongoClient; - - private Map<String, TenantInfo> tenants; - - public boolean exists(String tenantName) { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.containsKey(tenantName); - } - - public TenantInfo getTenantInfo(String tenantName) { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.get(tenantName); - } - - public Collection<TenantInfo> listTenantInfo() { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.values(); - } - - public <V> ICache<String, V> createCache(String tenantName, String host, int port, - int expireTimeSeconds, Class<V> classOfV) { - return null; - } - - public void flushCache() { - } - - private void initTenants() { - this.tenants = new HashMap<>(); - MongoCollection<Document> mongoCollection = mongoClient - .getMongoCollection(MAIN_DATABASE, TENANT_INFO); - FindIterable<Document> results = mongoCollection.find(); - if (Objects.isNull(results) && Objects.isNull(results.first())) { - LOG.error(String.format("Collection \'%s\' is empty.", results)); - } - for (Document document : results) { - TenantInfo tenantInfo = new Gson().fromJson(document.toJson(),TenantInfo.class); - this.tenants.put(tenantInfo.getName(), tenantInfo); - } - } + private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); + + public static final String MAIN_DATABASE = "main"; + public static final String TENANT_INFO = "tenantinfo"; + + + private final MongoDdmsClient mongoClient; + + @Autowired + public TenantFactoryImpl(MongoDdmsClient mongoClient) { + this.mongoClient = mongoClient; + } + + private Map<String, TenantInfo> tenants; + + public boolean exists(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.containsKey(tenantName); + } + + public TenantInfo getTenantInfo(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.get(tenantName); + } + + public Collection<TenantInfo> listTenantInfo() { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.values(); + } + + public <V> ICache<String, V> createCache(String tenantName, String host, int port, + int expireTimeSeconds, Class<V> classOfV) { + return null; + } + + public void flushCache() { + } + + private void initTenants() { + this.tenants = new HashMap<>(); + MongoCollection<Document> mongoCollection = mongoClient + .getMongoCollection(MAIN_DATABASE, TENANT_INFO); + FindIterable<Document> results = mongoCollection.find(); + if (Objects.isNull(results) && Objects.isNull(results.first())) { + LOG.error(String.format("Collection \'%s\' is empty.", results)); + } + for (Document document : results) { + TenantInfo tenantInfo = new Gson().fromJson(document.toJson(), TenantInfo.class); + this.tenants.put(tenantInfo.getName(), tenantInfo); + } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java deleted file mode 100644 index 2adc6336b182750c215b84de565f37e7ebb8da74..0000000000000000000000000000000000000000 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.kms; - -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.cloudkms.v1.CloudKMS; -import com.google.api.services.cloudkms.v1.CloudKMSScopes; -import com.google.api.services.cloudkms.v1.model.DecryptRequest; -import com.google.api.services.cloudkms.v1.model.DecryptResponse; -import com.google.api.services.cloudkms.v1.model.EncryptRequest; -import com.google.api.services.cloudkms.v1.model.EncryptResponse; -import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; -import org.opengroup.osdu.core.common.search.Preconditions; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -@Component -@RequestScope -public class KmsClient implements IKmsClient { - - @Value("${GOOGLE_CLOUD_PROJECT}") - private String GOOGLE_CLOUD_PROJECT; - - @Value("${KMS_KEY}") - private String KMS_KEY; - - @Value("${KEY_RING}") - private String KEY_RING; - - private static final String KEY_NAME = "projects/%s/locations/global/keyRings/%s/cryptoKeys/%s"; - - /** - * Encrypts the given plaintext using the specified crypto key. - * Google KMS automatically uses the new primary key version to encrypt data, so this could be directly used for key rotation - */ - public String encryptString(String textToBeEncrypted) throws IOException { - Preconditions.checkNotNullOrEmpty(textToBeEncrypted, "textToBeEncrypted cannot be null"); - - byte[] plaintext = textToBeEncrypted.getBytes(StandardCharsets.UTF_8); - String resourceName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY); - CloudKMS kms = createAuthorizedClient(); - EncryptRequest request = new EncryptRequest().encodePlaintext(plaintext); - EncryptResponse response = kms.projects().locations().keyRings().cryptoKeys() - .encrypt(resourceName, request) - .execute(); - return response.getCiphertext(); - } - - /** - * Decrypts the provided ciphertext with the specified crypto key. - * Google KMS automatically uses the correct key version to decrypt data, as long as the key version is not disabled - */ - public String decryptString(String textToBeDecrypted) throws IOException { - Preconditions.checkNotNullOrEmpty(textToBeDecrypted, "textToBeDecrypted cannot be null"); - - CloudKMS kms = createAuthorizedClient(); - String cryptoKeyName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY); - DecryptRequest request = new DecryptRequest().setCiphertext(textToBeDecrypted); - DecryptResponse response = kms.projects().locations().keyRings().cryptoKeys() - .decrypt(cryptoKeyName, request) - .execute(); - return new String(response.decodePlaintext(), StandardCharsets.UTF_8).trim(); - } - - /** - * Creates an authorized CloudKMS client service using Application Default Credentials. - * - * @return an authorized CloudKMS client - * @throws IOException if there's an error getting the default credentials. - */ - private CloudKMS createAuthorizedClient() throws IOException { - HttpTransport transport = new NetHttpTransport(); - JsonFactory jsonFactory = new JacksonFactory(); - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - if (credential.createScopedRequired()) { - credential = credential.createScoped(CloudKMSScopes.all()); - } - return new CloudKMS.Builder(transport, jsonFactory, credential) - .setApplicationName("CloudKMS snippets") - .build(); - } -} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java index 1679a6d63d3b005eb5a807f881299aa11cd377c3..21316cc68c9d5fa64615a8616f3f6a0e533f8ed4 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java index 95bbd9be85eb877f5b5025ca2056c80eb14111da..12cffcf01340bf7c7ce763e2ba7a19202ed1a8e6 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,104 +18,110 @@ package org.opengroup.osdu.indexer.middleware; import com.google.common.base.Strings; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import lombok.extern.java.Log; import org.apache.http.HttpStatus; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.http.ResponseHeaders; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Component; -import javax.inject.Inject; -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - @Log @Component public class IndexFilter implements Filter { - @Inject - private DpsHeaders dpsHeaders; - - @Inject - private IRequestInfo requestInfo; - @Value("${DEPLOYMENT_ENVIRONMENT}") - private String DEPLOYMENT_ENVIRONMENT; - - private FilterConfig filterConfig; - - private static final String PATH_SWAGGER = "/swagger.json"; - private static final String PATH_TASK_HANDLERS = "task-handlers"; - private static final String PATH_CRON_HANDLERS = "cron-handlers"; + private final IndexerConfigurationProperties indexerConfigurationProperties; + private final DpsHeaders dpsHeaders; + private final IRequestInfo requestInfo; + + private FilterConfig filterConfig; + + private static final String PATH_SWAGGER = "/swagger.json"; + private static final String PATH_TASK_HANDLERS = "task-handlers"; + private static final String PATH_CRON_HANDLERS = "cron-handlers"; + + @Autowired + public IndexFilter(IndexerConfigurationProperties indexerConfigurationProperties, + DpsHeaders dpsHeaders, IRequestInfo requestInfo) { + this.indexerConfigurationProperties = indexerConfigurationProperties; + this.dpsHeaders = dpsHeaders; + this.requestInfo = requestInfo; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, + FilterChain filterChain) + throws IOException, ServletException { + HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; + String uri = httpRequest.getRequestURI().toLowerCase(); + + if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri + .contains(PATH_TASK_HANDLERS)) { + if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment()) + != DeploymentEnvironment.LOCAL) { + checkWorkerApiAccess(requestInfo); + } + } - @Override - public void init(FilterConfig filterConfig) throws ServletException { + if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri + .contains(PATH_CRON_HANDLERS)) { + checkWorkerApiAccess(requestInfo); } - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; - String uri = httpRequest.getRequestURI().toLowerCase(); - - if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri.contains(PATH_TASK_HANDLERS)) { - if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) != DeploymentEnvironment.LOCAL) { - checkWorkerApiAccess(requestInfo); - } - } - - if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri.contains(PATH_CRON_HANDLERS)) { - checkWorkerApiAccess(requestInfo); - } - -// if (!httpRequest.isSecure()) { -// throw new AppException(302, "Redirect", "HTTP is not supported. Use HTTPS."); -// } - - filterChain.doFilter(servletRequest, servletResponse); - - HttpServletResponse httpResponse = (HttpServletResponse) servletResponse; - Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS; - for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) { - httpResponse.addHeader(header.getKey(), header.getValue().toString()); - } - if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) { - httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId()); - } + filterChain.doFilter(servletRequest, servletResponse); + HttpServletResponse httpResponse = (HttpServletResponse) servletResponse; + Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS; + for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) { + httpResponse.addHeader(header.getKey(), header.getValue().toString()); } - - @Override - public void destroy() { + if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) { + httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId()); } + } - private void checkWorkerApiAccess(IRequestInfo requestInfo) { -// if (requestInfo.isTaskQueueRequest()) return; -// throw AppException.createForbidden("invalid user agent, AppEngine Task Queue only"); - } + @Override + public void destroy() { + } - private List<String> validateAccountId(DpsHeaders requestHeaders) { - String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId(); - String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader); + private void checkWorkerApiAccess(IRequestInfo requestInfo) { + } - if (Strings.isNullOrEmpty(accountHeader)) { - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo); - } + private List<String> validateAccountId(DpsHeaders requestHeaders) { + String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId(); + String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader); - List<String> dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*")); - if (dataPartitions.isEmpty() || dataPartitions.size() > 1) { - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo); - } - return dataPartitions; + if (Strings.isNullOrEmpty(accountHeader)) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", + "invalid or empty data partition", debuggingInfo); } + List<String> dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*")); + if (dataPartitions.isEmpty() || dataPartitions.size() > 1) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", + "invalid or empty data partition", debuggingInfo); + } + return dataPartitions; + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java index c3285bd78e05d172f3923a667fa5593be86fdb5c..9b7f8b1f5edd751f35dcd850ec57073c594f838f 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,82 +38,86 @@ import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; public class DatastoreCredential extends GoogleCredentials { - private static final long serialVersionUID = 8344377091688956815L; - private static final JsonFactory JSON_FACTORY = new JacksonFactory(); - private Iam iam; + private static final long serialVersionUID = 8344377091688956815L; + private static final JsonFactory JSON_FACTORY = new JacksonFactory(); + private Iam iam; - private final TenantInfo tenant; - private final DatastoreCredentialCache cache; + private final TenantInfo tenant; + private final DatastoreCredentialCache cache; - protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) { - this.tenant = tenant; - this.cache = cache; - } + protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) { + this.tenant = tenant; + this.cache = cache; + } - @Override - public AccessToken refreshAccessToken() { + @Override + public AccessToken refreshAccessToken() { - String cacheKey = this.getCacheKey(); + String cacheKey = this.getCacheKey(); - AccessToken accessToken = this.cache.get(cacheKey); + AccessToken accessToken = this.cache.get(cacheKey); - if (accessToken != null) { - return accessToken; - } + if (accessToken != null) { + return accessToken; + } - try { - SignJwtRequest signJwtRequest = new SignJwtRequest(); - signJwtRequest.setPayload(this.getPayload()); + try { + SignJwtRequest signJwtRequest = new SignJwtRequest(); + signJwtRequest.setPayload(this.getPayload()); - String serviceAccountName = String.format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount()); + String serviceAccountName = String + .format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount()); - SignJwt signJwt = this.getIam().projects().serviceAccounts().signJwt(serviceAccountName, signJwtRequest); + SignJwt signJwt = this.getIam().projects().serviceAccounts() + .signJwt(serviceAccountName, signJwtRequest); - SignJwtResponse signJwtResponse = signJwt.execute(); - String signedJwt = signJwtResponse.getSignedJwt(); + SignJwtResponse signJwtResponse = signJwt.execute(); + String signedJwt = signJwtResponse.getSignedJwt(); - accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600)); + accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600)); - this.cache.put(cacheKey, accessToken); + this.cache.put(cacheKey, accessToken); - return accessToken; - } catch (Exception e) { - throw new RuntimeException("Error creating datastore credential", e); - } - } + return accessToken; + } catch (Exception e) { + throw new RuntimeException("Error creating datastore credential", e); + } + } - private String getPayload() { - JsonObject payload = new JsonObject(); - payload.addProperty("iss", this.tenant.getServiceAccount()); - payload.addProperty("sub", this.tenant.getServiceAccount()); - payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore"); - payload.addProperty("iat", System.currentTimeMillis() / 1000); + private String getPayload() { + JsonObject payload = new JsonObject(); + payload.addProperty("iss", this.tenant.getServiceAccount()); + payload.addProperty("sub", this.tenant.getServiceAccount()); + payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore"); + payload.addProperty("iat", System.currentTimeMillis() / 1000); - return payload.toString(); - } + return payload.toString(); + } - protected void setIam(Iam iam) { - this.iam = iam; - } + protected void setIam(Iam iam) { + this.iam = iam; + } - private Iam getIam() throws Exception { - if (this.iam == null) { + private Iam getIam() throws Exception { + if (this.iam == null) { - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - if (credential.createScopedRequired()) { - credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM)); - } + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + if (credential.createScopedRequired()) { + credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM)); + } - Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY, - credential).setApplicationName("Search Service"); + Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), + JSON_FACTORY, + credential).setApplicationName("Search Service"); - this.iam = builder.build(); - } - return this.iam; - } + this.iam = builder.build(); + } + return this.iam; + } - private String getCacheKey() { - return Crc32c.hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName())); - } + private String getCacheKey() { + return Crc32c + .hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName())); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java index db90ee7efb2dfee6fb3995d09b0fe158a03ad9e1..144edee0eb4c1bd99054bd1a1859588adb9be3ae 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,49 +22,54 @@ import com.google.cloud.TransportOptions; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.http.HttpTransportOptions; +import java.util.HashMap; +import java.util.Map; +import javax.inject.Inject; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.threeten.bp.Duration; -import javax.inject.Inject; -import java.util.HashMap; -import java.util.Map; - @Component public class DatastoreFactory { - @Inject - private DatastoreCredentialCache cache; + @Inject + private DatastoreCredentialCache cache; + + @Autowired + public DatastoreFactory(DatastoreCredentialCache cache) { + this.cache = cache; + } - private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>(); + private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>(); - private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() - .setMaxAttempts(6) - .setInitialRetryDelay(Duration.ofSeconds(10)) - .setMaxRetryDelay(Duration.ofSeconds(32)) - .setRetryDelayMultiplier(2.0) - .setTotalTimeout(Duration.ofSeconds(50)) - .setInitialRpcTimeout(Duration.ofSeconds(50)) - .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofSeconds(50)) - .build(); + private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() + .setMaxAttempts(6) + .setInitialRetryDelay(Duration.ofSeconds(10)) + .setMaxRetryDelay(Duration.ofSeconds(32)) + .setRetryDelayMultiplier(2.0) + .setTotalTimeout(Duration.ofSeconds(50)) + .setInitialRpcTimeout(Duration.ofSeconds(50)) + .setRpcTimeoutMultiplier(1.0) + .setMaxRpcTimeout(Duration.ofSeconds(50)) + .build(); - private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder() - .setReadTimeout(30000) - .build(); + private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder() + .setReadTimeout(30000) + .build(); - public Datastore getDatastoreInstance(TenantInfo tenantInfo) { - if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) { - Datastore googleDatastore = DatastoreOptions.newBuilder() - .setCredentials(new DatastoreCredential(tenantInfo, this.cache)) - .setRetrySettings(RETRY_SETTINGS) - .setTransportOptions(TRANSPORT_OPTIONS) - .setNamespace(tenantInfo.getName()) - .setProjectId(tenantInfo.getProjectId()) - .build().getService(); - DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore); - } - return DATASTORE_CLIENTS.get(tenantInfo.getName()); + public Datastore getDatastoreInstance(TenantInfo tenantInfo) { + if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) { + Datastore googleDatastore = DatastoreOptions.newBuilder() + .setCredentials(new DatastoreCredential(tenantInfo, this.cache)) + .setRetrySettings(RETRY_SETTINGS) + .setTransportOptions(TRANSPORT_OPTIONS) + .setNamespace(tenantInfo.getName()) + .setProjectId(tenantInfo.getProjectId()) + .build().getService(); + DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore); } + return DATASTORE_CLIENTS.get(tenantInfo.getName()); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java index a3ab89c5d9dc1824af87ad823e547befbc6c4fa8..1c4d4fba54520a6fe22fc1d32c129dda80309432 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ package org.opengroup.osdu.indexer.persistence; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import java.util.Objects; @@ -27,7 +26,6 @@ import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; -import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -36,53 +34,54 @@ import org.springframework.stereotype.Component; @Component public class ElasticRepositoryMongoDB implements IElasticRepository { - private static final Logger LOG = LoggerFactory.getLogger(ElasticRepositoryMongoDB.class); + private static final Logger LOG = LoggerFactory.getLogger(ElasticRepositoryMongoDB.class); - private static final String SCHEMA_DATABASE = "local"; - private static final String SEARCH_SETTINGS = "SearchSettings"; - private static final String HOST = "host"; - private static final String PORT = "port"; - private static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration"; + private static final String SCHEMA_DATABASE = "local"; + private static final String SEARCH_SETTINGS = "SearchSettings"; + private static final String HOST = "host"; + private static final String PORT = "port"; + private static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration"; - @Autowired - private MongoDdmsClient mongoClient; + private final MongoDdmsClient mongoClient; - @Autowired - private IKmsClient kmsClient; + @Autowired + public ElasticRepositoryMongoDB(MongoDdmsClient mongoClient) { + this.mongoClient = mongoClient; + } - @Override - public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) { - MongoCollection<Document> mongoCollection = mongoClient - .getMongoCollection(SCHEMA_DATABASE, SEARCH_SETTINGS); + @Override + public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) { + MongoCollection<Document> mongoCollection = mongoClient + .getMongoCollection(SCHEMA_DATABASE, SEARCH_SETTINGS); - FindIterable<Document> results = mongoCollection.find(); + FindIterable<Document> results = mongoCollection.find(); - if (Objects.isNull(results) && Objects.isNull(results.first())) { - LOG.error(String.format("Collection \'%s\' is empty.", results)); - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", - "An error has occurred fetching cluster settings from the database."); - } + if (Objects.isNull(results) && Objects.isNull(results.first())) { + LOG.error(String.format("Collection \'%s\' is empty.", results)); + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", + "An error has occurred fetching cluster settings from the database."); + } - Document document = results.first(); + Document document = results.first(); - String encryptedConfiguration = document.get(XPACK_RESTCLIENT_CONFIGURATION).toString(); - String encryptedHost = document.get(HOST).toString(); - String encryptedPort = document.get(PORT).toString(); + String encryptedConfiguration = document.get(XPACK_RESTCLIENT_CONFIGURATION).toString(); + String encryptedHost = document.get(HOST).toString(); + String encryptedPort = document.get(PORT).toString(); - try { + try { - String host = encryptedHost;//this.kmsClient.decryptString(encryptedHost); - String portString = encryptedPort;//this.kmsClient.decryptString(encryptedPort); - String usernameAndPassword = encryptedConfiguration;//this.kmsClient.decryptString(encryptedConfiguration); + String host = encryptedHost;//this.kmsClient.decryptString(encryptedHost); + String portString = encryptedPort;//this.kmsClient.decryptString(encryptedPort); + String usernameAndPassword = encryptedConfiguration;//this.kmsClient.decryptString(encryptedConfiguration); - int port = Integer.parseInt(portString); - ClusterSettings clusterSettings = new ClusterSettings(host, port, usernameAndPassword); - clusterSettings.setHttps(false); - return clusterSettings; + int port = Integer.parseInt(portString); + ClusterSettings clusterSettings = new ClusterSettings(host, port, usernameAndPassword); + clusterSettings.setHttps(false); + return clusterSettings; - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", - "An error has occurred fetching cluster settings from the database.", e); - } - } + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", + "An error has occurred fetching cluster settings from the database.", e); + } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java index 43b3fd5b1b519006aa781af227f594c81f2d459a..6cb02a3a7fbfb36d6a2ada3f3b3906a1116aaaa4 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,14 +27,15 @@ import org.springframework.stereotype.Component; @Component public class MongoDdmsClient { - @Autowired - private MongoClientHandler mongoClientHandler; + private final MongoClientHandler mongoClientHandler; @Autowired - private TenantInfo tenantInfo; + public MongoDdmsClient(MongoClientHandler mongoClientHandler) { + this.mongoClientHandler = mongoClientHandler; + } - public MongoCollection<Document> getMongoCollection(String dbName, String collectionName) { - return mongoClientHandler.getMongoClient().getDatabase(dbName) - .getCollection(collectionName); - } + public MongoCollection<Document> getMongoCollection(String dbName, String collectionName) { + return mongoClientHandler.getMongoClient().getDatabase(dbName) + .getCollection(collectionName); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java index d2a4d74cd3ee5969714eb84317bcef27813bcb75..a9a82d8cffb4ea3b86d56d66e210f00575f0a034 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,61 +43,67 @@ import org.springframework.web.context.annotation.RequestScope; @RequestScope public class PublisherImpl implements IPublisher { - private static final Logger LOG = LoggerFactory.getLogger(PublisherImpl.class); - - @Autowired - IMessageFactory mq; - - - @Override - public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception { - - String tenant = headers.getPartitionId(); - if (Strings.isNullOrEmpty(tenant)) { - tenant = headers.getAccountId(); - } - - Map<String, String> message = new HashMap<>(); - message.put(tenant, headers.getPartitionIdWithFallbackToAccountId()); - headers.addCorrelationIdIfMissing(); - message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); - - RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers, indexerBatchStatus); - message.put("data", recordChangedMessages.toString()); - - try { - LOG.info("Indexer publishes message " + headers.getCorrelationId()); - mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString()); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - } - - private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers, JobStatus indexerBatchStatus) { - - Gson gson = new GsonBuilder().create(); - Map<String, String> attributesMap = new HashMap<>(); - Type listType = new TypeToken<List<RecordStatus>>() { - }.getType(); - - JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType); - String statusChangedTagsData = (statusChangedTagsJson.toString()); - - String tenant = headers.getPartitionId(); - // This code it to provide backward compatibility to slb-account-id - if (!Strings.isNullOrEmpty(tenant)) { - attributesMap.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); - } else { - attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); - } - headers.addCorrelationIdIfMissing(); - attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); - - RecordChangedMessages recordChangedMessages = new RecordChangedMessages(); - // statusChangedTagsData is not ByteString but String - recordChangedMessages.setData(statusChangedTagsData); - recordChangedMessages.setAttributes(attributesMap); - - return recordChangedMessages; - } + private static final Logger LOG = LoggerFactory.getLogger(PublisherImpl.class); + private final IMessageFactory mq; + + @Autowired + public PublisherImpl(IMessageFactory mq) { + this.mq = mq; + } + + @Override + public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) { + + String tenant = headers.getPartitionId(); + if (Strings.isNullOrEmpty(tenant)) { + tenant = headers.getAccountId(); + } + + Map<String, String> message = new HashMap<>(); + message.put(tenant, headers.getPartitionIdWithFallbackToAccountId()); + headers.addCorrelationIdIfMissing(); + message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + + RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers, + indexerBatchStatus); + message.put("data", recordChangedMessages.toString()); + + try { + LOG.info("Indexer publishes message " + headers.getCorrelationId()); + mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString()); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers, + JobStatus indexerBatchStatus) { + + Gson gson = new GsonBuilder().create(); + Map<String, String> attributesMap = new HashMap<>(); + Type listType = new TypeToken<List<RecordStatus>>() { + }.getType(); + + JsonElement statusChangedTagsJson = gson + .toJsonTree(indexerBatchStatus.getStatusesList(), listType); + String statusChangedTagsData = (statusChangedTagsJson.toString()); + + String tenant = headers.getPartitionId(); + // This code it to provide backward compatibility to slb-account-id + if (!Strings.isNullOrEmpty(tenant)) { + attributesMap + .put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + } else { + attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); + } + headers.addCorrelationIdIfMissing(); + attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + + RecordChangedMessages recordChangedMessages = new RecordChangedMessages(); + // statusChangedTagsData is not ByteString but String + recordChangedMessages.setData(statusChangedTagsData); + recordChangedMessages.setAttributes(attributesMap); + + return recordChangedMessages; + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java index 7927d6f3b55656168683b497d67597d446f0246b..0db2f9e557efcc69fbd0419df57fc3a821e024de 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,18 +29,16 @@ import org.springframework.security.config.annotation.web.configuration.WebSecur @EnableGlobalMethodSecurity(prePostEnabled = true) public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter { - @Override - protected void configure(HttpSecurity http) throws Exception { - - http.httpBasic().disable() - .csrf().disable(); //disable default authN. AuthN handled by endpoints proxy - - } - - @Override - public void configure(WebSecurity web) throws Exception { - web.ignoring().antMatchers("/api-docs") - .antMatchers("/swagger"); - } + @Override + protected void configure(HttpSecurity http) throws Exception { + http.httpBasic().disable() + .csrf().disable(); + } + + @Override + public void configure(WebSecurity web) throws Exception { + web.ignoring().antMatchers("/api-docs") + .antMatchers("/swagger"); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java index 0b558b08ccb00f8666d4e98ecf4961bef37d69c9..558e30bcc6d7c6653f51f1d023a56b6f015a5c18 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java @@ -1,16 +1,19 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.opengroup.osdu.indexer.service; @@ -68,458 +71,544 @@ import org.springframework.stereotype.Service; @Primary public class IndexerServiceImpl implements IndexerService { - private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1); - - private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE)); - - private final Gson gson = new Gson(); - - @Inject - private JaxRsDpsLog jaxRsDpsLog; - @Inject - private AuditLogger auditLogger; - @Inject - private StorageService storageService; - @Inject - private IndexSchemaService schemaService; - @Inject - private IndicesService indicesService; - @Inject - private IndexerMappingService mappingService; - @Inject - private IPublisher progressPublisher; - @Inject - private ElasticClientHandler elasticClientHandler; - @Inject - private IndexerQueueTaskBuilder indexerQueueTaskBuilder; - @Inject - private ElasticIndexNameResolver elasticIndexNameResolver; - @Inject - private StorageIndexerPayloadMapper storageIndexerPayloadMapper; - @Inject - private IRequestInfo requestInfo; - @Inject - private JobStatus jobStatus; - - private DpsHeaders headers; - - @Override - public JobStatus processRecordChangedMessages(RecordChangedMessages message, List<RecordInfo> recordInfos) throws Exception { - - // this should not happen - if (recordInfos.size() == 0) return null; - - String errorMessage = ""; - List<String> retryRecordIds = new LinkedList<>(); - - // get auth header with service account Authorization - this.headers = this.requestInfo.getHeaders(); - String authorization = message.getAttributes().get("authorization"); - headers.put("authorization", authorization); - // initialize status for all messages. - this.jobStatus.initialize(recordInfos); - - try { - // get upsert records - Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos); - if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) { - List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap); - retryRecordIds.addAll(upsertFailureRecordIds); - } - - // get delete records - Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos); - if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) { - List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap); - retryRecordIds.addAll(deleteFailureRecordIds); - } - - // process schema change messages - Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); - if (schemaMsgs != null && !schemaMsgs.isEmpty()) { - this.schemaService.processSchemaMessages(schemaMsgs); - } - - // process failed records - if (retryRecordIds.size() > 0) { - retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message); - } - } catch (IOException e) { - errorMessage = e.getMessage(); - throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e); - } catch (AppException e) { - errorMessage = e.getMessage(); - throw e; - } catch (Exception e) { - errorMessage = "error indexing records"; - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e); - } finally { - this.jobStatus.finalizeRecordStatus(errorMessage); - this.updateAuditLog(); - this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus); - } - - return jobStatus; + private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1); + + private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays + .asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, + RestStatus.SERVICE_UNAVAILABLE)); + + private final Gson gson = new Gson(); + + @Inject + private JaxRsDpsLog jaxRsDpsLog; + @Inject + private AuditLogger auditLogger; + @Inject + private StorageService storageService; + @Inject + private IndexSchemaService schemaService; + @Inject + private IndicesService indicesService; + @Inject + private IndexerMappingService mappingService; + @Inject + private IPublisher progressPublisher; + @Inject + private ElasticClientHandler elasticClientHandler; + @Inject + private IndexerQueueTaskBuilder indexerQueueTaskBuilder; + @Inject + private ElasticIndexNameResolver elasticIndexNameResolver; + @Inject + private StorageIndexerPayloadMapper storageIndexerPayloadMapper; + @Inject + private IRequestInfo requestInfo; + @Inject + private JobStatus jobStatus; + + private DpsHeaders headers; + + @Override + public JobStatus processRecordChangedMessages(RecordChangedMessages message, + List<RecordInfo> recordInfos) throws Exception { + + // this should not happen + if (recordInfos.size() == 0) { + return null; } - @Override - public void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException { - Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); - if (schemaMsgs != null && !schemaMsgs.isEmpty()) { - try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) { - schemaMsgs.entrySet().forEach(msg -> { - try { - processSchemaEvents(restClient, msg); - } catch (IOException | ElasticsearchStatusException e) { - throw new AppException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), "unable to process schema delete", e.getMessage()); - } - }); - } - } + String errorMessage = ""; + List<String> retryRecordIds = new LinkedList<>(); + + // get auth header with service account Authorization + this.headers = this.requestInfo.getHeaders(); + String authorization = message.getAttributes().get("authorization"); + headers.put("authorization", authorization); + // initialize status for all messages. + this.jobStatus.initialize(recordInfos); + + try { + // get upsert records + Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo + .getUpsertRecordIds(recordInfos); + if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) { + List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap); + retryRecordIds.addAll(upsertFailureRecordIds); + } + + // get delete records + Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos); + if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) { + List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap); + retryRecordIds.addAll(deleteFailureRecordIds); + } + + // process schema change messages + Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); + if (schemaMsgs != null && !schemaMsgs.isEmpty()) { + this.schemaService.processSchemaMessages(schemaMsgs); + } + + // process failed records + if (retryRecordIds.size() > 0) { + retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message); + } + } catch (IOException e) { + errorMessage = e.getMessage(); + throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", + errorMessage, e); + } catch (AppException e) { + errorMessage = e.getMessage(); + throw e; + } catch (Exception e) { + errorMessage = "error indexing records"; + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", + "An unknown error has occurred.", e); + } finally { + this.jobStatus.finalizeRecordStatus(errorMessage); + this.updateAuditLog(); + this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus); } - private void processSchemaEvents(RestHighLevelClient restClient, - Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException { - String kind = msg.getKey(); - String index = elasticIndexNameResolver.getIndexNameFromKind(kind); - - boolean indexExist = indicesService.isIndexExist(restClient, index); - if (indexExist && msg.getValue() == OperationType.purge_schema) { - indicesService.deleteIndex(restClient, index); - } + return jobStatus; + } + + @Override + public void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException { + Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); + if (schemaMsgs != null && !schemaMsgs.isEmpty()) { + try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) { + schemaMsgs.entrySet().forEach(msg -> { + try { + processSchemaEvents(restClient, msg); + } catch (IOException | ElasticsearchStatusException e) { + throw new AppException( + org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), + "unable to process schema delete", e.getMessage()); + } + }); + } } + } - private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) throws Exception { - // get schema for kind - Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap); - - if (schemas.isEmpty()) return new LinkedList<>(); - - // get recordIds with valid upsert index-status - List<String> recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus(); - - if (recordIds.isEmpty()) return new LinkedList<>(); - - // get records via storage api - Records storageRecords = this.storageService.getStorageRecords(recordIds); - List<String> failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords()); - - // map storage records to indexer payload - RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords); + private void processSchemaEvents(RestHighLevelClient restClient, + Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException { + String kind = msg.getKey(); + String index = elasticIndexNameResolver.getIndexNameFromKind(kind); - jaxRsDpsLog.info(String.format("records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", recordIds.size(), storageRecords.getRecords().size(), recordIndexerPayload.getRecords().size())); + boolean indexExist = indicesService.isIndexExist(restClient, index); + if (indexExist && msg.getValue() == OperationType.purge_schema) { + indicesService.deleteIndex(restClient, index); + } + } - // index records - failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload)); + private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) + throws Exception { + // get schema for kind + Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap); - return failedOrRetryRecordIds; + if (schemas.isEmpty()) { + return new LinkedList<>(); } - private Map<String, IndexSchema> getSchema(Map<String, Map<String, OperationType>> upsertRecordMap) { + // get recordIds with valid upsert index-status + List<String> recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus(); - Map<String, IndexSchema> schemas = new HashMap<>(); + if (recordIds.isEmpty()) { + return new LinkedList<>(); + } - try { - for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) { + // get records via storage api + Records storageRecords = this.storageService.getStorageRecords(recordIds); + List<String> failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords()); - String kind = entry.getKey(); - IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false); - if (schemaObj.isDataSchemaMissing()) { - this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind)); - } + // map storage records to indexer payload + RecordIndexerPayload recordIndexerPayload = this + .getIndexerPayload(upsertRecordMap, schemas, storageRecords); - schemas.put(kind, schemaObj); - } - } catch (AppException e) { - throw e; - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e); - } - - return schemas; - } + jaxRsDpsLog.info(String.format( + "records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", + recordIds.size(), storageRecords.getRecords().size(), + recordIndexerPayload.getRecords().size())); - private RecordIndexerPayload getIndexerPayload(Map<String, Map<String, OperationType>> upsertRecordMap, Map<String, IndexSchema> kindSchemaMap, Records records) { - List<Records.Entity> storageValidRecords = records.getRecords(); - List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>(); - List<IndexSchema> schemas = new ArrayList<>(); + // index records + failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload)); - for (Records.Entity storageRecord : storageValidRecords) { + return failedOrRetryRecordIds; + } - Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind()); + private Map<String, IndexSchema> getSchema( + Map<String, Map<String, OperationType>> upsertRecordMap) { - // skip if storage returned record with same id but different kind - if (idOperationMap == null) { - String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind()); - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId())); - continue; - } + Map<String, IndexSchema> schemas = new HashMap<>(); - IndexSchema schema = kindSchemaMap.get(storageRecord.getKind()); - schemas.add(schema); + try { + for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) { - // skip indexing of records if data block is empty - RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap); - if (document != null) { - indexerPayload.add(document); - } + String kind = entry.getKey(); + IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false); + if (schemaObj.isDataSchemaMissing()) { + this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, + HttpStatus.SC_NOT_FOUND, "schema not found", + String.format("schema not found | kind: %s", kind)); } - // this should only happen if storage service returned WRONG records with kind for all the records in the messages - if (indexerPayload.isEmpty()) { - throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records"); - } + schemas.put(kind, schemaObj); + } + } catch (AppException e) { + throw e; + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", + "An error has occurred while getting schema", e); + } - return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build(); + return schemas; + } + + private RecordIndexerPayload getIndexerPayload( + Map<String, Map<String, OperationType>> upsertRecordMap, + Map<String, IndexSchema> kindSchemaMap, Records records) { + List<Records.Entity> storageValidRecords = records.getRecords(); + List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>(); + List<IndexSchema> schemas = new ArrayList<>(); + + for (Records.Entity storageRecord : storageValidRecords) { + + Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind()); + + // skip if storage returned record with same id but different kind + if (idOperationMap == null) { + String message = String.format( + "storage service returned incorrect record | requested kind: %s | received kind: %s", + this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind()); + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, + RequestStatus.STORAGE_CONFLICT, message, + String.format("%s | record-id: %s", message, storageRecord.getId())); + continue; + } + + IndexSchema schema = kindSchemaMap.get(storageRecord.getKind()); + schemas.add(schema); + + // skip indexing of records if data block is empty + RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, + idOperationMap); + if (document != null) { + indexerPayload.add(document); + } } - private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) { - - RecordIndexerPayload.Record document = null; - - try { - Map<String, Object> storageRecordData = storageRecord.getData(); - document = new RecordIndexerPayload.Record(); - if (storageRecordData == null || storageRecordData.isEmpty()) { - String message = "empty or null data block found in the storage record"; - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message)); - } else if (schemaObj.isDataSchemaMissing()) { - document.setSchemaMissing(true); - } else { - Map<String, Object> dataMap = this.storageIndexerPayloadMapper.mapDataPayload(schemaObj, storageRecordData, storageRecord.getId()); - if (dataMap.isEmpty()) { - document.setMappingMismatch(true); - String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData); - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message)); - } - document.setData(dataMap); - } - } catch (AppException e) { - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage()); - jaxRsDpsLog.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e); - } catch (Exception e) { - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage())); - jaxRsDpsLog.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e); - } + // this should only happen if storage service returned WRONG records with kind for all the records in the messages + if (indexerPayload.isEmpty()) { + throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", + "upsert record failed, storage service returned incorrect records"); + } - try { - // index individual parts of kind - String[] kindParts = storageRecord.getKind().split(":"); - - document.setKind(storageRecord.getKind()); - document.setNamespace(kindParts[0] + ":" + kindParts[1]); - document.setType(kindParts[2]); - document.setId(storageRecord.getId()); - document.setVersion(storageRecord.getVersion()); - document.setAcl(storageRecord.getAcl()); - document.setLegal(storageRecord.getLegal()); - RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId()); - if (recordStatus.getIndexProgress().getStatusCode() == 0) { - recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK); - } - document.setIndexProgress(recordStatus.getIndexProgress()); - if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry()); - document.setOperationType(idToOperationMap.get(storageRecord.getId())); - } catch (Exception e) { - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage())); - jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e); + return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build(); + } + + private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, + Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) { + + RecordIndexerPayload.Record document = null; + + try { + Map<String, Object> storageRecordData = storageRecord.getData(); + document = new RecordIndexerPayload.Record(); + if (storageRecordData == null || storageRecordData.isEmpty()) { + String message = "empty or null data block found in the storage record"; + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, + HttpStatus.SC_NOT_FOUND, message, + String.format("record-id: %s | %s", storageRecord.getId(), message)); + } else if (schemaObj.isDataSchemaMissing()) { + document.setSchemaMissing(true); + } else { + Map<String, Object> dataMap = this.storageIndexerPayloadMapper + .mapDataPayload(schemaObj, storageRecordData, storageRecord.getId()); + if (dataMap.isEmpty()) { + document.setMappingMismatch(true); + String message = String.format( + "complete schema mismatch: none of the data attribute can be mapped | data: %s", + storageRecordData); + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, + HttpStatus.SC_NOT_FOUND, message, + String.format("record-id: %s | %s", storageRecord.getId(), message)); } - return document; + document.setData(dataMap); + } + } catch (AppException e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, + HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + jaxRsDpsLog + .warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e); + } catch (Exception e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, + HttpStatus.SC_INTERNAL_SERVER_ERROR, + String.format("error parsing records against schema, error-message: %s", e.getMessage())); + jaxRsDpsLog.error(String + .format("record-id: %s | error parsing records against schema, error-message: %s", + storageRecord.getId(), e.getMessage()), e); } - private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception { - - try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { - List<IndexSchema> schemas = recordIndexerPayload.getSchemas(); - if (schemas == null || schemas.isEmpty()) { - return new LinkedList<>(); - } - - // process the schema - this.cacheOrCreateElasticMapping(schemas, restClient); - - // process the records - return this.upsertRecords(recordIndexerPayload.getRecords(), restClient); - } + try { + // index individual parts of kind + String[] kindParts = storageRecord.getKind().split(":"); + + document.setKind(storageRecord.getKind()); + document.setNamespace(kindParts[0] + ":" + kindParts[1]); + document.setType(kindParts[2]); + document.setId(storageRecord.getId()); + document.setVersion(storageRecord.getVersion()); + document.setAcl(storageRecord.getAcl()); + document.setLegal(storageRecord.getLegal()); + RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId()); + if (recordStatus.getIndexProgress().getStatusCode() == 0) { + recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK); + } + document.setIndexProgress(recordStatus.getIndexProgress()); + if (storageRecord.getAncestry() != null) { + document.setAncestry(storageRecord.getAncestry()); + } + document.setOperationType(idToOperationMap.get(storageRecord.getId())); + } catch (Exception e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, + HttpStatus.SC_INTERNAL_SERVER_ERROR, + String.format("error parsing meta data, error-message: %s", e.getMessage())); + jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", + storageRecord.getId(), e.getMessage()), e); } + return document; + } - private void cacheOrCreateElasticMapping(List<IndexSchema> schemas, RestHighLevelClient restClient) throws Exception { + private List<String> processElasticMappingAndUpsertRecords( + RecordIndexerPayload recordIndexerPayload) throws Exception { - for (IndexSchema schema : schemas) { - String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind()); + try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { + List<IndexSchema> schemas = recordIndexerPayload.getSchemas(); + if (schemas == null || schemas.isEmpty()) { + return new LinkedList<>(); + } - // check if index exist - if (this.indicesService.isIndexExist(restClient, index)) continue; + // process the schema + this.cacheOrCreateElasticMapping(schemas, restClient); - // create index - Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema); - if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index)); - } - } + // process the records + return this.upsertRecords(recordIndexerPayload.getRecords(), restClient); + } + } + + private void cacheOrCreateElasticMapping(List<IndexSchema> schemas, + RestHighLevelClient restClient) throws Exception { + + for (IndexSchema schema : schemas) { + String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind()); + + // check if index exist + if (this.indicesService.isIndexExist(restClient, index)) { + continue; + } + + // create index + Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema); + if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", + "Error creating index.", + String.format("Failed to get confirmation from elastic server for index: %s", index)); + } } + } - private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException { - if (records == null || records.isEmpty()) return new LinkedList<>(); - - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.timeout(BULK_REQUEST_TIMEOUT); - - for (RecordIndexerPayload.Record record : records) { - if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) { - // it will come here when schema is missing - // TODO: rollback once we know what is causing the problem - jaxRsDpsLog.warning(String.format("data not found for record: %s", record)); - } - - OperationType operation = record.getOperationType(); - Map<String, Object> sourceMap = getSourceMap(record); - String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind()); - - if (operation == OperationType.create) { - IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON); - bulkRequest.add(indexRequest); - } else if (operation == OperationType.update) { - UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON); - bulkRequest.add(updateRequest); - } - } + private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, + RestHighLevelClient restClient) throws AppException { + if (records == null || records.isEmpty()) { + return new LinkedList<>(); + } - return processBulkRequest(restClient, bulkRequest); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.timeout(BULK_REQUEST_TIMEOUT); + + for (RecordIndexerPayload.Record record : records) { + if ((record.getData() == null || record.getData().isEmpty()) && !record + .skippedDataIndexing()) { + // it will come here when schema is missing + // TODO: rollback once we know what is causing the problem + jaxRsDpsLog.warning(String.format("data not found for record: %s", record)); + } + + OperationType operation = record.getOperationType(); + Map<String, Object> sourceMap = getSourceMap(record); + String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind()); + + if (operation == OperationType.create) { + IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()) + .source(this.gson.toJson(sourceMap), XContentType.JSON); + bulkRequest.add(indexRequest); + } else if (operation == OperationType.update) { + UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()) + .upsert(this.gson.toJson(sourceMap), XContentType.JSON); + bulkRequest.add(updateRequest); + } } - private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap) throws Exception { - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.timeout(BULK_REQUEST_TIMEOUT); + return processBulkRequest(restClient, bulkRequest); + } - for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) { + private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap) + throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.timeout(BULK_REQUEST_TIMEOUT); - String[] kindParts = record.getKey().split(":"); - String type = kindParts[2]; + for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) { - String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey()); + String[] kindParts = record.getKey().split(":"); + String type = kindParts[2]; - for (String id : record.getValue()) { - DeleteRequest deleteRequest = new DeleteRequest(index, type, id); - bulkRequest.add(deleteRequest); - } - } + String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey()); - try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { - return processBulkRequest(restClient, bulkRequest); - } + for (String id : record.getValue()) { + DeleteRequest deleteRequest = new DeleteRequest(index, type, id); + bulkRequest.add(deleteRequest); + } } - private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException { - - List<String> failureRecordIds = new LinkedList<>(); - if (bulkRequest.numberOfActions() == 0) return failureRecordIds; - - - - try { - BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT); - - // log failed bulk requests - ArrayList<String> bulkFailures = new ArrayList<>(); - int succeededResponses = 0; - int failedResponses = 0; - - for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { - if (bulkItemResponse.isFailed()) { - BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); - bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s", failure.getStatus(), failure.getId(), failure.getMessage())); - this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage()); - if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) { - failureRecordIds.add(bulkItemResponse.getId()); - } - failedResponses++; - } else { - succeededResponses++; - this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully"); - } - } - if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures); - - jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", bulkRequest.numberOfActions(), succeededResponses, failedResponses)); - } catch (IOException e) { - // throw explicit 504 for IOException - throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e); - } catch (ElasticsearchStatusException e) { - throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e); - } - return failureRecordIds; + try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { + return processBulkRequest(restClient, bulkRequest); } + } - private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) { + private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) + throws AppException { - Map<String, Object> indexerPayload = new HashMap<>(); - - // get the key and get the corresponding object from the individualRecord object - if (record.getData() != null) { - Map<String, Object> data = new HashMap<>(); - for (Map.Entry<String, Object> entry : record.getData().entrySet()) { - data.put(entry.getKey(), entry.getValue()); - } - indexerPayload.put(Constants.DATA, data); - } + List<String> failureRecordIds = new LinkedList<>(); + if (bulkRequest.numberOfActions() == 0) { + return failureRecordIds; + } - indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId()); - indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind()); - indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace()); - indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType()); - indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion()); - indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl()); - indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl())); - indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal()); - indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress()); - if (record.getAncestry() != null) { - indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry()); + try { + BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT); + + // log failed bulk requests + ArrayList<String> bulkFailures = new ArrayList<>(); + int succeededResponses = 0; + int failedResponses = 0; + + for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { + if (bulkItemResponse.isFailed()) { + BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); + bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s", + failure.getStatus(), failure.getId(), failure.getMessage())); + this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, + failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage()); + if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) { + failureRecordIds.add(bulkItemResponse.getId()); + } + failedResponses++; + } else { + succeededResponses++; + this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, + HttpStatus.SC_OK, "Indexed Successfully"); } - return indexerPayload; + } + if (!bulkFailures.isEmpty()) { + this.jaxRsDpsLog.warning(bulkFailures); + } + + jaxRsDpsLog.info(String + .format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", + bulkRequest.numberOfActions(), succeededResponses, failedResponses)); + } catch (IOException e) { + // throw explicit 504 for IOException + throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", + "Request cannot be completed in specified time.", e); + } catch (ElasticsearchStatusException e) { + throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", + "Error indexing records.", e); } + return failureRecordIds; + } - private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, List<String> failuresRecordIds, RecordChangedMessages message) throws IOException { - - jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds)); - List<RecordInfo> retryRecordInfos = new LinkedList<>(); - for (String recordId : failuresRecordIds) { - for (RecordInfo origMessage : recordInfos) { - if (origMessage.getId().equalsIgnoreCase(recordId)) { - retryRecordInfos.add(origMessage); - } - } - } + private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) { - RecordChangedMessages newMessage = RecordChangedMessages.builder() - .messageId(message.getMessageId()) - .publishTime(message.getPublishTime()) - .data(this.gson.toJson(retryRecordInfos)) - .attributes(message.getAttributes()).build(); + Map<String, Object> indexerPayload = new HashMap<>(); - String payLoad = this.gson.toJson(newMessage); - this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers); + // get the key and get the corresponding object from the individualRecord object + if (record.getData() != null) { + Map<String, Object> data = new HashMap<>(); + for (Map.Entry<String, Object> entry : record.getData().entrySet()) { + data.put(entry.getKey(), entry.getValue()); + } + indexerPayload.put(Constants.DATA, data); } - private void updateAuditLog() { - logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail); - logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail); - logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail); - logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail); + indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId()); + indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind()); + indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace()); + indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType()); + indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion()); + indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl()); + indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl())); + indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal()); + indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress()); + if (record.getAncestry() != null) { + indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry()); } - - private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) { - List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType); - if(!succeededRecords.isEmpty()) { - successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList())); - } - List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType); - List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType); - failedRecords.addAll(skippedRecords); - if(!failedRecords.isEmpty()) { - failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList())); + return indexerPayload; + } + + private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, + List<String> failuresRecordIds, RecordChangedMessages message) throws IOException { + + jaxRsDpsLog.info(String.format( + "queuing bulk failed records back to task-queue for retry | count: %s | records: %s", + failuresRecordIds.size(), failuresRecordIds)); + List<RecordInfo> retryRecordInfos = new LinkedList<>(); + for (String recordId : failuresRecordIds) { + for (RecordInfo origMessage : recordInfos) { + if (origMessage.getId().equalsIgnoreCase(recordId)) { + retryRecordInfos.add(origMessage); } + } + } + + RecordChangedMessages newMessage = RecordChangedMessages.builder() + .messageId(message.getMessageId()) + .publishTime(message.getPublishTime()) + .data(this.gson.toJson(retryRecordInfos)) + .attributes(message.getAttributes()).build(); + + String payLoad = this.gson.toJson(newMessage); + this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers); + } + + private void updateAuditLog() { + logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, + this.auditLogger::indexCreateRecordFail); + logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, + this.auditLogger::indexUpdateRecordFail); + logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, + this.auditLogger::indexPurgeRecordFail); + logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, + this.auditLogger::indexDeleteRecordFail); + } + + private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, + Consumer<List<String>> failedEvent) { + List<RecordStatus> succeededRecords = this.jobStatus + .getRecordStatuses(IndexingStatus.SUCCESS, operationType); + if (!succeededRecords.isEmpty()) { + successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage) + .collect(Collectors.toList())); + } + List<RecordStatus> skippedRecords = this.jobStatus + .getRecordStatuses(IndexingStatus.SKIP, operationType); + List<RecordStatus> failedRecords = this.jobStatus + .getRecordStatuses(IndexingStatus.FAIL, operationType); + failedRecords.addAll(skippedRecords); + if (!failedRecords.isEmpty()) { + failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage) + .collect(Collectors.toList())); } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java index 0e3c201656b9c61bdcb537c552d16fd50bef9188..86fb43449ea75340340b0ee43de9e85304c79697 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,13 @@ package org.opengroup.osdu.indexer.util; +import com.google.common.base.Strings; import java.util.Collections; import java.util.Map; import java.util.stream.Collectors; - -import javax.servlet.http.HttpServletRequest; import javax.inject.Inject; - -import com.google.common.base.Strings; +import javax.servlet.http.HttpServletRequest; import org.opengroup.osdu.core.common.model.http.DpsHeaders; - import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; import org.opengroup.osdu.core.gcp.util.TraceIdExtractor; import org.springframework.context.annotation.Primary; @@ -38,23 +35,23 @@ import org.springframework.web.context.annotation.RequestScope; @Primary public class DpsHeaderFactoryGcp extends DpsHeaders { - @Inject - public DpsHeaderFactoryGcp(HttpServletRequest request) { + @Inject + public DpsHeaderFactoryGcp(HttpServletRequest request) { - Map<String, String> headers = Collections - .list(request.getHeaderNames()) - .stream() - .collect(Collectors.toMap(h -> h, request::getHeader)); + Map<String, String> headers = Collections + .list(request.getHeaderNames()) + .stream() + .collect(Collectors.toMap(h -> h, request::getHeader)); - String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT); + String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT); - if(!Strings.isNullOrEmpty(traceContext)){ - headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); - } + if (!Strings.isNullOrEmpty(traceContext)) { + headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); + } - this.addFromMap(headers); + this.addFromMap(headers); - // Add Correlation ID if missing - this.addCorrelationIdIfMissing(); - } + // Add Correlation ID if missing + this.addCorrelationIdIfMissing(); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java index b60ef1da6a11f4f21c4206448743ecb05e81c59e..8e53b593ec171e2962ca3b286d328e89a1a7716c 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java @@ -23,9 +23,10 @@ import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.indexer.config.MongoDBConfigProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component @@ -36,16 +37,8 @@ public class MongoClientHandler { private static final String MONGO_PREFIX = "mongodb://"; private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000"; - private MongoClient mongoClient = null; - - @Value("${mongo.db.url:#{null}}") - private String dbUrl; - - @Value("${mongo.db.user:#{null}}") - private String dbUser; - - @Value("${mongo.db.password:#{null}}") - private String dbPassword; + private com.mongodb.client.MongoClient mongoClient = null; + private MongoDBConfigProperties mongoDBConfigProperties; private MongoClient getOrInitMongoClient() throws RuntimeException { if (mongoClient != null) { @@ -54,9 +47,9 @@ public class MongoClientHandler { final String connectionString = String.format("%s%s:%s@%s/?%s", MONGO_PREFIX, - dbUser, - dbPassword, - dbUrl, + mongoDBConfigProperties.getMongoDbUser(), + mongoDBConfigProperties.getMongoDbPassword(), + mongoDBConfigProperties.getMongoDbUrl(), MONGO_OPTIONS); ConnectionString connString = new ConnectionString(connectionString); MongoClientSettings settings = MongoClientSettings.builder() @@ -80,4 +73,8 @@ public class MongoClientHandler { return mongoClient; } + @Autowired + public void setMongoDBConfigProperties(MongoDBConfigProperties mongoDBConfigProperties) { + this.mongoDBConfigProperties = mongoDBConfigProperties; + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java index 2396dd1e55b99ee7853d4e9b94e752be5a559261..1e3c3f244cb37470a91c6a72185a2007c2578698 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,101 +17,108 @@ package org.opengroup.osdu.indexer.util; +import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION; + import com.google.common.base.Strings; +import java.util.Map; import lombok.extern.java.Log; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.Constants; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; -import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import javax.inject.Inject; -import java.util.Map; - -import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION; - - @Log @Component @RequestScope public class RequestInfoImpl implements IRequestInfo { - @Inject - private DpsHeaders dpsHeaders; - - @Inject - private IServiceAccountJwtClient serviceAccountJwtClient; - - @Inject - private TenantInfo tenantInfo; - - @Value("${DEPLOYMENT_ENVIRONMENT}") - private String DEPLOYMENT_ENVIRONMENT; - - private static final String expectedCronHeaderValue = "true"; - - @Override - public DpsHeaders getHeaders() { - - return this.dpsHeaders; - } - - @Override - public String getPartitionId() { - return this.dpsHeaders.getPartitionId(); - } - - @Override - public Map<String, String> getHeadersMap() { - return this.dpsHeaders.getHeaders(); - } - - @Override - public Map<String, String> getHeadersMapWithDwdAuthZ() { - return getHeadersWithDwdAuthZ().getHeaders(); - } - - @Override - public DpsHeaders getHeadersWithDwdAuthZ() { - // Update DpsHeaders so that service account creds are passed down - this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader()); - return this.dpsHeaders; - } - - @Override - public boolean isCronRequest() { - String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(AppEngineHeaders.CRON_SERVICE, null); - return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader); - } - - @Override - public boolean isTaskQueueRequest() { - if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) return false; - - String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME); - return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER); + private final IndexerConfigurationProperties indexerConfigurationProperties; + private final TenantInfo tenantInfo; + private final IServiceAccountJwtClient serviceAccountJwtClient; + private final DpsHeaders dpsHeaders; + + @Autowired + public RequestInfoImpl(IndexerConfigurationProperties indexerConfigurationProperties, + TenantInfo tenantInfo, IServiceAccountJwtClient serviceAccountJwtClient, + DpsHeaders dpsHeaders) { + this.indexerConfigurationProperties = indexerConfigurationProperties; + this.tenantInfo = tenantInfo; + this.serviceAccountJwtClient = serviceAccountJwtClient; + this.dpsHeaders = dpsHeaders; + } + + private static final String expectedCronHeaderValue = "true"; + + @Override + public DpsHeaders getHeaders() { + + return this.dpsHeaders; + } + + @Override + public String getPartitionId() { + return this.dpsHeaders.getPartitionId(); + } + + @Override + public Map<String, String> getHeadersMap() { + return this.dpsHeaders.getHeaders(); + } + + @Override + public Map<String, String> getHeadersMapWithDwdAuthZ() { + return getHeadersWithDwdAuthZ().getHeaders(); + } + + @Override + public DpsHeaders getHeadersWithDwdAuthZ() { + // Update DpsHeaders so that service account creds are passed down + this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader()); + return this.dpsHeaders; + } + + @Override + public boolean isCronRequest() { + String appEngineCronHeader = this.dpsHeaders.getHeaders() + .getOrDefault(AppEngineHeaders.CRON_SERVICE, null); + return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader); + } + + @Override + public boolean isTaskQueueRequest() { + if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) { + return false; } - public String checkOrGetAuthorizationHeader() { - if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) == DeploymentEnvironment.LOCAL) { - String authHeader = this.dpsHeaders.getAuthorization(); - if (Strings.isNullOrEmpty(authHeader)) { - throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty"); - } - String user = this.dpsHeaders.getUserEmail(); - if (Strings.isNullOrEmpty(user)) { - throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", "User header cannot be empty"); - } - return authHeader; - } else { - return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName()); - } + String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME); + return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER); + } + + public String checkOrGetAuthorizationHeader() { + if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment()) + == DeploymentEnvironment.LOCAL) { + String authHeader = this.dpsHeaders.getAuthorization(); + if (Strings.isNullOrEmpty(authHeader)) { + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", + "Authorization token cannot be empty"); + } + String user = this.dpsHeaders.getUserEmail(); + if (Strings.isNullOrEmpty(user)) { + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", + "User header cannot be empty"); + } + return authHeader; + } else { + return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName()); } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java index 6da39c9ed5a45a7f792936e2c7b88bdb8ee8e236..98605a1509c597b4b5f7332560374ff072248d44 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,12 +30,10 @@ import com.google.api.services.iam.v1.model.SignJwtRequest; import com.google.api.services.iam.v1.model.SignJwtResponse; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import java.io.FileInputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.inject.Inject; import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; import org.apache.http.NameValuePair; @@ -55,7 +53,8 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; @@ -63,134 +62,141 @@ import org.springframework.web.context.annotation.RequestScope; @RequestScope public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient { - private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token"; - private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s"; - - private final JsonFactory JSON_FACTORY = new JacksonFactory(); - - private Iam iam; - - @Inject - private ITenantFactory tenantInfoServiceProvider; - @Inject - private IJwtCache cacheService; - @Inject - private JaxRsDpsLog log; - @Inject - private DpsHeaders dpsHeaders; - - @Value("${GOOGLE_AUDIENCES}") - public String GOOGLE_AUDIENCES; - - @Value("${INDEXER_HOST}") - public String INDEXER_HOST; - - public String getIdToken(String tenantName) { - this.log.info("Tenant name received for auth token is: " + tenantName); - TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName); - if (tenant == null) { - this.log.error("Invalid tenant name receiving from pubsub"); - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub"); - } - try { - - IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); - // Add the user to DpsHeaders directly - this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); - - if (!IdToken.refreshToken(cachedToken)) { - return cachedToken.getTokenValue(); - } - - // Getting signed JWT - Map<String, Object> signJwtPayload = this.getJWTCreationPayload(tenant); - - SignJwtRequest signJwtRequest = new SignJwtRequest(); - signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload)); - - String serviceAccountName = String - .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount()); - - Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts() - .signJwt(serviceAccountName, signJwtRequest); - SignJwtResponse signJwtResponse = signJwt.execute(); - String signedJwt = signJwtResponse.getSignedJwt(); - - // Getting id token - List<NameValuePair> postParameters = new ArrayList<>(); - postParameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); - postParameters.add(new BasicNameValuePair("assertion", signedJwt)); - - HttpPost post = new HttpPost(JWT_AUDIENCE); - post.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType()); - post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); - - try (CloseableHttpClient httpclient = HttpClients.createDefault(); - CloseableHttpResponse httpResponse = httpclient.execute(post)) { - JsonObject jsonContent = new JsonParser().parse(EntityUtils.toString(httpResponse.getEntity())) - .getAsJsonObject(); - - if (!jsonContent.has("id_token")) { - log.error(String.format("Google IAM response: %s", jsonContent.toString())); - throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", - "The user is not authorized to perform this action"); - } - - String token = jsonContent.get("id_token").getAsString(); - IdToken idToken = IdToken.builder().tokenValue(token) - .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build(); - - this.cacheService.put(tenant.getServiceAccount(), idToken); - - return token; - } - } catch (JWTDecodeException e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", - "Invalid token, error decoding", e); - } catch (AppException e) { - throw e; - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", - e); - } - } - - public Iam getIam() throws Exception { - - if (this.iam == null) { - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - - // Authenticate using Google Application Default Credentials. - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - if (credential.createScopedRequired()) { - List<String> scopes = new ArrayList<>(); - // Enable full Cloud Platform scope. - scopes.add(IamScopes.CLOUD_PLATFORM); - credential = credential.createScoped(scopes); - } - - // Create IAM API object associated with the authenticated transport. - this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(INDEXER_HOST) - .build(); - } - - return this.iam; - } - - private Map<String, Object> getJWTCreationPayload(TenantInfo tenantInfo) { - - Map<String, Object> payload = new HashMap<>(); - String googleAudience = GOOGLE_AUDIENCES; - if (googleAudience.contains(",")) { - googleAudience = googleAudience.split(",")[0]; - } - payload.put("target_audience", googleAudience); - payload.put("exp", System.currentTimeMillis() / 1000 + 3600); - payload.put("iat", System.currentTimeMillis() / 1000); - payload.put("iss", tenantInfo.getServiceAccount()); - payload.put("aud", JWT_AUDIENCE); - - return payload; - } + private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token"; + private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s"; + + private final JsonFactory JSON_FACTORY = new JacksonFactory(); + private final IndexerConfigurationProperties indexerConfigurationProperties; + private final ITenantFactory tenantInfoServiceProvider; + private final IJwtCache cacheService; + private final JaxRsDpsLog log; + private final DpsHeaders dpsHeaders; + + private Iam iam; + + @Autowired + public ServiceAccountJwtGcpClientImpl( + IndexerConfigurationProperties indexerConfigurationProperties, + ITenantFactory tenantInfoServiceProvider, + IJwtCache cacheService, JaxRsDpsLog log, DpsHeaders dpsHeaders) { + this.indexerConfigurationProperties = indexerConfigurationProperties; + this.tenantInfoServiceProvider = tenantInfoServiceProvider; + this.cacheService = cacheService; + this.log = log; + this.dpsHeaders = dpsHeaders; + } + + public String getIdToken(String tenantName) { + this.log.info("Tenant name received for auth token is: " + tenantName); + TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName); + if (tenant == null) { + this.log.error("Invalid tenant name receiving from pubsub"); + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", + "Invalid tenant Name from pubsub"); + } + try { + + IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); + // Add the user to DpsHeaders directly + this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); + + if (!IdToken.refreshToken(cachedToken)) { + return cachedToken.getTokenValue(); + } + + // Getting signed JWT + Map<String, Object> signJwtPayload = this.getJWTCreationPayload(tenant); + + SignJwtRequest signJwtRequest = new SignJwtRequest(); + signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload)); + + String serviceAccountName = String + .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount()); + + Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts() + .signJwt(serviceAccountName, signJwtRequest); + SignJwtResponse signJwtResponse = signJwt.execute(); + String signedJwt = signJwtResponse.getSignedJwt(); + + // Getting id token + List<NameValuePair> postParameters = new ArrayList<>(); + postParameters + .add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); + postParameters.add(new BasicNameValuePair("assertion", signedJwt)); + + HttpPost post = new HttpPost(JWT_AUDIENCE); + post.setHeader(HttpHeaders.CONTENT_TYPE, + ContentType.APPLICATION_FORM_URLENCODED.getMimeType()); + post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); + + try (CloseableHttpClient httpclient = HttpClients.createDefault(); + CloseableHttpResponse httpResponse = httpclient.execute(post)) { + JsonObject jsonContent = new JsonParser() + .parse(EntityUtils.toString(httpResponse.getEntity())) + .getAsJsonObject(); + + if (!jsonContent.has("id_token")) { + log.error(String.format("Google IAM response: %s", jsonContent.toString())); + throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", + "The user is not authorized to perform this action"); + } + + String token = jsonContent.get("id_token").getAsString(); + IdToken idToken = IdToken.builder().tokenValue(token) + .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build(); + + this.cacheService.put(tenant.getServiceAccount(), idToken); + + return token; + } + } catch (JWTDecodeException e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", + "Invalid token, error decoding", e); + } catch (AppException e) { + throw e; + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", + "Error generating token", + e); + } + } + + public Iam getIam() throws Exception { + + if (this.iam == null) { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + + // Authenticate using Google Application Default Credentials. + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + if (credential.createScopedRequired()) { + List<String> scopes = new ArrayList<>(); + // Enable full Cloud Platform scope. + scopes.add(IamScopes.CLOUD_PLATFORM); + credential = credential.createScoped(scopes); + } + + // Create IAM API object associated with the authenticated transport. + this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(indexerConfigurationProperties.getIndexerHost()) + .build(); + } + + return this.iam; + } + + private Map<String, Object> getJWTCreationPayload(TenantInfo tenantInfo) { + + Map<String, Object> payload = new HashMap<>(); + String googleAudience = indexerConfigurationProperties.getGoogleAudiences(); + if (googleAudience.contains(",")) { + googleAudience = googleAudience.split(",")[0]; + } + payload.put("target_audience", googleAudience); + payload.put("exp", System.currentTimeMillis() / 1000 + 3600); + payload.put("iat", System.currentTimeMillis() / 1000); + payload.put("iss", tenantInfo.getServiceAccount()); + payload.put("aud", JWT_AUDIENCE); + + return payload; + } } diff --git a/provider/indexer-reference/src/main/resources/application-dev.properties b/provider/indexer-reference/src/main/resources/application-dev.properties index 09ea204df6cd8cb54bceb57f3b3b3e2d44613b34..ea75c38d4866a1a690158a7e7efbd184c68ad58c 100644 --- a/provider/indexer-reference/src/main/resources/application-dev.properties +++ b/provider/indexer-reference/src/main/resources/application-dev.properties @@ -17,24 +17,28 @@ GOOGLE_CLOUD_PROJECT=nice-etching-277309 -INDEXER_HOST=os-indexer-dot-nice-etching-277309.uc.r.appspot.com -STORAGE_HOSTNAME=os-storage-dot-nice-etching-277309.uc.r.appspot.com +mongo-db-url=localhost:27017 +mongo-db-user= +mongo-db-password= -STORAGE_SCHEMA_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas -STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records -STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch -STORAGE_RECORDS_BATCH_SIZE=20 +indexer-host=os-indexer-dot-nice-etching-277309.uc.r.appspot.com +storage-hostname=os-storage-dot-nice-etching-277309.uc.r.appspot.com -INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue +storage-schema-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas +storage-query-record-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records +storage-query-record-for-conversion-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch +storage-records-batch-size=20 -AUTHORIZE_API=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 -LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 -CRS_API=example.com +indexer-queue-host=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue + +authorize-api=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 +legaltag-api=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 +crs-api=example.com ## use below values for gcp: opendes -REDIS_GROUP_HOST=127.0.0.1 -REDIS_SEARCH_HOST=127.0.0.1 +redis-group-host=127.0.0.1 +redis-search-host=127.0.0.1 -GOOGLE_AUDIENCES=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com +google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com -mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672 +mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672 diff --git a/provider/indexer-reference/src/main/resources/application.properties b/provider/indexer-reference/src/main/resources/application.properties index 4d62d4abed0705ab986b75574fbbb0ec3e22f418..8690c57a906c0aae8cf6ecc5b0c282be392bbafb 100644 --- a/provider/indexer-reference/src/main/resources/application.properties +++ b/provider/indexer-reference/src/main/resources/application.properties @@ -1,6 +1,6 @@ # -# Copyright 2020 Google LLC -# Copyright 2020 EPAM Systems, Inc +# Copyright 2021 Google LLC +# Copyright 2021 EPAM Systems, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,56 +23,54 @@ server.port=8080 JAVA_OPTS=-Xms3072m -Xmx3072m JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 -DEPLOYMENT_ENVIRONMENT=CLOUD +environment=CLOUD -REDIS_GROUP_PORT=6379 -REDIS_SEARCH_PORT=6379 -DEFAULT_DATA_COUNTRY=US +#default cache settings +schema-cache-expiration=60 +index-cache-expiration=60 +elastic-cache-expiration=1440 +cursor-cache-expiration=60 +# kinds cache expiration 2*24*60 +kinds-cache-expiration=2880 +# attributes cache expiration 2*24*60 +attributes-cache-expiration=2880 -#Default Cache Settings -SCHEMA_CACHE_EXPIRATION=60 -INDEX_CACHE_EXPIRATION=60 -ELASTIC_CACHE_EXPIRATION=1440 -CURSOR_CACHE_EXPIRATION=60 -# Kinds Cache expiration 2*24*60 -KINDS_CACHE_EXPIRATION=2880 -# Attributes Cache expiration 2*24*60 -ATTRIBUTES_CACHE_EXPIRATION=2880 +kinds-redis-database=1 +cron-index-cleanup-threshold-days=3 +cron-empty-index-cleanup-threshold-days=7 -KINDS_REDIS_DATABASE=1 -CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3 -CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7 +gae-service=indexer +key-ring=csqp +kms-key=searchservice -GAE_SERVICE=indexer -KEY_RING=csqp -KMS_KEY=searchService +elastic-datastore-kind=searchsettings +elastic-datastore-id=indexer-service -ELASTIC_DATASTORE_KIND=SearchSettings -ELASTIC_DATASTORE_ID=indexer-service +mongo-db-url=localhost:27017 +mongo-db-user= +mongo-db-password= -mongo.db.url=localhost:27017 -mongo.db.user= -mongo.db.password= +indexer-host=os-indexer-dot-nice-etching-277309.uc.r.appspot.com +storage-hostname=os-storage-dot-nice-etching-277309.uc.r.appspot.com -INDEXER_HOST=os-indexer-dot-nice-etching-277309.uc.r.appspot.com -STORAGE_HOSTNAME=os-storage-dot-nice-etching-277309.uc.r.appspot.com +storage-schema-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas +storage-query-record-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records +storage-query-record-for-conversion-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch +storage-records-batch-size=20 -STORAGE_SCHEMA_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas -STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records -STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch -STORAGE_RECORDS_BATCH_SIZE=20 +indexer-queue-host=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue -INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue - -AUTHORIZE_API=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 -LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 -CRS_API=example.com +authorize-api=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 +legaltag-api=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 +crs-api=example.com ## use below values for gcp: opendes -REDIS_GROUP_HOST=127.0.0.1 -REDIS_SEARCH_HOST=127.0.0.1 +redis-group-host=127.0.0.1 +redis-search-host=127.0.0.1 + +google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com + +mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672 -GOOGLE_AUDIENCES=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com -mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672 diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java index 7ebfab8dba438872ec892048d02cea3dd0d9e1a7..9ec5fb7a30044f7f105c9a97229cb37322f218b9 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,12 @@ package org.opengroup.osdu.indexer.middleware; +import java.io.IOException; +import java.util.Collections; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -25,46 +31,51 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import javax.servlet.FilterChain; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import java.io.IOException; -import java.util.Collections; - @RunWith(MockitoJUnitRunner.class) public class IndexFilterTest { - @InjectMocks - private IndexFilter indexFilter; + @InjectMocks + private IndexFilter indexFilter; - @Mock - private DpsHeaders dpsHeaders; + @Mock + private DpsHeaders dpsHeaders; - @Test - public void shouldSetCorrectResponseHeaders() throws IOException, ServletException { - HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class); - HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class); - FilterChain filterChain = Mockito.mock(FilterChain.class); - Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com"); - Mockito.when(httpServletRequest.getMethod()).thenReturn("POST"); - Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value"); + @Test + public void shouldSetCorrectResponseHeaders() throws IOException, ServletException { + HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class); + HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class); + FilterChain filterChain = Mockito.mock(FilterChain.class); + Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com"); + Mockito.when(httpServletRequest.getMethod()).thenReturn("POST"); + Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value"); - indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain); + indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain); - Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString()); - Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString()); - Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString()); - Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString()); - Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString()); - Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString()); - Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString()); - Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString()); - Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString()); - Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString()); - Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString()); - Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value"); - Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse); - } + Mockito.verify(httpServletResponse) + .addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections + .singletonList( + "origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey") + .toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", + Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", + Collections.singletonList("true").toString()); + Mockito.verify(httpServletResponse) + .addHeader("X-Frame-Options", Collections.singletonList("DENY").toString()); + Mockito.verify(httpServletResponse) + .addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString()); + Mockito.verify(httpServletResponse) + .addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString()); + Mockito.verify(httpServletResponse).addHeader("Cache-Control", + Collections.singletonList("no-cache, no-store, must-revalidate").toString()); + Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", + Collections.singletonList("default-src 'self'").toString()); + Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", + Collections.singletonList("max-age=31536000; includeSubDomains").toString()); + Mockito.verify(httpServletResponse) + .addHeader("Expires", Collections.singletonList("0").toString()); + Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value"); + Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse); + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java index 9633d12896795fff32cab3d22218ca4123ade815..e1c53edb8ab313727f54a905a3494a74cb3f6961 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,24 @@ package org.opengroup.osdu.indexer.service; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import org.elasticsearch.client.RestHighLevelClient; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.IndexInfo; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; @@ -34,107 +42,111 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; -import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import static org.mockito.Mockito.*; - @RunWith(SpringRunner.class) @PrepareForTest({RestHighLevelClient.class}) public class CronServiceImplTest { - @Mock - private RestHighLevelClient restHighLevelClient; - @Mock - private IndicesService indicesService; - @Mock - private ElasticClientHandler elasticClientHandler; - @Mock - private IndexerConfigurationProperties configurationProperties; - @Mock - private IRequestInfo requestInfo; - @Mock - private JaxRsDpsLog log; - @InjectMocks - private CronServiceImpl sut; - - @InjectMocks - private DpsHeaders dpsHeaders; - - @Before - public void setup() { - - when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders); - - when(configurationProperties.getCronIndexCleanupThresholdDays()).thenReturn(3); - when(configurationProperties.getCronEmptyIndexCleanupThresholdDays()).thenReturn(3); - } - - @Test - public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception { - final String indexPattern = "tenant1-index-*"; - - IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())).build(); - - when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)).thenReturn(Lists.newArrayList(info)); - - this.sut.cleanupIndices(indexPattern); - - verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); - verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern); - } - - @Test(expected = IOException.class) - public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception { - IOException exception = new IOException("blah"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")).thenThrow(exception); - - this.sut.cleanupIndices("tenant1-test-*"); - - verify(this.indicesService, times(0)).deleteIndex(any(), any()); - } - - @Test - public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception { - IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build(); - - when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info)); - - this.sut.cleanupEmptyStaleIndices(); - - verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null); - verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); - } - - @Test - public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception { - IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build(); - - when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info)); - - this.sut.cleanupEmptyStaleIndices(); - - verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); - verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); - } - - @Test(expected = IOException.class) - public void run_cleanup_when_backend_throws_exception() throws Exception { - IOException exception = new IOException("blah"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception); - - this.sut.cleanupEmptyStaleIndices(); - - verify(this.indicesService, times(0)).deleteIndex(any(), any()); - } + @Mock + private RestHighLevelClient restHighLevelClient; + @Mock + private IndicesService indicesService; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private IndexerConfigurationProperties configurationProperties; + @Mock + private IRequestInfo requestInfo; + @Mock + private JaxRsDpsLog log; + @InjectMocks + private CronServiceImpl sut; + + @InjectMocks + private DpsHeaders dpsHeaders; + + @Before + public void setup() { + + when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders); + + when(configurationProperties.getCronIndexCleanupThresholdDays()).thenReturn(3); + when(configurationProperties.getCronEmptyIndexCleanupThresholdDays()).thenReturn(3); + } + + @Test + public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception { + final String indexPattern = "tenant1-index-*"; + + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10") + .creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())) + .build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)) + .thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupIndices(indexPattern); + + verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern); + } + + @Test(expected = IOException.class) + public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception { + IOException exception = new IOException("blah"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")) + .thenThrow(exception); + + this.sut.cleanupIndices("tenant1-test-*"); + + verify(this.indicesService, times(0)).deleteIndex(any(), any()); + } + + @Test + public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception { + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10") + .creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())) + .build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)) + .thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); + } + + @Test + public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception { + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0") + .creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())) + .build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)) + .thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); + } + + @Test(expected = IOException.class) + public void run_cleanup_when_backend_throws_exception() throws Exception { + IOException exception = new IOException("blah"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(0)).deleteIndex(any(), any()); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java index f9645a5096b1d7fa639169fc42e5a4704950e8cc..b09fbc40ddb38ffd327be67375a4b7718d687722 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,101 +17,102 @@ package org.opengroup.osdu.indexer.service; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.when; + import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService; import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; -import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.test.context.junit4.SpringRunner; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.when; - @RunWith(SpringRunner.class) public class ElasticSettingServiceTest { - @Mock - private ITenantInfoService tenantInfoService; - @Mock - private IElasticRepository elasticRepository; - @Mock - private IElasticCredentialsCache elasticCredentialCache; - @Mock - private IndexerConfigurationProperties configurationProperties; - @Mock - private TenantInfo tenantInfo; - @InjectMocks - private ElasticSettingServiceImpl sut; - @Mock - private ClusterSettings clusterSettings; - @Mock - private DpsHeaders headersInfo; + @Mock + private ITenantInfoService tenantInfoService; + @Mock + private IElasticRepository elasticRepository; + @Mock + private IElasticCredentialsCache elasticCredentialCache; + @Mock + private IndexerConfigurationProperties configurationProperties; + @Mock + private TenantInfo tenantInfo; + @InjectMocks + private ElasticSettingServiceImpl sut; + @Mock + private ClusterSettings clusterSettings; + @Mock + private DpsHeaders headersInfo; - @Mock - private JaxRsDpsLog log; + @Mock + private JaxRsDpsLog log; - public String GAE_SERVICE = "indexer"; + public String GAE_SERVICE = "indexer"; - private final String host = "db5c51c1.us-central1.gcp.cloud.es.io"; - private final int port = 9243; - private final String credentials = "name:password"; + private final String host = "db5c51c1.us-central1.gcp.cloud.es.io"; + private final int port = 9243; + private final String credentials = "name:password"; - String cacheKey = ""; + String cacheKey = ""; - @Before - public void setup() { - when(tenantInfo.getName()).thenReturn("tenant1"); - when(this.headersInfo.getPartitionId()).thenReturn("tenant1"); - when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); - when(configurationProperties.getGaeService()).thenReturn("indexer"); - clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build(); - cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName()); - } + @Before + public void setup() { + when(tenantInfo.getName()).thenReturn("tenant1"); + when(this.headersInfo.getPartitionId()).thenReturn("tenant1"); + when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); + when(configurationProperties.getGaeService()).thenReturn("indexer"); + clusterSettings = ClusterSettings.builder().host(host).port(port) + .userNameAndPassword(credentials).build(); + cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName()); + } - @Test - public void should_getValid_clusterSettings_fromCache() { + @Test + public void should_getValid_clusterSettings_fromCache() { - when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); + when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); - ClusterSettings response = this.sut.getElasticClusterInformation(); - assertNotNull(response); - assertEquals(response.getHost(), host); - assertEquals(response.getPort(), port); - assertEquals(response.getUserNameAndPassword(), credentials); - } + ClusterSettings response = this.sut.getElasticClusterInformation(); + assertNotNull(response); + assertEquals(response.getHost(), host); + assertEquals(response.getPort(), port); + assertEquals(response.getUserNameAndPassword(), credentials); + } - @Test - public void should_getValid_clusterSettings_fromCosmosDB() { + @Test + public void should_getValid_clusterSettings_fromCosmosDB() { - when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); + when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); - when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings); + when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings); - ClusterSettings response = this.sut.getElasticClusterInformation(); - assertNotNull(response); - assertEquals(response.getHost(), host); - assertEquals(response.getPort(), port); - assertEquals(response.getUserNameAndPassword(), credentials); - } + ClusterSettings response = this.sut.getElasticClusterInformation(); + assertNotNull(response); + assertEquals(response.getHost(), host); + assertEquals(response.getPort(), port); + assertEquals(response.getUserNameAndPassword(), credentials); + } - @Test(expected = AppException.class) - public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException { + @Test(expected = AppException.class) + public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException { - when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null); + when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null); - this.sut.getElasticClusterInformation(); + this.sut.getElasticClusterInformation(); - } + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java index 9ef4a6adbe21c802885ee6b4caf93840b2728c1e..6a438da28b492752abe329f329c0cbd85ec09977 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,17 @@ package org.opengroup.osdu.indexer.service; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; + import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; - +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; import org.apache.http.HttpEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; @@ -34,160 +42,162 @@ import org.junit.runner.RunWith; import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.search.ClusterSettings; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.indexer.logging.AuditLogger; -import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.logging.AuditLogger; import org.opengroup.osdu.indexer.util.ElasticClientHandler; -import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.powermock.api.mockito.PowerMockito.when; - @RunWith(SpringRunner.class) -@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, EntityUtils.class}) +@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, + EntityUtils.class}) public class IndexCopyServiceImplTest { - private final String correlationId = UUID.randomUUID().toString(); - - @Mock - private HttpEntity httpEntity; - @Mock - private HttpEntity httpEntityRequest; - @Mock - private IRequestInfo requestInfo; - @Mock - private DpsHeaders headersInfo; - @Mock - private RestClient restClient; - @Mock - private RestHighLevelClient restHighLevelClient; - @Mock - private IndicesService indicesService; - @Mock - private IndexerMappingService mappingService; - @Mock - private ElasticClientHandler elasticClientHandler; - @Mock - private ElasticIndexNameResolver elasticIndexNameResolver; - @Mock - private Response response; - @Mock - private IElasticSettingService elasticSettingService; - @Mock - private AuditLogger auditLogger; - @Mock - private Map<String, String> httpHeaders; - @InjectMocks - private IndexCopyServiceImpl sut; - - private ClusterSettings commonCluster; - - private Map<String, Object> correctMap; - - @Before - public void setup() { - - commonCluster = ClusterSettings.builder().host("commonhost").port(8080).userNameAndPassword("username:pwd").build(); - - httpHeaders = new HashMap<>(); - httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); - httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); - when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); - when(response.getEntity()).thenReturn(httpEntity); - - Type mapType = new TypeToken<Map<String, Object>>() {}.getType(); - String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}"; - correctMap = new Gson().fromJson(afterFormat, mapType); - - restHighLevelClient = mock(RestHighLevelClient.class); + private final String correlationId = UUID.randomUUID().toString(); + + @Mock + private HttpEntity httpEntity; + @Mock + private HttpEntity httpEntityRequest; + @Mock + private IRequestInfo requestInfo; + @Mock + private DpsHeaders headersInfo; + @Mock + private RestClient restClient; + @Mock + private RestHighLevelClient restHighLevelClient; + @Mock + private IndicesService indicesService; + @Mock + private IndexerMappingService mappingService; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private ElasticIndexNameResolver elasticIndexNameResolver; + @Mock + private Response response; + @Mock + private IElasticSettingService elasticSettingService; + @Mock + private AuditLogger auditLogger; + @Mock + private Map<String, String> httpHeaders; + @InjectMocks + private IndexCopyServiceImpl sut; + + private ClusterSettings commonCluster; + + private Map<String, Object> correctMap; + + @Before + public void setup() { + + commonCluster = ClusterSettings.builder().host("commonhost").port(8080) + .userNameAndPassword("username:pwd").build(); + + httpHeaders = new HashMap<>(); + httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); + httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); + when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); + when(response.getEntity()).thenReturn(httpEntity); + + Type mapType = new TypeToken<Map<String, Object>>() { + }.getType(); + String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}"; + correctMap = new Gson().fromJson(afterFormat, mapType); + + restHighLevelClient = mock(RestHighLevelClient.class); + + } + + @Test(expected = IOException.class) + public void should_throwIOException_when_indexMappingNotFound() throws Exception { + IOException exception = new IOException( + "Fail to get mapping for the given index from common cluster."); + + when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenThrow(exception); + + this.sut.copyIndex("common:metadata:entity:1.0.0"); + } + + @Test(expected = IllegalArgumentException.class) + public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() { + try { + this.sut.copyIndex(null); + } catch (IOException e) { + fail("Should not throw IOException but illegalArgumentException."); } - - @Test(expected = IOException.class) - public void should_throwIOException_when_indexMappingNotFound() throws Exception { - IOException exception = new IOException("Fail to get mapping for the given index from common cluster."); - - when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(exception); - - this.sut.copyIndex("common:metadata:entity:1.0.0"); + } + + @Test + public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() { + String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}"; + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + try { + when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenReturn(mappingJson); + Map<String, Object> resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test"); + Assert.assertEquals(resultMap, correctMap); + } catch (Exception ignored) { } + } - @Test(expected = IllegalArgumentException.class) - public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() { - try { - this.sut.copyIndex(null); - } catch (IOException e) { - fail("Should not throw IOException but illegalArgumentException."); - } - } + @Test + public void should_returnClusterInfo_getCommonClusterInformationtest() { + try { + String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; - @Test - public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() { - String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}"; - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - try { - when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(mappingJson); - Map<String, Object> resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test"); - Assert.assertEquals(resultMap, correctMap); - } catch (Exception ignored) { - } - } + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - @Test - public void should_returnClusterInfo_getCommonClusterInformationtest() { - try { - String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; + when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - - when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster); - - String[] resultCommonCluster = this.sut.getCommonClusterInformation(); - Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]); - Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]); - Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]); - } catch (IOException ignored) { - fail("Should not throw this exception " + ignored.getMessage()); - } + String[] resultCommonCluster = this.sut.getCommonClusterInformation(); + Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]); + Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]); + Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]); + } catch (IOException ignored) { + fail("Should not throw this exception " + ignored.getMessage()); } - - @Test(expected = AppException.class) - public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() { - try { - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); - this.sut.createIndexInTenantCluster("test", "test", "test", correctMap); - } catch (IOException ignored) { - fail("Should not throw this exception " + ignored.getMessage()); - } + } + + @Test(expected = AppException.class) + public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() { + try { + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + when(indicesService + .createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); + this.sut.createIndexInTenantCluster("test", "test", "test", correctMap); + } catch (IOException ignored) { + fail("Should not throw this exception " + ignored.getMessage()); } - - @Ignore - public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() { - //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest(). - try { - String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; - Request request = new Request("POST", "/_reindex?wait_for_completion=false"); - request.setEntity(httpEntityRequest); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); - when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient); - when(restClient.performRequest(request)).thenReturn(response); - when(response.getEntity()).thenReturn(httpEntity); - Assert.assertEquals(httpEntity, this.sut.reindexInTenantCluster("test", "test", correctCommonCluster)); - } catch (IOException ignored) { - } + } + + @Ignore + public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() { + //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest(). + try { + String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; + Request request = new Request("POST", "/_reindex?wait_for_completion=false"); + request.setEntity(httpEntityRequest); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + when(indicesService + .createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); + when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient); + when(restClient.performRequest(request)).thenReturn(response); + when(response.getEntity()).thenReturn(httpEntity); + Assert.assertEquals(httpEntity, + this.sut.reindexInTenantCluster("test", "test", correctCommonCluster)); + } catch (IOException ignored) { } + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java deleted file mode 100644 index b4eca3a4482df93f2bdea68818cd7e9fb9237359..0000000000000000000000000000000000000000 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.service; - -import com.google.gson.Gson; -import org.apache.http.StatusLine; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; -import org.elasticsearch.action.bulk.BulkItemResponse.Failure; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.*; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.indexer.IndexSchema; -import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.opengroup.osdu.indexer.util.ElasticClientHandler; -import org.opengroup.osdu.indexer.util.TypeMapper; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.springframework.test.context.junit4.SpringRunner; - -import java.io.IOException; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.when; - -@Ignore -@RunWith(SpringRunner.class) -@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class}) -public class IndexerMappingServiceTest { - - private final String kind = "tenant:test:test:1.0.0"; - private final String index = "tenant-test-test-1.0.0"; - private final String type = "test"; - private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"analyzer\":\"de_indexer_analyzer\",\"search_analyzer\":\"de_search_analyzer\"},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}"; - - @Mock - private IndexerConfigurationProperties configurationProperties; - @Mock - private RestClient restClient; - @Mock - private Response response; - @Mock - private StatusLine statusLine; - - @InjectMocks - private IndexerMappingServiceImpl sut; - - @Mock - private ElasticClientHandler elasticClientHandler; - - @InjectMocks - private RestHighLevelClient restHighLevelClient; - - @InjectMocks - private IndexSchema indexSchema; - @InjectMocks - private IndicesClient indicesClient; - - @InjectMocks - private AcknowledgedResponse mappingResponse; - - @Before - public void setup() throws IOException { - initMocks(this); - when(configurationProperties.isPreDemo()).thenReturn(true); - Map<String, String> dataMapping = new HashMap<>(); - dataMapping.put("Location", "geo_point"); - dataMapping.put("Msg", "text"); - Map<String, Object> metaMapping = new HashMap<>(); - metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); - metaMapping.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL)); - this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping) - .build(); - - this.indicesClient = PowerMockito.mock(IndicesClient.class); - this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class); - - when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient); - when(this.restClient.performRequest(any())).thenReturn(response); - when(this.response.getStatusLine()).thenReturn(statusLine); - when(this.statusLine.getStatusCode()).thenReturn(200); - } - - @Test - public void should_returnValidMapping_givenFalseMerge_createMappingTest() { - try { - String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false); - assertEquals(mappingValid, mapping); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test - public void should_returnValidMapping_givenTrueMerge_createMappingTest() { - try { - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - - String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true); - assertEquals(this.mappingValid, mapping); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test - public void should_returnValidMapping_givenExistType_createMappingTest() { - try { - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - - IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl()); - doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any()); - String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true); - assertEquals(this.mappingValid, mapping); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test - public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception { - try { - Set<String> indices = new HashSet<String>(); - indices.add("indices 1"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - - this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field"); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test(expected = AppException.class) - public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception { - try { - Set<String> indices = new HashSet<String>(); - indices.add("invalid 1"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); - } catch (Exception e) { - throw e; - } - } - - @Test(expected = AppException.class) - public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception { - try { - Set<String> indices = new HashSet<String>(); - indices.add("indices 1"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid"); - } catch (Exception e) { - throw e; - } - } - - @Test(expected = AppException.class) - public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception { - try { - - Set<String> indices = new HashSet<String>(); - indices.add("indices 1"); - indices.add("indices Invalid"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException("")); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); - } catch (AppException e) { - throw e; - } - } - - @Test(expected = AppException.class) - public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() { - try { - Set<String> indices = new HashSet<String>(); - indices.add("indices 1"); - indices.add("indices Invalid"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); - when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException("")); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); - } catch (AppException e) { - throw e; - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - - @Test - public void should_returnDocumentMapping_givenValidIndexSchema() { - - try { - Map<String, Object> documentMapping = this.sut.getIndexMappingFromRecordSchema(this.indexSchema); - String documentMappingJson = new Gson().toJson(documentMapping); - assertEquals(this.mappingValid, documentMappingJson); - - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } -} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java index 27b5f786cf840b6bafee1a6ddbde73565f75526b..4ce6eb4e8af4a45eddbc163baad9bb9cd6c2ef12 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,27 @@ package org.opengroup.osdu.indexer.service; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyBoolean; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; import org.apache.http.HttpStatus; import org.elasticsearch.client.RestHighLevelClient; import org.junit.Assert; @@ -25,380 +46,400 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.indexer.OperationType; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; -import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.RequestStatus; +import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.indexer.OperationType; +import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.opengroup.osdu.indexer.service.impl.SchemaProviderImpl; import org.opengroup.osdu.indexer.util.ElasticClientHandler; -import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.mock; -import static org.powermock.api.mockito.PowerMockito.when; - @RunWith(SpringRunner.class) @PrepareForTest({RestHighLevelClient.class}) public class IndexerSchemaServiceTest { - private final String kind = "tenant:test:test:1.0.0"; - private final String emptySchema = null; - private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}"; - - @Mock - private JaxRsDpsLog log; - @Mock - private SchemaProviderImpl schemaProvider; - @Mock - private ElasticClientHandler elasticClientHandler; - @Mock - private ElasticIndexNameResolver elasticIndexNameResolver; - @Mock - private IndexerMappingService mappingService; - @Mock - private IndicesService indicesService; - @Mock - private ISchemaCache schemaCache; - @InjectMocks - private IndexSchemaServiceImpl sut; - - @Before - public void setup() { - initMocks(this); - RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - } - - @Test - public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception { - when(schemaProvider.getSchema(any())).thenReturn(emptySchema); - - IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); - - Assert.assertNotNull(indexSchema); - } - - @Test - public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception { - when(schemaProvider.getSchema(any())).thenReturn(someSchema); - - IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); - - Assert.assertEquals(kind, indexSchema.getKind()); - } - - @Test - public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception { - when(schemaProvider.getSchema(any())).thenReturn(someSchema); - when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema); - - IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); - - Assert.assertEquals(kind, indexSchema.getKind()); - } - - @Test - public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() { - try { - String invalidSchema = "{}}"; - when(schemaProvider.getSchema(any())).thenReturn(invalidSchema); - - this.sut.getIndexerInputSchema(kind, false); - fail("Should throw exception"); - } catch (AppException e) { - Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode()); - Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage()); - } catch (Exception e) { - fail("Should not throw exception" + e.getMessage()); - } - } - - @Test - public void should_return_basic_schema_when_storage_returns_no_schema() { - IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false); - - assertNotNull(returnedSchema.getDataSchema()); - assertNotNull(returnedSchema); - assertEquals(kind, returnedSchema.getKind()); - } - - @Test - public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"startDate\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"endDate\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"type \"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"itemguid\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - Map<String, OperationType> schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); - verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); - verifyNoMoreInteractions(this.mappingService); - } - - @Test - public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"startDate\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - Map<String, OperationType> schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any()); - verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean()); - verifyNoMoreInteractions(this.mappingService); - } - - @Test - public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-")); - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - Map<String, OperationType> schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, "")); - - try { - this.sut.processSchemaMessages(schemaMessages); - } catch (AppException e) { - assertEquals(RequestStatus.SCHEMA_CONFLICT, e.getError().getCode()); - assertEquals("error creating or merging index mapping", e.getError().getMessage()); - assertEquals(reason, e.getError().getReason()); - } catch (Exception e) { - fail("Should not throw this exception " + e.getMessage()); - } - } - - @Test - public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-")); - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - Map<String, OperationType> schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah")); - - try { - this.sut.processSchemaMessages(schemaMessages); - } catch (AppException e) { - assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode()); - assertEquals("blah", e.getError().getMessage()); - assertEquals(reason, e.getError().getReason()); - } catch (Exception e) { - fail("Should not throw this exception " + e.getMessage()); - } - } - - @Test - public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"" + - "}"; - Map<String, OperationType> schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0")); + private final String kind = "tenant:test:test:1.0.0"; + private final String emptySchema = null; + private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}"; + + @Mock + private JaxRsDpsLog log; + @Mock + private SchemaProviderImpl schemaProvider; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private ElasticIndexNameResolver elasticIndexNameResolver; + @Mock + private IndexerMappingService mappingService; + @Mock + private IndicesService indicesService; + @Mock + private ISchemaCache schemaCache; + @InjectMocks + private IndexSchemaServiceImpl sut; + + @Before + public void setup() { + initMocks(this); + RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + } + + @Test + public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() + throws Exception { + when(schemaProvider.getSchema(any())).thenReturn(emptySchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertNotNull(indexSchema); + } + + @Test + public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() + throws Exception { + when(schemaProvider.getSchema(any())).thenReturn(someSchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertEquals(kind, indexSchema.getKind()); + } + + @Test + public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() + throws Exception { + when(schemaProvider.getSchema(any())).thenReturn(someSchema); + when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertEquals(kind, indexSchema.getKind()); + } + + @Test + public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() { + try { + String invalidSchema = "{}}"; + when(schemaProvider.getSchema(any())).thenReturn(invalidSchema); + + this.sut.getIndexerInputSchema(kind, false); + fail("Should throw exception"); + } catch (AppException e) { + Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode()); + Assert.assertEquals("An error has occurred while normalizing the schema.", + e.getError().getMessage()); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); } - - @Test - public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - Map<String, OperationType> schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.purge_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaCache.get(kind)).thenReturn("schema"); - when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema"); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.schemaCache, times(2)).get(anyString()); - verify(this.schemaCache, times(2)).delete(anyString()); + } + + @Test + public void should_return_basic_schema_when_storage_returns_no_schema() { + IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false); + + assertNotNull(returnedSchema.getDataSchema()); + assertNotNull(returnedSchema); + assertEquals(kind, returnedSchema.getKind()); + } + + @Test + public void should_create_schema_when_storage_returns_valid_schema() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"startDate\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"endDate\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"type \"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"itemguid\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map<String, OperationType> schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_merge_mapping_when_storage_returns_valid_schema() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"startDate\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map<String, OperationType> schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any()); + verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String reason = String + .format("Could not create type mapping %s/completion.", kind.replace(":", "-")); + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map<String, OperationType> schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())) + .thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, "")); + + try { + this.sut.processSchemaMessages(schemaMessages); + } catch (AppException e) { + assertEquals(RequestStatus.SCHEMA_CONFLICT, e.getError().getCode()); + assertEquals("error creating or merging index mapping", e.getError().getMessage()); + assertEquals(reason, e.getError().getReason()); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); } - - @Test - public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - Map<String, OperationType> schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.purge_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.log).warning(eq(String.format("Kind: %s not found", kind))); + } + + @Test + public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String reason = String + .format("Could not create type mapping %s/completion.", kind.replace(":", "-")); + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map<String, OperationType> schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())) + .thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah")); + + try { + this.sut.processSchemaMessages(schemaMessages); + } catch (AppException e) { + assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode()); + assertEquals("blah", e.getError().getMessage()); + assertEquals(reason, e.getError().getReason()); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); } - - @Test - public void should_sync_schema_with_storage() throws Exception { - String kind = "tenant1:avocet:completion:1.0.0"; - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.indicesService.deleteIndex(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - - this.sut.syncIndexMappingWithStorageSchema(kind); - - verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); - verify(this.indicesService, times(1)).isIndexExist(any(), any()); - verify(this.indicesService, times(1)).deleteIndex(any(), any()); - verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); - verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_log_and_do_nothing_when_storage_returns_invalid_schema() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"" + + "}"; + Map<String, OperationType> schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0")); + } + + @Test + public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() + throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + Map<String, OperationType> schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.purge_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaCache.get(kind)).thenReturn("schema"); + when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema"); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.schemaCache, times(2)).get(anyString()); + verify(this.schemaCache, times(2)).delete(anyString()); + } + + @Test + public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() + throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + Map<String, OperationType> schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.purge_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.log).warning(eq(String.format("Kind: %s not found", kind))); + } + + @Test + public void should_sync_schema_with_storage() throws Exception { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.indicesService.deleteIndex(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + + this.sut.syncIndexMappingWithStorageSchema(kind); + + verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, times(1)).isIndexExist(any(), any()); + verify(this.indicesService, times(1)).deleteIndex(any(), any()); + verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() + throws Exception { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.indicesService.deleteIndex(any(), any())) + .thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah")); + + try { + this.sut.syncIndexMappingWithStorageSchema(kind); + } catch (AppException e) { + assertEquals(HttpStatus.SC_CONFLICT, e.getError().getCode()); + assertEquals("blah", e.getError().getMessage()); + assertEquals("Index deletion error", e.getError().getReason()); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); } - @Test - public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() throws Exception { - String kind = "tenant1:avocet:completion:1.0.0"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.indicesService.deleteIndex(any(), any())).thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah")); - - try { - this.sut.syncIndexMappingWithStorageSchema(kind); - } catch (AppException e) { - assertEquals(HttpStatus.SC_CONFLICT, e.getError().getCode()); - assertEquals("blah", e.getError().getMessage()); - assertEquals("Index deletion error", e.getError().getReason()); - } catch (Exception e) { - fail("Should not throw this exception " + e.getMessage()); - } - - verify(this.indicesService, times(1)).isIndexExist(any(), any()); - verify(this.indicesService, times(1)).deleteIndex(any(), any()); - verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any()); - verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any()); - } - - @Test - public void should_return_true_while_if_forceClean_requested() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - - assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true)); - } - - @Test - public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); - - assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false)); - } - - @Test - public void should_return_false_while_if_forceClean_notRequested_and_indexExist() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - - assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false)); - } + verify(this.indicesService, times(1)).isIndexExist(any(), any()); + verify(this.indicesService, times(1)).deleteIndex(any(), any()); + verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any()); + } + + @Test + public void should_return_true_while_if_forceClean_requested() throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + + assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true)); + } + + @Test + public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() + throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + + assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false)); + } + + @Test + public void should_return_false_while_if_forceClean_notRequested_and_indexExist() + throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + + assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false)); + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java index 8cc2f75c8371ebcc410ed08e81898a2487dc904c..bf7fa01a337c13971b47efc97c5b20f96788cb9d 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,17 @@ package org.opengroup.osdu.indexer.service; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.when; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -35,119 +46,116 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunnerDelegate; import org.springframework.test.context.junit4.SpringRunner; -import java.util.*; - -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.mockStatic; -import static org.powermock.api.mockito.PowerMockito.when; - @RunWith(PowerMockRunner.class) @PowerMockRunnerDelegate(SpringRunner.class) public class ReindexServiceTest { - private final String cursor = "100"; - - private final String correlationId = UUID.randomUUID().toString(); - - @Mock - private IndexerConfigurationProperties configurationProperties; - @Mock - private StorageService storageService; - @Mock - private IRequestInfo requestInfo; - @Mock - private IndexerQueueTaskBuilder indexerQueueTaskBuilder; - @Mock - private JaxRsDpsLog log; - @InjectMocks - private ReindexServiceImpl sut; - - private RecordReindexRequest recordReindexRequest; - private RecordQueryResponse recordQueryResponse; - - private Map<String, String> httpHeaders; - - @Before - public void setup() { - initMocks(this); - - mockStatic(UUID.class); - - recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build(); - recordQueryResponse = new RecordQueryResponse(); - - httpHeaders = new HashMap<>(); - httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); - httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); - DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders); - when(requestInfo.getHeaders()).thenReturn(standardHeaders); - when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); - when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders); + private final String cursor = "100"; + + private final String correlationId = UUID.randomUUID().toString(); + + @Mock + private IndexerConfigurationProperties configurationProperties; + @Mock + private StorageService storageService; + @Mock + private IRequestInfo requestInfo; + @Mock + private IndexerQueueTaskBuilder indexerQueueTaskBuilder; + @Mock + private JaxRsDpsLog log; + @InjectMocks + private ReindexServiceImpl sut; + + private RecordReindexRequest recordReindexRequest; + private RecordQueryResponse recordQueryResponse; + + private Map<String, String> httpHeaders; + + @Before + public void setup() { + initMocks(this); + + mockStatic(UUID.class); + + recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0") + .cursor(cursor).build(); + recordQueryResponse = new RecordQueryResponse(); + + httpHeaders = new HashMap<>(); + httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); + httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); + DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders); + when(requestInfo.getHeaders()).thenReturn(standardHeaders); + when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); + when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders); + } + + @Test + public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setResults(null); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + + String response = sut.reindexRecords(recordReindexRequest, false); + + Assert.assertNull(response); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); } + } - @Test - public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() { - try { - recordQueryResponse.setResults(null); - when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + @Test + public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setResults(new ArrayList<>()); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); - String response = sut.reindexRecords(recordReindexRequest, false); + String response = sut.reindexRecords(recordReindexRequest, false); - Assert.assertNull(response); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } + Assert.assertNull(response); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); } + } - @Test - public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() { - try { - recordQueryResponse.setResults(new ArrayList<>()); - when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + @Ignore + @Test + public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setCursor(cursor); + List<String> results = new ArrayList<>(); + results.add("test1"); + recordQueryResponse.setResults(results); - String response = sut.reindexRecords(recordReindexRequest, false); + when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1); - Assert.assertNull(response); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - @Ignore - @Test - public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() { - try { - recordQueryResponse.setCursor(cursor); - List<String> results = new ArrayList<>(); - results.add("test1"); - recordQueryResponse.setResults(results); - - when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1); - - when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); - String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); + String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); - Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload); - } catch (Exception e) { - fail("Should not throw exception" + e.getMessage()); - } + Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", + taskQueuePayload); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); } - - @Test - public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() { - try { - List<String> results = new ArrayList<>(); - results.add("test1"); - recordQueryResponse.setResults(results); - when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); - - String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); - - Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload); - } catch (Exception e) { - fail("Should not throw exception" + e.getMessage()); - } + } + + @Test + public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() { + try { + List<String> results = new ArrayList<>(); + results.add("test1"); + recordQueryResponse.setResults(results); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + + String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); + + Assert.assertEquals(String.format( + "{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", + correlationId), taskQueuePayload); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); } + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java index b4ea76b8e80b1e8ea1a1274ee53c974562cd36e4..4b46abf2e3380742203290ead746c39dff3a4049 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,23 @@ package org.opengroup.osdu.indexer.service; +import static java.util.Collections.singletonList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; + import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; - +import java.lang.reflect.Type; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -27,227 +41,232 @@ import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.http.IUrlFetchService; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.http.HttpResponse; import org.opengroup.osdu.core.common.model.indexer.IndexingStatus; import org.opengroup.osdu.core.common.model.indexer.JobStatus; import org.opengroup.osdu.core.common.model.indexer.RecordInfo; -import org.opengroup.osdu.core.common.model.http.HttpResponse; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.http.IUrlFetchService; import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse; import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest; import org.opengroup.osdu.core.common.model.indexer.Records; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.http.HttpStatus; import org.springframework.test.context.junit4.SpringRunner; -import java.lang.reflect.Type; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -import static java.util.Collections.singletonList; -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.powermock.api.mockito.PowerMockito.when; - @RunWith(SpringRunner.class) public class StorageServiceTest { - @Mock - private IUrlFetchService urlFetchService; - @Mock - private JobStatus jobStatus; - @Mock - private JaxRsDpsLog log; - @Mock - private IRequestInfo requestInfo; - @Mock - private IndexerConfigurationProperties configurationProperties; - @InjectMocks - private StorageServiceImpl sut; + @Mock + private IUrlFetchService urlFetchService; + @Mock + private JobStatus jobStatus; + @Mock + private JaxRsDpsLog log; + @Mock + private IRequestInfo requestInfo; + @Mock + private IndexerConfigurationProperties configurationProperties; + @InjectMocks + private StorageServiceImpl sut; - private List<String> ids; - private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465"; - private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813"; + private List<String> ids; + private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465"; + private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813"; - @Before - public void setup() { + @Before + public void setup() { - String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," + - "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]"; + String recordChangedMessages = + "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," + + + "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]"; - when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>()); - when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders()); + when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>()); + when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders()); - Type listType = new TypeToken<List<RecordInfo>>() {}.getType(); + Type listType = new TypeToken<List<RecordInfo>>() { + }.getType(); - List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType); - jobStatus.initialize(msgs); - ids = Arrays.asList(RECORD_ID1, RECORDS_ID2); + List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType); + jobStatus.initialize(msgs); + ids = Arrays.asList(RECORD_ID1, RECORDS_ID2); - when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20); - } + when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20); + } - @Test - public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException { + @Test + public void should_return404_givenNullData_getValidStorageRecordsTest() + throws URISyntaxException { - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(null); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(null); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - should_return404_getValidStorageRecordsTest(); - } + should_return404_getValidStorageRecordsTest(); + } - @Test - public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException { + @Test + public void should_return404_givenEmptyData_getValidStorageRecordsTest() + throws URISyntaxException { - String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}"; + String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}"; - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - should_return404_getValidStorageRecordsTest(); - } + should_return404_getValidStorageRecordsTest(); + } - @Test - public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException { + @Test + public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() + throws URISyntaxException { - String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}"; + String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}"; - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - Records storageRecords = this.sut.getStorageRecords(ids); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); - assertEquals(1, storageRecords.getRecords().size()); - } + assertEquals(1, storageRecords.getRecords().size()); + } - @Test - public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException { + @Test + public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException { - String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}"; + String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}"; - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); - when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); - Records storageRecords = this.sut.getStorageRecords(ids); + when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); - assertEquals(1, storageRecords.getRecords().size()); - verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, HttpStatus.NOT_FOUND.value(), "Partial response received from Storage service - missing records", "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813"); - } + assertEquals(1, storageRecords.getRecords().size()); + verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, + HttpStatus.NOT_FOUND.value(), + "Partial response received from Storage service - missing records", + "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813"); + } - @Test - public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException { - String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}"; + @Test + public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() + throws URISyntaxException { + String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}"; - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); - when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); - Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2)); + when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2)); - assertEquals(1, storageRecords.getRecords().size()); - verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed")); - } + assertEquals(1, storageRecords.getRecords().size()); + verify(this.jobStatus) + .addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), + "crs conversion failed", String + .format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", + "crs conversion failed")); + } - @Test - public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception { + @Test + public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() + throws Exception { - RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build(); + RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder() + .kind("tenant:test:test:1.0.0").cursor("100").build(); - HttpResponse httpResponse = new HttpResponse(); - httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class)); + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class)); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest); + RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest); - assertEquals("100", recordQueryResponse.getCursor()); - assertNull(recordQueryResponse.getResults()); - } + assertEquals("100", recordQueryResponse.getCursor()); + assertNull(recordQueryResponse.getResults()); + } - @Test - public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception { - - String validSchemaFromStorage = "{" + - " \"kind\": \"tenant:test:test:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"msg\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"references.entity\"," + - " \"kind\": \"string\"" + - " }" + - " ]," + - " \"ext\": null" + - "}"; - String kind = "tenant:test:test:1.0.0"; - - HttpResponse httpResponse = new HttpResponse(); - httpResponse.setResponseCode(HttpStatus.OK.value()); - httpResponse.setBody(validSchemaFromStorage); - - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - - String recordSchemaResponse = this.sut.getStorageSchema(kind); - - assertNotNull(recordSchemaResponse); - } + @Test + public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception { - @Test - public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception { + String validSchemaFromStorage = "{" + + " \"kind\": \"tenant:test:test:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"msg\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"references.entity\"," + + " \"kind\": \"string\"" + + " }" + + " ]," + + " \"ext\": null" + + "}"; + String kind = "tenant:test:test:1.0.0"; - String kind = "tenant:test:test:1.0.0"; + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setResponseCode(HttpStatus.OK.value()); + httpResponse.setBody(validSchemaFromStorage); - HttpResponse httpResponse = new HttpResponse(); - httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value()); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + String recordSchemaResponse = this.sut.getStorageSchema(kind); - String recordSchemaResponse = this.sut.getStorageSchema(kind); + assertNotNull(recordSchemaResponse); + } - assertNull(recordSchemaResponse); - } + @Test + public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception { - @Test - public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException { + String kind = "tenant:test:test:1.0.0"; - String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}"; + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value()); - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - Records storageRecords = this.sut.getStorageRecords(ids); + String recordSchemaResponse = this.sut.getStorageSchema(kind); - assertEquals(1, storageRecords.getRecords().size()); + assertNull(recordSchemaResponse); + } - assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size()); + @Test + public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() + throws URISyntaxException { - assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0)); - } + String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}"; + + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); + + assertEquals(1, storageRecords.getRecords().size()); + + assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size()); + + assertEquals("conversion error occurred", + storageRecords.getConversionStatuses().get(0).getErrors().get(0)); + } - private void should_return404_getValidStorageRecordsTest() { - try { - this.sut.getStorageRecords(ids); - fail("Should throw exception"); - } catch (AppException e) { - assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode()); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } + private void should_return404_getValidStorageRecordsTest() { + try { + this.sut.getStorageRecords(ids); + fail("Should throw exception"); + } catch (AppException e) { + assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode()); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); } + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java deleted file mode 100644 index 2c681985f31965ec738d6c220f2645c31fc121cf..0000000000000000000000000000000000000000 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.util; - -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.services.iam.v1.Iam; -import com.google.api.services.iam.v1.model.SignJwtResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Spy; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; -import org.opengroup.osdu.core.common.model.search.IdToken; -import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.springframework.test.context.junit4.SpringRunner; - -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.when; - -@Ignore -@RunWith(SpringRunner.class) -@PrepareForTest({GoogleNetHttpTransport.class, GoogleCredential.class, NetHttpTransport.class, SignJwtResponse.class, Iam.Builder.class, HttpClients.class, EntityUtils.class, IndexerConfigurationProperties.class}) -public class ServiceAccountJwtGcpClientImplTest { - - private static final String JWT_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ik1UVXlPREE0TXpFd09BPT0ifQ.eyJzdWIiOiJtemh1OUBzbGIuY29tIiwiaXNzIjoic2F1dGgtcHJldmlldy5zbGIuY29tIiwiYXVkIjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJpYXQiOjE1MjgxNDg5MTUsImV4cCI6MTUyODIzNTMxNSwicHJvdmlkZXIiOiJzbGIuY29tIiwiY2xpZW50IjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJ1c2VyaWQiOiJtemh1OUBzbGIuY29tIiwiZW1haWwiOiJtemh1OUBzbGIuY29tIiwiYXV0aHoiOiJ7XCJhY2NvdW50Q291bnRyeVwiOntcImNvZGVcIjpcInVzXCIsXCJpZFwiOjU3MTU5OTkxMDE4MTI3MzYsXCJuYW1lXCI6XCJVbml0ZWQgU3RhdGVzIG9mIEFtZXJpY2FcIn0sXCJhY2NvdW50SWRcIjo1NjkxODc4ODMzOTEzODU2LFwiYWNjb3VudE5hbWVcIjpcIlNJUyBJbnRlcm5hbCBIUVwiLFwiY3JlYXRlZFwiOlwiMjAxOC0wNS0wM1QxNzoyNTo1NS40NDNaXCIsXCJkZXBhcnRtZW50TWFuYWdlclwiOm51bGwsXCJzdWJzY3JpcHRpb25zXCI6W3tcImFjY291bnRJZFwiOjU2OTE4Nzg4MzM5MTM4NTYsXCJjb250cmFjdElkXCI6NTc1MTcwMDIxMjE1NDM2OCxcImNyZWF0ZWRcIjpcIjIwMTgtMDUtMDNUMTc6MzM6MDkuNTczWlwiLFwiY3JtQ29udHJhY3RJZFwiOlwiU0lTLUlOVEVSTkFMLUhRLVFBXCIsXCJjcm1Db250cmFjdEl0ZW1JZFwiOlwiZGV2bGlcIixcImV4cGlyYXRpb25cIjpcIjE5NzAtMDEtMDFUMDA6MDA6MDAuMDAwWlwiLFwiaWRcIjo1MDc5Mjg4NTA0MTIzMzkyLFwicHJvZHVjdFwiOntcImNvZGVcIjpcImRldmVsb3Blci1saWdodFwiLFwiY29tY2F0TmFtZVwiOlwiTm90IGluIENvbUNhdFwiLFwiZmVhdHVyZVNldHNcIjpbe1wiYXBwbGljYXRpb25cIjp7XCJjb2RlXCI6XCJhcGlkZXZlbG9wZXJwb3J0YWxcIixcImlkXCI6NTE2ODkzMDY5NTkzODA0OCxcIm5hbWVcIjpcIkFQSSBEZXZlbG9wZXIgUG9ydGFsXCIsXCJ0eXBlXCI6XCJXZWJBcHBcIn0sXCJjbGFpbXNcIjpudWxsLFwiaWRcIjo1MTkxNTcyMjg3MTI3NTUyLFwibmFtZVwiOlwiRGV2ZWxvcGVyXCIsXCJ0eXBlXCI6XCJCQVNFXCJ9XSxcImlkXCI6NTE1MDczMDE1MTI2NDI1NixcIm5hbWVcIjpcIkRldmVsb3BlciBQb3J0YWxcIixcInBhcnROdW1iZXJcIjpcIlNERUwtUEItU1VCVVwifX1dLFwidXNlckVtYWlsXCI6XCJtemh1OUBzbGIuY29tXCIsXCJ1c2VyTmFtZVwiOlwiTWluZ3lhbmcgWmh1XCJ9XG4iLCJsYXN0bmFtZSI6IlpodSIsImZpcnN0bmFtZSI6Ik1pbmd5YW5nIiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiNDE3YjczMjktYmMwNy00OTFmLWJiYzQtZTQ1YjRhMWFiYjVjLVd3U0c0dyIsImlkcCI6ImNvcnAyIiwiaGQiOiJzbGIuY29tIn0.WQfGr1Xu-6IdaXdoJ9Fwzx8O2el1UkFPWo1vk_ujiAfdOjAR46UG5SrBC7mzC7gYRyK3a4fimBmbv3uRVJjTNXdxXRLZDw0SvXUMIOqjUGLom491ESbrtka_Xz7vGO-tWyDcEQDTfFzQ91LaVN7XdzL18_EDTXZoPhKb-zquyk9WLQxP9Mw-3Yh-UrbvC9nl1-GRn1IVbzp568kqkpOVUFM9alYSGw-oMGDZNt1DIYOJnpGaw2RB5B3AKvNivZH_Xdac7ZTzQbsDOt8B8DL2BphuxcJ9jshCJkM2SHQ15uErv8sfnzMwdF08e_0QcC_30I8eX9l8yOu6TnwwqlXunw"; - - @Mock - private IndexerConfigurationProperties indexerConfigurationProperties; - @Mock - private JaxRsDpsLog log; - @Mock - private GoogleCredential credential; - @Mock - private NetHttpTransport httpTransport; - @Mock - private SignJwtResponse signJwtResponse; - @Mock - private Iam iam; - @Mock - private Iam.Projects iamProject; - @Mock - private Iam.Projects.ServiceAccounts iamProjectServiceAccounts; - @Mock - private Iam.Projects.ServiceAccounts.SignJwt signJwt; - @Mock - private CloseableHttpClient httpClient; - @Mock - private CloseableHttpResponse httpResponse; -// @InjectMocks -// private TenantInfoServiceImpl tenantInfoServiceProvider; -// @Mock -// private TenantInfoServiceImpl tenantInfoService; - @Mock - private IJwtCache cacheService; - @InjectMocks @Spy - private ServiceAccountJwtGcpClientImpl sut; - @Before - public void setup() throws Exception { - initMocks(this); - -// mockStatic(GoogleNetHttpTransport.class); -// mockStatic(GoogleCredential.class); -// mockStatic(HttpClients.class); -// mockStatic(EntityUtils.class); -// mockStatic(Config.class); - - when(GoogleNetHttpTransport.newTrustedTransport()).thenReturn(httpTransport); - when(GoogleCredential.getApplicationDefault()).thenReturn(credential); - when(credential.createScopedRequired()).thenReturn(true); - when(credential.createScoped(any())).thenReturn(credential); - when(HttpClients.createDefault()).thenReturn(httpClient); - when(httpClient.execute(any())).thenReturn(httpResponse); - when(indexerConfigurationProperties.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); - when(indexerConfigurationProperties.getGoogleAudiences()).thenReturn("aud"); - -// when(this.tenantInfoServiceProvider).thenReturn(this.tenantInfoService); - - TenantInfo tenantInfo = new TenantInfo(); - tenantInfo.setServiceAccount("tenant"); -// when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); - - when(this.sut.getIam()).thenReturn(iam); - when(this.iam.projects()).thenReturn(iamProject); - when(this.iamProject.serviceAccounts()).thenReturn(iamProjectServiceAccounts); - when(this.iamProjectServiceAccounts.signJwt(any(), any())).thenReturn(signJwt); - when(this.signJwt.execute()).thenReturn(signJwtResponse); - when(this.signJwtResponse.getSignedJwt()).thenReturn("testJwt"); - - } - - @Test - public void should_returnCachedToken_givenCachedToken_getIdTokenTest() { - String tokenValue = "tokenValue"; - IdToken idToken = IdToken.builder().tokenValue(tokenValue).expirationTimeMillis(System.currentTimeMillis() + 10000000L).build(); - when(this.cacheService.get(any())).thenReturn(idToken); - - String returnedIdToken = this.sut.getIdToken(tokenValue); - - Assert.assertEquals(tokenValue, returnedIdToken); - } - - @Test - public void should_returnValidToken_getIdTokenTest() throws Exception { - when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", JWT_TOKEN)); - - String returnedToken = this.sut.getIdToken("tenant"); - - Assert.assertEquals(JWT_TOKEN, returnedToken); - } - - @Test - public void should_return500_given_invalidJWTResponse_getIdTokenException() { - try { - when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", "invalid jwt")); - - this.sut.getIdToken("tenant"); - fail("Should throw exception"); - } catch (AppException e) { - Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode()); - Assert.assertEquals("Invalid token, error decoding", e.getError().getMessage()); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test - public void should_return403_given_missingIdTokenResponse_getIdTokenException() { - try { - when(EntityUtils.toString(any())).thenReturn("{}"); - - this.sut.getIdToken("tenant"); - fail("Should throw exception"); - } catch (AppException e) { - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode()); - Assert.assertEquals("The user is not authorized to perform this action", e.getError().getMessage()); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } -}