From c2a2f127309356e1218e5b0794ca7cc87a82b6f3 Mon Sep 17 00:00:00 2001
From: ZMai <zmai@slb.com>
Date: Wed, 25 Jan 2023 13:32:08 -0600
Subject: [PATCH] Use alias instead of index name if the given index name list
is too large (> 3840 bytes)
---
pom.xml | 2 +-
search-core/pom.xml | 1 -
.../osdu/search/cache/IndexAliasCache.java | 25 ++++
.../search/service/IndexAliasService.java | 20 +++
.../search/service/IndexAliasServiceImpl.java | 140 ++++++++++++++++++
.../osdu/search/util/CrossTenantUtils.java | 29 +++-
6 files changed, 212 insertions(+), 5 deletions(-)
create mode 100644 search-core/src/main/java/org/opengroup/osdu/search/cache/IndexAliasCache.java
create mode 100644 search-core/src/main/java/org/opengroup/osdu/search/service/IndexAliasService.java
create mode 100644 search-core/src/main/java/org/opengroup/osdu/search/service/IndexAliasServiceImpl.java
diff --git a/pom.xml b/pom.xml
index 0fd44516f..07dd7ac01 100644
--- a/pom.xml
+++ b/pom.xml
@@ -40,7 +40,7 @@
<log4j-core.version>2.17.1</log4j-core.version>
<google-oauth-client.version>1.34.1</google-oauth-client.version>
<commons-compress.version>1.21</commons-compress.version>
- <osdu.oscorecommon.version>0.18.0</osdu.oscorecommon.version>
+ <osdu.oscorecommon.version>0.19.0-SNAPSHOT-ALIAS</osdu.oscorecommon.version>
<tomcat-embed-core.version>9.0.67</tomcat-embed-core.version>
<openapi.version>1.6.9</openapi.version>
<json-smart.version>2.4.7</json-smart.version>
diff --git a/search-core/pom.xml b/search-core/pom.xml
index dc6453dbb..c74e09ec5 100644
--- a/search-core/pom.xml
+++ b/search-core/pom.xml
@@ -38,7 +38,6 @@
<project.main.basedir>${project.basedir}</project.main.basedir>
<nimbus-jose-jwt.version>9.1.2</nimbus-jose-jwt.version>
<elasticsearch.version>7.8.1</elasticsearch.version>
- <osdu.oscorecommon.version>0.17.0</osdu.oscorecommon.version>
<netty.version>4.1.70.Final</netty.version>
<spring-security-web.version>5.7.3</spring-security-web.version>
<spring-webmvc.version>5.3.22</spring-webmvc.version>
diff --git a/search-core/src/main/java/org/opengroup/osdu/search/cache/IndexAliasCache.java b/search-core/src/main/java/org/opengroup/osdu/search/cache/IndexAliasCache.java
new file mode 100644
index 000000000..9861d743f
--- /dev/null
+++ b/search-core/src/main/java/org/opengroup/osdu/search/cache/IndexAliasCache.java
@@ -0,0 +1,25 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.search.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.springframework.stereotype.Component;
+
+@Component
+public class IndexAliasCache extends VmCache<String, String> {
+ public IndexAliasCache() {
+ super(7 * 24 * 3600, 2000);
+ }
+}
diff --git a/search-core/src/main/java/org/opengroup/osdu/search/service/IndexAliasService.java b/search-core/src/main/java/org/opengroup/osdu/search/service/IndexAliasService.java
new file mode 100644
index 000000000..16ebc6e82
--- /dev/null
+++ b/search-core/src/main/java/org/opengroup/osdu/search/service/IndexAliasService.java
@@ -0,0 +1,20 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.search.service;
+
+public interface IndexAliasService {
+ boolean hasIndexAlias(String kind);
+ String getIndexAlias(String kind);
+}
diff --git a/search-core/src/main/java/org/opengroup/osdu/search/service/IndexAliasServiceImpl.java b/search-core/src/main/java/org/opengroup/osdu/search/service/IndexAliasServiceImpl.java
new file mode 100644
index 000000000..be57189e7
--- /dev/null
+++ b/search-core/src/main/java/org/opengroup/osdu/search/service/IndexAliasServiceImpl.java
@@ -0,0 +1,140 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.search.service;
+
+import com.google.api.client.util.Strings;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.GetAliasesResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.rest.RestStatus;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.search.cache.IndexAliasCache;
+import org.opengroup.osdu.search.util.ElasticClientHandler;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Component
+public class IndexAliasServiceImpl implements IndexAliasService {
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+ @Inject
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Inject
+ private IndexAliasCache indexAliasCache;
+ @Inject
+ private JaxRsDpsLog log;
+
+ @Override
+ public boolean hasIndexAlias(String kind) {
+ if(!elasticIndexNameResolver.isIndexAliasSupported(kind))
+ return false;
+
+ String alias = indexAliasCache.get(kind);
+ if(Strings.isNullOrEmpty(alias)) {
+ try {
+ alias = getOrCreateIndexAliases(kind);
+ if(!Strings.isNullOrEmpty(alias)) {
+ indexAliasCache.put(kind, alias);
+ }
+ } catch (Exception e) {
+ log.error(String.format("Fail to get or create index alias for kind '%s'", kind), e);
+ }
+ }
+
+ return (!Strings.isNullOrEmpty(alias));
+ }
+
+ public String getIndexAlias(String kind) {
+ if(hasIndexAlias(kind)) {
+ return indexAliasCache.get(kind);
+ }
+ return null;
+ }
+
+ private String getOrCreateIndexAliases(String kind) throws IOException {
+ String alias = elasticIndexNameResolver.getIndexAliasFromKind(kind);
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ GetAliasesRequest request = new GetAliasesRequest(alias);
+ if(restClient.indices().existsAlias(request, RequestOptions.DEFAULT)) {
+ return alias;
+ }
+
+ String index = elasticIndexNameResolver.getIndexNameFromKind(kind);
+ // To create an alias for an index, the index name must the concrete index name, not alias
+ index = resolveConcreteIndexName(restClient, index);
+ if(!Strings.isNullOrEmpty(index)) {
+ IndicesAliasesRequest addRequest = new IndicesAliasesRequest();
+ IndicesAliasesRequest.AliasActions aliasActions = new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
+ .index(index)
+ .alias(alias);
+ addRequest.addAliasAction(aliasActions);
+ AcknowledgedResponse response = restClient.indices().updateAliases(addRequest, RequestOptions.DEFAULT);
+ if(response.isAcknowledged()) {
+ return alias;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ private String resolveConcreteIndexName(RestHighLevelClient restClient, String index) throws IOException {
+ GetAliasesRequest request = new GetAliasesRequest(index);
+ GetAliasesResponse response = restClient.indices().getAlias(request, RequestOptions.DEFAULT);
+ if(response.status() == RestStatus.NOT_FOUND) {
+ /* index resolved from kind is actual concrete index
+ * Example:
+ * {
+ * "opendes-wke-well-1.0.7": {
+ * "aliases": {}
+ * }
+ * }
+ */
+ return index;
+ }
+ if(response.status() == RestStatus.OK) {
+ /* index resolved from kind is NOT actual create index. It is just an alias
+ * The concrete index name in this example is "opendes-osdudemo-wellbore-1.0.0_1649167113090"
+ * Example:
+ * {
+ * "opendes-osdudemo-wellbore-1.0.0_1649167113090": {
+ * "aliases": {
+ * "opendes-osdudemo-wellbore-1.0.0": {}
+ * }
+ * }
+ * }
+ */
+ Map<String, Set<AliasMetadata>> aliases = response.getAliases();
+ for (Map.Entry<String, Set<AliasMetadata>> entry: aliases.entrySet()) {
+ String actualIndex = entry.getKey();
+ List<String> aliaseNames = entry.getValue().stream().map(a -> a.getAlias()).collect(Collectors.toList());
+ if(aliaseNames.contains(index))
+ return actualIndex;
+ }
+ }
+ return null;
+ }
+}
diff --git a/search-core/src/main/java/org/opengroup/osdu/search/util/CrossTenantUtils.java b/search-core/src/main/java/org/opengroup/osdu/search/util/CrossTenantUtils.java
index a21c0b1f6..ba2b7a439 100644
--- a/search-core/src/main/java/org/opengroup/osdu/search/util/CrossTenantUtils.java
+++ b/search-core/src/main/java/org/opengroup/osdu/search/util/CrossTenantUtils.java
@@ -14,9 +14,11 @@
package org.opengroup.osdu.search.util;
+import com.google.api.client.util.Strings;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.core.common.model.search.Query;
import org.opengroup.osdu.core.common.util.KindParser;
+import org.opengroup.osdu.search.service.IndexAliasService;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
@@ -24,16 +26,37 @@ import java.util.List;
@Component
public class CrossTenantUtils {
+ // For details, please refer to implementation of class
+ // org.opengroup.osdu.core.common.model.search.validation.MultiKindValidator
+ private static final int MAX_INDEX_NAME_LENGTH = 3840;
@Inject
private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Inject
+ private IndexAliasService indexAliasService;
public String getIndexName(Query searchRequest) {
- StringBuilder builder = new StringBuilder();
List<String> kinds = KindParser.parse(searchRequest.getKind());
+ String indexNames = buildIndexNames(kinds, false);
+ if(indexNames.length() > MAX_INDEX_NAME_LENGTH) {
+ indexNames = buildIndexNames(kinds, true);
+ }
+
+ return indexNames;
+ }
+
+ private String buildIndexNames(List<String> kinds, boolean useAlias) {
+ StringBuilder builder = new StringBuilder();
for(String kind : kinds) {
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
- builder.append(index + ",");
+ String index = null;
+ if(useAlias) {
+ index = this.indexAliasService.getIndexAlias(kind);
+ }
+ if(Strings.isNullOrEmpty(index)) {
+ index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
+ }
+ builder.append(index);
+ builder.append(",");
}
builder.append("-.*"); // Exclude Lucene/ElasticSearch internal indices
return builder.toString();
--
GitLab