Skip to content
Snippets Groups Projects
Commit 16d1b837 authored by Rustam Lotsmanenko (EPAM)'s avatar Rustam Lotsmanenko (EPAM)
Browse files

fix get mapping

parent 575e9075
No related branches found
No related tags found
1 merge request!774Search backend (Elasticsearch) Upgrade. Part 1 - Latest v7.17.x Upgrade.
Pipeline #270256 failed
Stage: review
Stage: build
Stage: coverage
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Stage: publish
Hide whitespace changes
Inline Side-by-side
Showing
with 69 additions and 122 deletions
indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexAliasServiceImpl.java
+ 33
− 9
View file @ 16d1b837
......@@ -16,6 +16,7 @@
package org.opengroup.osdu.indexer.service;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.aggregations.StringTermsAggregate;
import co.elastic.clients.elasticsearch._types.aggregations.TermsAggregation;
import co.elastic.clients.elasticsearch.core.SearchRequest;
......@@ -150,15 +151,32 @@ public class IndexAliasServiceImpl implements IndexAliasService{
if (!isCompleteVersionKind(kind)) {
return index;
}
GetAliasRequest request = new GetAliasRequest.Builder().name(index).build();
GetAliasResponse response = client.indices().getAlias(request);
if (response.result().isEmpty()) {
// index resolved from kind is actual concrete index
return index;
}else {
// index resolved from kind is NOT actual concrete index. It is just an alias
try {
GetAliasResponse response = client.indices().getAlias(request);
if(response.result().isEmpty()){
/* index resolved from kind is actual concrete index
* Example:
* {
* "opendes-wke-well-1.0.7": {
* "aliases": {}
* }
* }
*/
return index;
}
/* index resolved from kind is NOT actual create index. It is just an alias
* The concrete index name in this example is "opendes-osdudemo-wellbore-1.0.0_1649167113090"
* Example:
* {
* "opendes-osdudemo-wellbore-1.0.0_1649167113090": {
* "aliases": {
* "opendes-osdudemo-wellbore-1.0.0": {}
* }
* }
* }
*/
for (Map.Entry<String, IndexAliases> entry: response.result().entrySet()){
String actualIndex = entry.getKey();
List<String> aliaseNames = entry.getValue().aliases().keySet().stream().toList();
......@@ -166,9 +184,15 @@ public class IndexAliasServiceImpl implements IndexAliasService{
return actualIndex;
}
}
return index;
}catch (ElasticsearchException e){
if(e.status() == HttpStatus.SC_NOT_FOUND){
jaxRsDpsLog.debug("Alias not found.", e.getMessage());
}else {
jaxRsDpsLog.error("Unexpected error response from Elasticsearch.", e);
}
return index;
}
return index;
}
private boolean isCompleteVersionKind(String kind) {
......
indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+ 2
− 2
View file @ 16d1b837
......@@ -146,9 +146,9 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
}
for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
if (entry.getKey() == RecordMetaAttribute.AUTHORITY.getValue()) {
if (entry.getKey().equals(RecordMetaAttribute.AUTHORITY.getValue())) {
metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), kind.getAuthority()));
} else if (entry.getKey() == RecordMetaAttribute.SOURCE.getValue()) {
} else if (entry.getKey().equals(RecordMetaAttribute.SOURCE.getValue())) {
metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), kind.getSource()));
} else {
metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), null));
......
indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
+ 9
− 2
View file @ 16d1b837
......@@ -16,9 +16,11 @@ package org.opengroup.osdu.indexer.service;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.Time;
import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
import co.elastic.clients.elasticsearch.indices.GetMappingRequest;
import co.elastic.clients.elasticsearch.indices.GetMappingResponse;
import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
import co.elastic.clients.json.JsonpUtils;
import java.io.IOException;
import java.util.Map;
import java.util.Objects;
......@@ -86,8 +88,13 @@ public abstract class MappingServiceImpl implements IMappingService {
if (mappingRecordMap.isEmpty()) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", String.format("Error retrieving mapping for kind %s", this.elasticIndexNameResolver.getKindFromIndexName(index)));
}
Optional<IndexMappingRecord> first = mappingRecordMap.values().stream().findFirst();
return first.get().toString();
Optional<IndexMappingRecord> mappingRecord = mappingRecordMap.values().stream().findFirst();
StringBuilder collector = new StringBuilder();
mappingRecord.ifPresent(indexMappingRecord -> {
TypeMapping mappings = indexMappingRecord.mappings();
JsonpUtils.toString(mappings, collector);
});
return collector.toString();
} catch (IOException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", String.format("Error retrieving mapping for kind %s", this.elasticIndexNameResolver.getKindFromIndexName(index)), e);
}
......
indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+ 21
− 1
View file @ 16d1b837
......@@ -35,10 +35,15 @@ import co.elastic.clients.elasticsearch._types.AcknowledgedResponse;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.ErrorCause;
import co.elastic.clients.elasticsearch._types.ErrorResponse;
import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
import co.elastic.clients.elasticsearch.indices.ElasticsearchIndicesClient;
import co.elastic.clients.elasticsearch.indices.GetMappingRequest;
import co.elastic.clients.elasticsearch.indices.GetMappingResponse;
import co.elastic.clients.elasticsearch.indices.PutMappingRequest;
import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
import com.google.gson.Gson;
import java.io.IOException;
import java.io.StringReader;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
......@@ -77,7 +82,7 @@ public class IndexerMappingServiceTest {
private final String validMapping = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"null_value\":\"null\",\"ignore_above\":256,\"type\":\"keyword\"}}},\"Intervals\":{\"properties\":{\"StopMarkerID\":{\"type\":\"keyword\"},\"GeologicUnitInterpretationIDs\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"null_value\":\"null\",\"ignore_above\":256,\"type\":\"keyword\"}}},\"StopMeasuredDepth\":{\"type\":\"double\"}}},\"Location\":{\"type\":\"geo_point\"}}},\"bagOfWords\":{\"store\":true,\"type\":\"text\",\"fields\":{\"autocomplete\":{\"type\":\"completion\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"tenant\"},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
private final String validKeywordLowerMapping = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"fields\":{\"keywordLower\":{\"normalizer\":\"lowercase\",\"null_value\":\"null\",\"ignore_above\":256,\"type\":\"keyword\"},\"keyword\":{\"null_value\":\"null\",\"ignore_above\":256,\"type\":\"keyword\"}}},\"Intervals\":{\"properties\":{\"StopMarkerID\":{\"type\":\"keyword\"},\"GeologicUnitInterpretationIDs\":{\"type\":\"text\",\"fields\":{\"keywordLower\":{\"normalizer\":\"lowercase\",\"null_value\":\"null\",\"ignore_above\":256,\"type\":\"keyword\"},\"keyword\":{\"null_value\":\"null\",\"ignore_above\":256,\"type\":\"keyword\"}}},\"StopMeasuredDepth\":{\"type\":\"double\"}}},\"Location\":{\"type\":\"geo_point\"}}},\"bagOfWords\":{\"store\":true,\"type\":\"text\",\"fields\":{\"autocomplete\":{\"type\":\"completion\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"tenant\"},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
private final String emptyDataValidMapping = "{\"dynamic\":false,\"properties\":{\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}},\"bagOfWords\":{\"store\":true,\"type\":\"text\",\"fields\":{\"autocomplete\":{\"type\":\"completion\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"tenant\"}}}";
private final String mapping = "{\"dynamic\":false,\"properties\":{\"ancestry\":{\"type\":\"object\",\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"data\":{\"type\":\"object\",\"properties\":{\"Address\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256,\"null_value\":\"null\"}}},\"Phone\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256,\"null_value\":\"null\"}}},\"Full Name\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256,\"null_value\":\"null\"}}}}},\"x-acl\":{\"type\":\"keyword\"},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"type\":\"object\",\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"namespace\":{\"type\":\"keyword\"},\"index\":{\"type\":\"object\",\"properties\":{\"trace\":{\"type\":\"text\"},\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"}}},\"acl\":{\"type\":\"object\",\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"tags\":{\"type\":\"flattened\"}}}";
@Mock
private RestClient restClient;
@Mock
......@@ -313,4 +318,19 @@ public class IndexerMappingServiceTest {
assertEquals(message, exception.getMessage());
}
@Test
public void testGetIndexMapping() throws Exception {
when(indicesService.isIndexExist(any(), any())).thenReturn(true);
when(restHighLevelClient.indices()).thenReturn(indicesClient);
TypeMapping typeMapping = TypeMapping.of(builder -> builder.withJson(new StringReader(mapping)));
GetMappingResponse getMappingResponse = GetMappingResponse.of(
responseBuilder -> responseBuilder.putResult(
"index", IndexMappingRecord.of(mappingRecordBuilder -> mappingRecordBuilder.mappings(typeMapping))
)
);
when(indicesClient.getMapping(any(GetMappingRequest.class))).thenReturn(getMappingResponse);
String actualMapping = this.sut.getIndexMapping(restHighLevelClient, "index");
assertEquals(mapping, actualMapping);
}
}
indexer-core/src/test/java/org/opengroup/osdu/indexer/service/MappingServiceImplTest.java deleted 100644 → 0
+ 0
− 104
View file @ 575e9075
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
package org.opengroup.osdu.is.core.service;
import org.apache.http.HttpStatus;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.client.IndicesClient;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
import org.opengroup.osdu.core.common.service.coreis.*;
import org.opengroup.osdu.is.core.util.ElasticClientHandler;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.*;
import static org.mockito.MockitoAnnotations.initMocks;
@RunWith(SpringRunner.class)
@PrepareForTest({RestHighLevelClient.class, IndicesClient.class, GetMappingsResponse.class})
public class MappingServiceImplTest {
@Mock
private IElasticSettingService elasticSettingService;
@Mock
private ElasticClientHandler elasticClientHandler;
@Mock
private ElasticIndexNameResolver elasticIndexNameResolver;
@Mock
private IndicesService indicesService;
@InjectMocks
private MappingServiceImpl sut;
private RestHighLevelClient restHighLevelClient;
private IndicesClient indicesClient;
private GetMappingsResponse mappingsResponse;
private String index = "anyindex";
@Before
public void setup() {
initMocks(this);
this.indicesClient = PowerMockito.mock(IndicesClient.class);
this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
this.mappingsResponse = PowerMockito.mock(GetMappingsResponse.class);
}
@Test
public void error_when_get_mapping_from_non_exists_elastic_index() throws Exception {
try {
when(this.indicesService.isIndexExist(restHighLevelClient, index)).thenReturn(false);
this.sut.getIndexMapping(restHighLevelClient, index);
fail("expected exception");
} catch (AppException ex) {
assertEquals(HttpStatus.SC_NOT_FOUND, ex.getError().getCode());
}
}
@Test(expected = AppException.class)
public void error_when_get_mapping_failed_from_existing_elastic_index() throws Exception {
when(this.indicesService.isIndexExist(this.restHighLevelClient, this.index)).thenReturn(true);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
doThrow(new IOException()).when(this.indicesClient).getMapping(any(), any(RequestOptions.class));
this.sut.getIndexMapping(this.restHighLevelClient, this.index);
fail("expected exception");
}
@Test
public void get_mapping_from_existing_elastic_index() throws Exception {
when(this.indicesService.isIndexExist(restHighLevelClient, index)).thenReturn(true);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
doReturn(this.mappingsResponse).when(this.indicesClient).getMapping(any(), any(RequestOptions.class));
doReturn("dummy").when(this.mappingsResponse).toString();
String mapping = this.sut.getIndexMapping(restHighLevelClient, index);
// TODO
assertEquals("dummy", mapping);
}
}*/
\ No newline at end of file
provider/indexer-gc/src/main/resources/application.properties
+ 3
− 3
View file @ 16d1b837
......@@ -65,9 +65,9 @@ reindex-topic-name=reindex
# Feature flag settings
featureFlag.strategy=appProperty
featureFlag.asIngestedCoordinates.enabled=false
featureFlag.keywordLower.enabled=false
featureFlag.bagOfWords.enabled=false
featureFlag.asIngestedCoordinates.enabled=true
featureFlag.keywordLower.enabled=true
featureFlag.bagOfWords.enabled=true
# Health checks
management.server.port=${MANAGEMENT_SERVER_PORT:8081}
......
testing/indexer-test-gc/src/test/resources/cucumber.properties
+ 1
− 1
View file @ 16d1b837
......@@ -17,4 +17,4 @@
# tag indexer-extended disabled by default
# tag as-ingested-coordinates disabled by default
# tag keyword-lower disabled by default
cucumber.options=--tags '@default'
cucumber.options=--tags '@as-ingested-coordinates,@default,@bag-of-words,@keyword-lower,@indexer-extended'
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment