Commit cbe70f52 authored by neelesh thakur's avatar neelesh thakur
Browse files

Merge branch 'score-sort' into 'master'

sort by relevancy score

See merge request !149
parents 8fd75e88 bf274be5
Pipeline #56521 failed with stages
in 41 minutes and 38 seconds
......@@ -77,7 +77,7 @@ The following software have components provided under the terms of this license:
- Brave Instrumentation: Http Adapters (from https://repo1.maven.org/maven2/io/zipkin/brave/brave-instrumentation-http)
- Brave instrumentation for Reactor Netty HTTP (from https://github.com/reactor/reactor-netty)
- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- Byte Buddy agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
- Cloud Storage JSON API v1-rev20210127-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
- Converter: Jackson (from https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson)
......@@ -123,7 +123,7 @@ The following software have components provided under the terms of this license:
- JAX-RS 2.1: The Java(TM) API for RESTful Web Services (from )
- JBoss Logging 3 (from http://www.jboss.org)
- JCIP Annotations under Apache License (from http://stephenc.github.com/jcip-annotations)
- JCL 1.2 implemented over SLF4J (from http://www.slf4j.org)
- JCL 1.1.1 implemented over SLF4J (from http://www.slf4j.org)
- JMES Path Query library (from https://aws.amazon.com/sdkforjava)
- JSON Small and Fast Parser (from https://repo1.maven.org/maven2/net/minidev/json-smart)
- JSON Web Token support for the JVM (from https://github.com/jwtk/jjwt)
......@@ -149,7 +149,7 @@ The following software have components provided under the terms of this license:
- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text)
- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names)
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access (from https://repo1.maven.org/maven2/net/java/dev/jna/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Java Servlet API (from http://servlet-spec.java.net)
- Java UUID Generator (from http://wiki.fasterxml.com/JugHome)
......@@ -258,17 +258,14 @@ The following software have components provided under the terms of this license:
- Retrofit (from https://github.com/square/retrofit)
- SnakeYAML (from http://code.google.com/p/snakeyaml/)
- Spatial4J (from https://projects.eclipse.org/projects/locationtech.spatial4j)
- Spring AOP (from https://github.com/spring-projects/spring-framework)
- Spring Beans (from https://github.com/spring-projects/spring-framework)
- Spring AOP (from https://repo1.maven.org/maven2/org/springframework/spring-aop)
- Spring Boot Data MongoDB Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-data-mongodb)
- Spring Boot Dependencies (from https://projects.spring.io/spring-boot/#)
- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json)
- Spring Boot Tomcat Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat)
- Spring Boot Web Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-web)
- Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework)
- Spring Context (from https://github.com/spring-projects/spring-framework)
- Spring Context Support (from https://github.com/spring-projects/spring-framework)
- Spring Core (from https://github.com/spring-projects/spring-framework)
- Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons)
- Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons)
- Spring Data KeyValue (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-keyvalue)
......@@ -283,9 +280,12 @@ The following software have components provided under the terms of this license:
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring TestContext Framework (from https://github.com/spring-projects/spring-framework)
- Spring Transaction (from https://github.com/spring-projects/spring-framework)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- Spring Web MVC (from https://github.com/spring-projects/spring-framework)
- Spring WebFlux (from https://github.com/spring-projects/spring-framework)
- Spring beans (from https://repo1.maven.org/maven2/org/springframework/spring-beans)
- Spring context (from https://repo1.maven.org/maven2/org/springframework/spring-context)
- Spring core (from https://repo1.maven.org/maven2/org/springframework/spring-core)
- Spring web (from https://repo1.maven.org/maven2/org/springframework/spring-web)
- Spring web MVC (from https://repo1.maven.org/maven2/org/springframework/spring-webmvc)
- T-Digest (from https://github.com/tdunning/t-digest)
- Woodstox (from https://github.com/FasterXML/woodstox)
- Zipkin Core Library (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin)
......@@ -372,11 +372,11 @@ The following software have components provided under the terms of this license:
- parent-join (from https://github.com/elastic/elasticsearch)
- parent-join (from https://github.com/elastic/elasticsearch)
- perfmark:perfmark-api (from https://github.com/perfmark/perfmark)
- powermock-api-support (from )
- powermock-api-support (from https://repo1.maven.org/maven2/org/powermock/powermock-api-support)
- powermock-core (from http://www.powermock.org)
- powermock-module-junit4 (from http://www.powermock.org)
- powermock-module-junit4-common (from )
- powermock-reflect (from )
- powermock-module-junit4-common (from https://repo1.maven.org/maven2/org/powermock/powermock-module-junit4-common)
- powermock-reflect (from https://repo1.maven.org/maven2/org/powermock/powermock-reflect)
- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
- proto-google-cloud-iamcredentials-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-iamcredentials-v1)
- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
......@@ -429,7 +429,7 @@ The following software have components provided under the terms of this license:
- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations)
- swagger-jaxrs (from )
- swagger-models (from https://repo1.maven.org/maven2/io/swagger/swagger-models)
- tomcat-annotations-api (from https://tomcat.apache.org/)
- tomcat-annotations-api (from https://repo1.maven.org/maven2/org/apache/tomcat/tomcat-annotations-api)
- tomcat-embed-core (from http://tomcat.apache.org/)
- tomcat-embed-el (from https://tomcat.apache.org/)
- tomcat-embed-websocket (from https://tomcat.apache.org/)
......@@ -488,8 +488,8 @@ The following software have components provided under the terms of this license:
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
- Reflections (from http://github.com/ronmamo/reflections)
- SnakeYAML (from http://code.google.com/p/snakeyaml/)
- Spring Core (from https://github.com/spring-projects/spring-framework)
- Spring Data KeyValue (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-keyvalue)
- Spring core (from https://repo1.maven.org/maven2/org/springframework/spring-core)
- ThreeTen backport (from https://www.threeten.org/threetenbp)
- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
......@@ -600,11 +600,11 @@ The following software have components provided under the terms of this license:
- Common Annotations 1.3 API (from )
- JAX-RS 2.1: The Java(TM) API for RESTful Web Services (from )
- Logback Classic Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-classic)
- Logback Classic Module (from http://logback.qos.ch)
- Logback Contrib :: JSON :: Classic (from )
- Logback Contrib :: JSON :: Core (from )
- Logback Contrib :: Jackson (from )
- Logback Core Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-core)
- Logback Core Module (from http://logback.qos.ch)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
......@@ -740,14 +740,14 @@ The following software have components provided under the terms of this license:
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access (from https://repo1.maven.org/maven2/net/java/dev/jna/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Javassist (from http://www.javassist.org/)
- Logback Classic Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-classic)
- Logback Classic Module (from http://logback.qos.ch)
- Logback Contrib :: JSON :: Classic (from )
- Logback Contrib :: JSON :: Core (from )
- Logback Contrib :: Jackson (from )
- Logback Core Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-core)
- Logback Core Module (from http://logback.qos.ch)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
......@@ -791,7 +791,7 @@ The following software have components provided under the terms of this license:
- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
- Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions)
- JCL 1.2 implemented over SLF4J (from http://www.slf4j.org)
- JCL 1.1.1 implemented over SLF4J (from http://www.slf4j.org)
- JOpt Simple (from http://pholser.github.io/jopt-simple)
- JUL to SLF4J bridge (from http://www.slf4j.org)
- Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
......@@ -934,7 +934,7 @@ The following software have components provided under the terms of this license:
- Project Lombok (from https://projectlombok.org)
- RESTEasy JAX-RS Implementation (from )
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- Spring web (from https://repo1.maven.org/maven2/org/springframework/spring-web)
- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- reactive-streams (from http://www.reactive-streams.org/)
......
......@@ -437,10 +437,24 @@ E.g. Given
```
The above request payload asks search service to sort on "data.Id" in ascending order, and the expected response will have "totalCount: 10" (instead of 20, please note that the 10 returned records are only from common:welldb:wellbore:1.0.0 because the data.Id in common:welldb:well:1.0.0 is of data type string, which is not currently supported - and therefore, will not be returned) and should list the 5 records which have empty data.Id value at last.
**NOTE:** Search service does not validate the provided sort field, whether it exists or is of the supported data types. Different kinds may have attributes with the same names, but are different data types. Therefore, it is the user's responsibility to be aware and validate this in one's own workflow.
Search results are by default ordered by relevancy `_score` in descending order. Users are not required to provide any sort query for this. Users can also make request to query record in reverse relevancy order.
```json
{
"kind": "*:*:*:*",
"query": "well",
"sort": {
"field": ["_score"],
"order": ["ASC"]
}
}
```
**NOTE:** Search service does not validate the provided sort field, whether it exists or is of the supported data types. Different kinds may have attributes with the same names, but are different data types. Therefore, it is the user's responsibility to be aware and validate this in one's own workflow.
The sort query could be very expensive, especially if the given kind is too broad (e.g. "kind": "*:*:*:*"). The current time-out threshold is 60 seconds; a 504 error ("Request timed out after waiting for 1m") will be returned if the request times out. The suggestion is to make the kind parameter as narrow as possible while using the sort feature.
### Sort by "nested" arrays objects <a name="nested-sort"></a>
We generally have several objects in each "nested" array.
The question is, how to choose the properties of which of them will be used for sorting of the owning documents.
......
package org.opengroup.osdu.search.util;
import java.io.IOException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import joptsimple.internal.Strings;
import org.apache.http.HttpStatus;
import org.elasticsearch.client.RestHighLevelClient;
......@@ -18,11 +13,21 @@ import org.opengroup.osdu.search.service.IFieldMappingTypeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Component
public class SortParserUtil implements ISortParserUtil {
private static final String SCORE_FIELD = "_score";
private static final String BAD_SORT_MESSAGE =
"Must be in format: nested(<path>, <field>, <mode>) OR nested(<parent_path>, .....nested(<child_path>, <field>, <mode>))";
"Must be in format: nested(<path>, <field>, <mode>) OR nested(<parent_path>, .....nested(<child_path>, <field>, <mode>))";
private static final String PATH_GROUP = "path";
private static final String FIELD_GROUP = "field";
private static final String MODE_GROUP = "mode";
......@@ -40,17 +45,21 @@ public class SortParserUtil implements ISortParserUtil {
if (sortString.contains("nested(")) {
return parseNestedSort(sortString, sortOrder);
} else {
if (sortString.equalsIgnoreCase(SCORE_FIELD)) {
return new FieldSortBuilder(sortString)
.order(SortOrder.fromString(sortOrder));
}
return new FieldSortBuilder(sortString)
.order(SortOrder.fromString(sortOrder))
.missing("_last")
.unmappedType("keyword");
.order(SortOrder.fromString(sortOrder))
.missing("_last")
.unmappedType("keyword");
}
}
public List<FieldSortBuilder> getSortQuery(RestHighLevelClient restClient, SortQuery sortQuery, String indexPattern) throws IOException {
List<String> dataFields = new ArrayList<>();
for (String field: sortQuery.getField()) {
if(field.startsWith("data.")) dataFields.add(field + ".keyword");
for (String field : sortQuery.getField()) {
if (field.startsWith("data.")) dataFields.add(field + ".keyword");
}
if (dataFields.isEmpty()) {
......@@ -107,11 +116,11 @@ public class SortParserUtil implements ISortParserUtil {
nestedSortBuilder = new NestedSortBuilder(path);
}
return new FieldSortBuilder(path + "." + field)
.setNestedSort(nestedSortBuilder)
.sortMode(SortMode.fromString(mode))
.order(SortOrder.fromString(sortOrder))
.missing("_last")
.unmappedType("keyword");
.setNestedSort(nestedSortBuilder)
.sortMode(SortMode.fromString(mode))
.order(SortOrder.fromString(sortOrder))
.missing("_last")
.unmappedType("keyword");
}
throw new AppException(HttpStatus.SC_BAD_REQUEST, String.format("Malformed nested sort : %s", sortString), BAD_SORT_MESSAGE);
}
......@@ -129,7 +138,7 @@ public class SortParserUtil implements ISortParserUtil {
return new NestedSortBuilder(path);
}
throw new AppException(HttpStatus.SC_BAD_REQUEST, String.format("Malformed nested sort group : %s", group),
BAD_SORT_MESSAGE);
BAD_SORT_MESSAGE);
}
}
......@@ -53,6 +53,15 @@ public class SortParserUtilTest {
assertEquals(expectedSortBuilder, actualFieldSortBuilder);
}
@Test
public void testScoreSortString() {
String SCORE_FIELD = "_score";
FieldSortBuilder actualFieldSortBuilder = sortParserUtil.parseSort(SCORE_FIELD, order);
FieldSortBuilder expectedSortBuilder = new FieldSortBuilder(SCORE_FIELD)
.order(SortOrder.valueOf(order));
assertEquals(expectedSortBuilder, actualFieldSortBuilder);
}
@Test
public void testSimpleNestedSortString() {
FieldSortBuilder actualFileSortBuilder = sortParserUtil.parseSort(simpleNestedSortString, order);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment