There is a security vulnerability in SSH key-generation using GitKraken < v8.0.1. If you used this tool to create SSH keys, please update GitKraken and regenerate. If you need help with this, contact forum-support@opengroup.org

Commit a0554110 authored by Matt Wise's avatar Matt Wise
Browse files

Merge remote-tracking branch 'upstream/master' into aws-integration

parents 0d4b17c2 78ee7ec5
Pipeline #61232 failed with stages
in 52 minutes and 4 seconds
......@@ -19,17 +19,16 @@ variables:
# --- osdu gcp specific variables for cloudrun ---
OSDU_GCP_SERVICE: search
OSDU_GCP_VENDOR: gcp
OSDU_GCP_ENV_VARS: ENVIRONMENT=$OSDU_GCP_SPRING_PROFILES_ACTIVE,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_SEARCH_PORT=$OSDU_GCP_REDIS_SEARCH_PORT,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST_SEARCH,AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_V2_URL,ENTITLEMENTS_HOST=$OSDU_GCP_ENTITLEMENTS_V2_URL,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,PARTITION_API=$OSDU_GCP_PARTITION_API,POLICY_API=$OSDU_GCP_POLICY_API,POLICY_ID=search --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
# Variables should be partially removed after migration on cloudrun gcp
OSDU_GCP_BUILD_SUBDIR: provider/search-gcp
OSDU_GCP_INT_TEST_SUBDIR: testing/integration-tests/search-test-gcp
OSDU_GCP_APPLICATION_NAME: os-search
OSDU_GCP_PROJECT: nice-etching-277309
OSDU_GCP_TENANT_NAME: osdu
OSDU_GCP_STORAGE_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true'
OSDU_GCP_TEST_SUBDIR: testing/integration-tests/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
OSDU_GCP_HELM_CONFIG_SERVICE: search-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: search-deploy
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.log_level=INFO --set data.google_cloud_project_id=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_host=$OSDU_GCP_INDEXER_API --set data.authorize_api=$OSDU_GCP_AUTHORIZE_API --set data.audiences=$GOOGLE_AUDIENCE --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.policy_api=$OSDU_GCP_POLICY_API"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-search"
IBM_BUILD_SUBDIR: provider/search-ibm
IBM_INT_TEST_SUBDIR: testing/integration-tests/search-test-ibm
......@@ -63,7 +62,7 @@ include:
file: "cloud-providers/azure.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
file: 'cloud-providers/osdu-gcp-gke.yml'
- project: 'osdu/platform/ci-cd-pipelines'
file: 'publishing/pages.yml'
......
This diff is collapsed.
......@@ -65,7 +65,4 @@ go-swagger brings to the go community a complete suite of fully-featured, high-p
where indexpattern is the index pattern regular expression which you want to delete
indexpattern is defined in web.xml (in indexer) file with an environment variable as CRON_INDEX_CLEANUP_PATTERN
The scheduling of cron is done in the following repository:
https://slb-swt.visualstudio.com/data-management/_git/deployment-init-scripts?path=%2F3_post_deploy%2F1_appengine_cron%2Fcron.yaml&version=GBmaster
#New Update2
https://slb-swt.visualstudio.com/data-management/_git/deployment-init-scripts?path=%2F3_post_deploy%2F1_appengine_cron%2Fcron.yaml&version=GBmaster
\ No newline at end of file
......@@ -3,6 +3,8 @@ global:
# Service(s) Replica Count
replicaCount: 3
nodepool: services
isAutoscalingEnabled: false
################################################################################
# Specify the Gitlab branch being used for image creation
......@@ -14,4 +16,4 @@ image:
tag: #{Build.SourceVersion}#
policy:
enabled: #{POLICY_ENABLED}#
\ No newline at end of file
enabled: #{POLICY_ENABLED}#
......@@ -28,6 +28,10 @@ spec:
app: {{ .Chart.Name }}
aadpodidbinding: osdu-identity
spec:
{{- if .Values.global.isAutoscalingEnabled }}
nodeSelector:
nodepool: {{ .Values.global.nodepool }}
{{- end }}
volumes:
- name: azure-keyvault
csi:
......
......@@ -14,6 +14,8 @@
global:
replicaCount: 2
nodepool: services
isAutoscalingEnabled: false
image:
repository: community.opengroup.org:5555/osdu/platform/system/search-service
......
......@@ -19,3 +19,4 @@ data:
GOOGLE_AUDIENCES: "{{ .Values.data.audiences }}"
PARTITION_API: "{{ .Values.data.partition_api }}"
POLICY_API: "{{ .Values.data.policy_api }}"
POLICY_ID: "{{ .Values.data.policy_id }}"
......@@ -15,6 +15,7 @@ data:
audiences: ""
partition_api: ""
policy_api: ""
policy_id: "search"
conf:
configmap: "search-config"
......
......@@ -4,7 +4,7 @@
data:
requests_cpu: "0.25"
requests_memory: "128M"
requests_memory: "256M"
limits_cpu: "1"
limits_memory: "1G"
serviceAccountName: ""
......
......@@ -13,6 +13,8 @@ tags:
- name: search
description: Service endpoints to search data in datalake
- name: Search
- name: info
description: Version info endpoint
schemes:
- https
consumes:
......@@ -203,6 +205,21 @@ paths:
security:
- Bearer: []
- google_id_token: []
/info:
get:
tags:
- info
summary: "Version info"
description: "For deployment available public `/info` endpoint, \
\ which provides build and git related information."
operationId: "Version info"
produces:
- "application/json"
responses:
200:
description: "Version info."
schema:
$ref: "#/definitions/VersionInfo"
definitions:
AppError:
type: object
......@@ -395,6 +412,46 @@ definitions:
latitude: 36.450727
longitude: 122.174762
returnedFields: [Location]
VersionInfo:
type: "object"
properties:
groupId:
type: "string"
description: "Maven artifact group ID."
actifactId:
type: "string"
description: "Maven artifact ID."
version:
type: "string"
description: "Maven artifact version"
buildTime:
type: "string"
description: "Maven artifact build time"
branch:
type: "string"
description: "Current git branch"
commitId:
type: "string"
description: "Latest commit hash"
commitMessage:
type: "string"
description: "Latest commit message"
connectedOuterServices:
type: "array"
description: "Connected outer services information"
items:
$ref: "#/definitions/ConnectedOuterService"
description: "Version info."
ConnectedOuterService:
type: "object"
properties:
name:
type: "string"
description: "Connected outer service name."
version:
type: "string"
description: "Connected outer service version."
description: "Connected outer service information."
# This section is the general responses shared by all apis
GeneralResponse:
properties:
......
This diff is collapsed.
......@@ -26,6 +26,7 @@
- [Query With Cursor](#query-with-cursor)
- [Get indexing status](#get-indexing-status)
- [Permissions](#permissions)
- [Version info endpoint](#version-info-endpoint)
## Introduction <a name="introduction"></a>
......@@ -346,6 +347,8 @@ The syntax of those queries is the same we learned from the above sections.
The only distinction is that their conditions are scoped by the own fields of objects of the array,
pointed in the first argument of the current nested(path,(conditions)) function.
For more details, ability and limitation about this feature, please refer to [ArrayOfObjects](#docs/tutorial/ArrayOfObjects.md)
### Single-level one condition nested query
It queries for wellboremarkerset WPCs having any Marker with MarkerMeasuredDepth field value greater than 10000
```json
......@@ -435,10 +438,24 @@ E.g. Given
```
The above request payload asks search service to sort on "data.Id" in ascending order, and the expected response will have "totalCount: 10" (instead of 20, please note that the 10 returned records are only from common:welldb:wellbore:1.0.0 because the data.Id in common:welldb:well:1.0.0 is of data type string, which is not currently supported - and therefore, will not be returned) and should list the 5 records which have empty data.Id value at last.
**NOTE:** Search service does not validate the provided sort field, whether it exists or is of the supported data types. Different kinds may have attributes with the same names, but are different data types. Therefore, it is the user's responsibility to be aware and validate this in one's own workflow.
Search results are by default ordered by relevancy `_score` in descending order. Users are not required to provide any sort query for this. Users can also make request to query record in reverse relevancy order.
```json
{
"kind": "*:*:*:*",
"query": "well",
"sort": {
"field": ["_score"],
"order": ["ASC"]
}
}
```
**NOTE:** Search service does not validate the provided sort field, whether it exists or is of the supported data types. Different kinds may have attributes with the same names, but are different data types. Therefore, it is the user's responsibility to be aware and validate this in one's own workflow.
The sort query could be very expensive, especially if the given kind is too broad (e.g. "kind": "*:*:*:*"). The current time-out threshold is 60 seconds; a 504 error ("Request timed out after waiting for 1m") will be returned if the request times out. The suggestion is to make the kind parameter as narrow as possible while using the sort feature.
### Sort by "nested" arrays objects <a name="nested-sort"></a>
We generally have several objects in each "nested" array.
The question is, how to choose the properties of which of them will be used for sorting of the owning documents.
......@@ -457,6 +474,7 @@ For the first level we use 'min' mode and then ASC sorting order, for the second
}
```
For more details, ability and limitation about this feature, please refer to [ArrayOfObjects](#docs/tutorial/ArrayOfObjects.md)
## Aggregation <a name="aggregation"></a>
Aggregation feature leads to "aggregations" node appearing in the result output.
......@@ -506,6 +524,8 @@ This is why we see multiple "keys" with "count" > 1 in the "aggregations" result
]
}
For more details, ability and limitation about this feature, please refer to [ArrayOfObjects](#docs/tutorial/ArrayOfObjects.md)
## Range Queries <a name="range-queries"></a>
Ranges can be specified for date, numeric or string fields. Inclusive ranges are specified with square brackets `[min TO max]` and exclusive ranges with curly brackets `{min TO max}`. Here are some of the examples:
......@@ -979,3 +999,39 @@ The above query will return all records which had problems due to fields mismatc
| /api/search/v2/query_with_cursor | POST | users.osdu.viewers | Yes |
[Back to table of contents](#TOC)
## Version info endpoint
For deployment available public `/info` endpoint, which provides build and git related information.
#### Example response:
```json
{
"groupId": "org.opengroup.osdu",
"artifactId": "storage-gcp",
"version": "0.10.0-SNAPSHOT",
"buildTime": "2021-07-09T14:29:51.584Z",
"branch": "feature/GONRG-2681_Build_info",
"commitId": "7777",
"commitMessage": "Added copyright to version info properties file",
"connectedOuterServices": [
{
"name": "elasticSearch",
"version":"..."
},
{
"name": "postgresSql",
"version":"..."
},
{
"name": "redis",
"version":"..."
}
]
}
```
This endpoint takes information from files, generated by `spring-boot-maven-plugin`,
`git-commit-id-plugin` plugins. Need to specify paths for generated files to matching
properties:
- `version.info.buildPropertiesPath`
- `version.info.gitPropertiesPath`
[Back to table of contents](#TOC)
\ No newline at end of file
......@@ -44,7 +44,7 @@
<log4j-core.version>2.13.2</log4j-core.version>
<google-oauth-client.version>1.31.0</google-oauth-client.version>
<commons-compress.version>1.20</commons-compress.version>
<osdu.oscorecommon.version>0.10.0</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.11.0-SNAPSHOT</osdu.oscorecommon.version>
<tomcat-embed-core.version>9.0.45</tomcat-embed-core.version>
</properties>
......@@ -374,4 +374,44 @@
</profile>
</profiles>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<id>build-info</id>
<goals>
<goal>build-info</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>4.0.5</version>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
</execution>
</executions>
<configuration>
<verbose>true</verbose>
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<generateGitPropertiesFilename>
${project.build.outputDirectory}/git.properties
</generateGitPropertiesFilename>
</configuration>
</plugin>
</plugins>
</build>
</project>
......@@ -38,7 +38,8 @@
<project.main.basedir>${project.parent.basedir}</project.main.basedir>
<aws.version>1.11.1018</aws.version>
<deployment.environment>dev</deployment.environment>
<osdu.oscorecommon.version>0.10.0</osdu.oscorecommon.version>
<version.number>0.0.4-SNAPSHOT</version.number>
<osdu.oscorecommon.version>0.11.0-SNAPSHOT</osdu.oscorecommon.version>
</properties>
<dependencies>
......
......@@ -14,6 +14,7 @@
package org.opengroup.osdu.search.provider.aws.provider.impl;
import java.util.ArrayList;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
......@@ -45,6 +46,11 @@ public class CrossTenantInfoServiceAwsImpl implements ITenantInfoService, ICross
return tenantInfo;
}
@Override
public List<TenantInfo> getAllTenantInfos() {
return new ArrayList<>(tenantFactory.listTenantInfo());
}
@Override
public List<TenantInfo> getAllTenantsFromPartitionId() {
List<TenantInfo> tenantInfos = new LinkedList<>();
......
......@@ -117,7 +117,9 @@ public class ScrollQueryServiceAwsImpl extends QueryBase implements IScrollQuery
} catch (AppException e) {
throw e;
} catch (ElasticsearchStatusException e) {
if (e.status() == NOT_FOUND && e.getMessage().startsWith("No search context found for id"))
String invalidScrollMessage = "No search context found for id";
if (e.status() == NOT_FOUND
&& (e.getMessage().startsWith(invalidScrollMessage)) || this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains(invalidScrollMessage)))
throw new AppException(HttpStatus.SC_BAD_REQUEST, "Can't find the given cursor", "The given cursor is invalid or expired", e);
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Search error", "Error processing search request", e);
} catch (Exception e) {
......
......@@ -71,4 +71,4 @@ service.policy.id=search
PARTITION_API=${ENTITLEMENTS_BASE_URL}/api/partition/v1
aws.environment=${ENVIRONMENT}
spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.SecurityAutoConfiguration
\ No newline at end of file
spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.SecurityAutoConfiguration
......@@ -10,7 +10,7 @@ In order to run this service locally, you will need the following:
- [Maven 3.6.0+](https://maven.apache.org/download.cgi)
- [AdoptOpenJDK8](https://adoptopenjdk.net/)
- Infrastructure dependencies, deployable through the relevant [infrastructure template](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/infrastructure-templates?path=%2Finfra&version=GBmaster&_a=contents)
- Infrastructure dependencies, deployable through the relevant [infrastructure template](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/blob/master/docs/service-automation.md)
- While not a strict dependency, example commands in this document use [bash](https://www.gnu.org/software/bash/)
### General Tips
......
......@@ -37,7 +37,7 @@
<project.main.basedir>${project.parent.basedir}</project.main.basedir>
<springboot.version>2.1.7.RELEASE</springboot.version>
<osdu.corelibazure.version>0.10.1</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.10.0</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.11.0-SNAPSHOT</osdu.oscorecommon.version>
<osdu.search-core.version>0.11.0-SNAPSHOT</osdu.search-core.version>
<spatial4j.version>0.7</spatial4j.version>
<jts-io-common.version>1.15.0</jts-io-common.version>
......
......@@ -14,6 +14,7 @@ package org.opengroup.osdu.search.provider.azure.provider.impl;
// limitations under the License.
import java.util.ArrayList;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
......@@ -44,6 +45,11 @@ public class CrossTenantInfoServiceImpl implements ITenantInfoService, ICrossTen
return tenantInfo;
}
@Override
public List<TenantInfo> getAllTenantInfos() {
return new ArrayList<>(tenantFactory.listTenantInfo());
}
public String getPartitionId(){
return dpsHeaders.getPartitionId();
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment