From 8c026ff3b0eb54e702d94172e0eb6a6efe992f16 Mon Sep 17 00:00:00 2001
From: "Aliaksandr Ramanovich (EPAM)" <aliaksandr_ramanovich1@epam.com>
Date: Wed, 25 Jan 2023 11:31:35 +0000
Subject: [PATCH] Gonrg 5894 gc rename
---
.fossa.yml | 56 +-
.gitlab-ci.yml | 4 +-
README.md | 4 +-
devops/azure/development-pipeline.yml | 44 +-
devops/azure/pipeline.yml | 44 +-
devops/{gcp => gc}/deploy/Chart.yaml | 2 +-
devops/{gcp => gc}/deploy/README.md | 4 +-
.../deploy/templates/configmap.yaml | 0
.../deploy/templates/deploy-redis.yaml | 0
.../deploy/templates/deployment.yaml | 0
.../deploy/templates/service-account.yaml | 0
.../deploy/templates/service-redis.yaml | 0
.../{gcp => gc}/deploy/templates/service.yaml | 0
.../deploy/templates/virtual-service.yaml | 0
devops/{gcp => gc}/deploy/values.yaml | 2 +-
devops/gc/pipeline/override-stages.yml | 11 +
devops/gcp/pipeline/override-stages.yml | 11 -
devops/gke-search-cicd.yml | 33 +-
devops/service-pipeline.yml | 34 +-
docs/tutorial/SearchService.md | 132 +-
pom.xml | 2 +-
provider/{search-gcp => search-gc}/README.md | 8 +-
.../cloudbuild/Dockerfile.cloudbuild | 0
.../cloudbuild/cloudbuild.yaml | 29 +-
.../docs/anthos/README.md | 21 +-
.../docs/gcp => search-gc/docs/gc}/README.md | 24 +-
provider/{search-gcp => search-gc}/pom.xml | 2 +-
.../src/main/appengine/app.yaml | 0
.../provider/gcp/SearchGcpApplication.java | 0
.../gcp/auth/GcpAuthorizationService.java | 0
.../provider/gcp/cache/CacheConfig.java | 0
.../provider/gcp/cache/CursorCacheImpl.java | 0
.../gcp/cache/ElasticCredentialsCache.java | 0
.../gcp/cache/FieldTypeMappingCache.java | 0
.../GcpSearchConfigurationProperties.java | 0
.../provider/gcp/di/ElasticSearchConfig.java | 0
.../ElasticSearchConfigurationProperties.java | 0
.../gcp/di/EntitlementsClientFactory.java | 0
.../gcp/di/GcpServiceAccountJwtClient.java | 0
.../impl/CrossTenantInfoServiceImpl.java | 0
.../provider/gcp/provider/impl/QueryBase.java | 940 ++++-----
.../gcp/provider/impl/QueryServiceImpl.java | 0
.../provider/impl/ScrollQueryServiceImpl.java | 0
.../gcp/security/GcpSecurityConfig.java | 0
.../service/ElasticSettingServiceImpl.java | 0
.../provider/gcp/utils/ConfigGcpModule.java | 0
.../provider/gcp/utils/TraceIdExtractor.java | 0
.../resources/application-anthos.properties | 0
.../main/resources/application-gcp.properties | 0
.../src/main/resources/application.properties | 0
.../src/main/resources/logback.xml | 0
.../RedirectHttpRequestsHandlerTest.java | 0
.../impl/CrossTenantInfoServiceTest.java | 0
.../gcp/provider/impl/QueryServiceTest.java | 1764 ++++++++---------
.../provider/impl/ScrollQueryServiceTest.java | 0
.../gcp/smart/filters/AllFiltersTest.java | 0
.../smart/filters/AttributeFilterTest.java | 0
.../gcp/smart/filters/SourceFilterTest.java | 0
.../gcp/smart/filters/TypeFilterTest.java | 0
.../elasticresponses/spatialfilters/1.json | 0
.../elasticresponses/spatialfilters/2.json | 0
.../deployment-os-search-service.yml | 35 -
provider/search-gcp/scripts/deploy.sh | 61 -
testing/integration-tests/pom.xml | 2 +-
.../pom.xml | 2 +-
.../osdu/step_definitions/info/RunTest.java | 0
.../osdu/step_definitions/info/Steps.java | 0
.../query/singlecluster/RunTest.java | 0
.../query/singlecluster/Steps.java | 0
.../querybycursor/singlecluster/RunTest.java | 0
.../querybycursor/singlecluster/Steps.java | 0
.../opengroup/osdu/util/GCPHTTPClient.java | 0
.../org/opengroup/osdu/util/JwtTokenUtil.java | 0
73 files changed, 1596 insertions(+), 1675 deletions(-)
rename devops/{gcp => gc}/deploy/Chart.yaml (97%)
rename devops/{gcp => gc}/deploy/README.md (98%)
rename devops/{gcp => gc}/deploy/templates/configmap.yaml (100%)
rename devops/{gcp => gc}/deploy/templates/deploy-redis.yaml (100%)
rename devops/{gcp => gc}/deploy/templates/deployment.yaml (100%)
rename devops/{gcp => gc}/deploy/templates/service-account.yaml (100%)
rename devops/{gcp => gc}/deploy/templates/service-redis.yaml (100%)
rename devops/{gcp => gc}/deploy/templates/service.yaml (100%)
rename devops/{gcp => gc}/deploy/templates/virtual-service.yaml (100%)
rename devops/{gcp => gc}/deploy/values.yaml (95%)
create mode 100644 devops/gc/pipeline/override-stages.yml
delete mode 100644 devops/gcp/pipeline/override-stages.yml
rename provider/{search-gcp => search-gc}/README.md (93%)
rename provider/{search-gcp => search-gc}/cloudbuild/Dockerfile.cloudbuild (100%)
rename provider/{search-gcp => search-gc}/cloudbuild/cloudbuild.yaml (51%)
rename provider/{search-gcp => search-gc}/docs/anthos/README.md (95%)
rename provider/{search-gcp/docs/gcp => search-gc/docs/gc}/README.md (95%)
rename provider/{search-gcp => search-gc}/pom.xml (99%)
rename provider/{search-gcp => search-gc}/src/main/appengine/app.yaml (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/SearchGcpApplication.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/auth/GcpAuthorizationService.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CacheConfig.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CursorCacheImpl.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/ElasticCredentialsCache.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/FieldTypeMappingCache.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/config/GcpSearchConfigurationProperties.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfig.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfigurationProperties.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/di/EntitlementsClientFactory.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/di/GcpServiceAccountJwtClient.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceImpl.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryBase.java (98%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceImpl.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceImpl.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/security/GcpSecurityConfig.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/service/ElasticSettingServiceImpl.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/ConfigGcpModule.java (100%)
rename provider/{search-gcp => search-gc}/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/TraceIdExtractor.java (100%)
rename provider/{search-gcp => search-gc}/src/main/resources/application-anthos.properties (100%)
rename provider/{search-gcp => search-gc}/src/main/resources/application-gcp.properties (100%)
rename provider/{search-gcp => search-gc}/src/main/resources/application.properties (100%)
rename provider/{search-gcp => search-gc}/src/main/resources/logback.xml (100%)
rename provider/{search-gcp => search-gc}/src/test/java/org/opengroup/osdu/search/provider/gcp/middleware/RedirectHttpRequestsHandlerTest.java (100%)
rename provider/{search-gcp => search-gc}/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceTest.java (100%)
rename provider/{search-gcp => search-gc}/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceTest.java (97%)
rename provider/{search-gcp => search-gc}/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceTest.java (100%)
rename provider/{search-gcp => search-gc}/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AllFiltersTest.java (100%)
rename provider/{search-gcp => search-gc}/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AttributeFilterTest.java (100%)
rename provider/{search-gcp => search-gc}/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/SourceFilterTest.java (100%)
rename provider/{search-gcp => search-gc}/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/TypeFilterTest.java (100%)
rename provider/{search-gcp => search-gc}/src/test/resources/elasticresponses/spatialfilters/1.json (100%)
rename provider/{search-gcp => search-gc}/src/test/resources/elasticresponses/spatialfilters/2.json (100%)
delete mode 100644 provider/search-gcp/kubernetes/deployments/deployment-os-search-service.yml
delete mode 100644 provider/search-gcp/scripts/deploy.sh
rename testing/integration-tests/{search-test-gcp => search-test-gc}/pom.xml (99%)
rename testing/integration-tests/{search-test-gcp => search-test-gc}/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java (100%)
rename testing/integration-tests/{search-test-gcp => search-test-gc}/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java (100%)
rename testing/integration-tests/{search-test-gcp => search-test-gc}/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/RunTest.java (100%)
rename testing/integration-tests/{search-test-gcp => search-test-gc}/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/Steps.java (100%)
rename testing/integration-tests/{search-test-gcp => search-test-gc}/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/RunTest.java (100%)
rename testing/integration-tests/{search-test-gcp => search-test-gc}/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/Steps.java (100%)
rename testing/integration-tests/{search-test-gcp => search-test-gc}/src/test/java/org/opengroup/osdu/util/GCPHTTPClient.java (100%)
rename testing/integration-tests/{search-test-gcp => search-test-gc}/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java (100%)
diff --git a/.fossa.yml b/.fossa.yml
index 132394783..2548b8d33 100644
--- a/.fossa.yml
+++ b/.fossa.yml
@@ -8,31 +8,31 @@ cli:
project: Search
analyze:
modules:
- - name: os-search
- type: mvn
- target: pom.xml
- path: .
- - name: search-core
- type: mvn
- target: search-core/pom.xml
- path: .
- - name: search-byoc
- type: mvn
- target: provider/search-byoc/pom.xml
- path: .
- - name: search-gcp
- type: mvn
- target: provider/search-gcp/pom.xml
- path: .
- - name: search-aws
- type: mvn
- target: provider/search-aws/pom.xml
- path: .
- - name: search-azure
- type: mvn
- target: provider/search-azure/pom.xml
- path: .
- - name: search-ibm
- type: mvn
- target: provider/search-ibm/pom.xml
- path: .
+ - name: os-search
+ type: mvn
+ target: pom.xml
+ path: .
+ - name: search-core
+ type: mvn
+ target: search-core/pom.xml
+ path: .
+ - name: search-byoc
+ type: mvn
+ target: provider/search-byoc/pom.xml
+ path: .
+ - name: search-gc
+ type: mvn
+ target: provider/search-gc/pom.xml
+ path: .
+ - name: search-aws
+ type: mvn
+ target: provider/search-aws/pom.xml
+ path: .
+ - name: search-azure
+ type: mvn
+ target: provider/search-azure/pom.xml
+ path: .
+ - name: search-ibm
+ type: mvn
+ target: provider/search-ibm/pom.xml
+ path: .
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 54983b2d5..6ab16e7e4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,9 +54,9 @@ include:
file: "cloud-providers/azure.yml"
- project: "osdu/platform/ci-cd-pipelines"
- file: "cloud-providers/osdu-gcp-global.yml"
+ file: "cloud-providers/gc-global.yml"
- - local: "devops/gcp/pipeline/override-stages.yml"
+ - local: "devops/gc/pipeline/override-stages.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "publishing/pages.yml"
diff --git a/README.md b/README.md
index 4ae65b0ac..c7339921e 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ All documentation for the Azure implementation of `os-search` lives [here](./pro
## Google Cloud Implementation
-All documentation for the GC implementation of `os-search` lives [here](./provider/search-gcp/README.md)
+All documentation for the GC implementation of `os-search` lives [here](./provider/search-gc/README.md)
## AWS Implementation
@@ -69,4 +69,4 @@ go-swagger brings to the go community a complete suite of fully-featured, high-p
where indexpattern is the index pattern regular expression which you want to delete
indexpattern is defined in web.xml (in indexer) file with an environment variable as CRON_INDEX_CLEANUP_PATTERN
The scheduling of cron is done in the following repository:
- https://slb-swt.visualstudio.com/data-management/_git/deployment-init-scripts?path=%2F3_post_deploy%2F1_appengine_cron%2Fcron.yaml&version=GBmaster
\ No newline at end of file
+ https://slb-swt.visualstudio.com/data-management/_git/deployment-init-scripts?path=%2F3_post_deploy%2F1_appengine_cron%2Fcron.yaml&version=GBmaster
diff --git a/devops/azure/development-pipeline.yml b/devops/azure/development-pipeline.yml
index 3570ee110..a7493b323 100644
--- a/devops/azure/development-pipeline.yml
+++ b/devops/azure/development-pipeline.yml
@@ -24,21 +24,21 @@ trigger:
- /docs
- /indexer/search-aws
- /indexer/search-byoc
- - /indexer/search-gcp
+ - /indexer/search-gc
- /indexer/search-ibm
resources:
repositories:
- - repository: FluxRepo
- type: git
- name: k8-gitops-manifests
- - repository: TemplateRepo
- type: git
- name: infra-azure-provisioning
+ - repository: FluxRepo
+ type: git
+ name: k8-gitops-manifests
+ - repository: TemplateRepo
+ type: git
+ name: infra-azure-provisioning
variables:
- - group: 'Azure - OSDU'
- - group: 'Azure - OSDU Secrets'
+ - group: "Azure - OSDU"
+ - group: "Azure - OSDU Secrets"
- name: serviceName
value: "search-service"
@@ -46,39 +46,39 @@ variables:
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- - name: 'MANIFEST_REPO'
+ - name: "MANIFEST_REPO"
value: $[ resources.repositories['FluxRepo'].name ]
- - name: 'MAVEN_CACHE_FOLDER'
+ - name: "MAVEN_CACHE_FOLDER"
value: $(Pipeline.Workspace)/.m2/repository
- name: SKIP_TESTS
- value: 'false'
+ value: "false"
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
- mavenGoal: 'package'
+ mavenGoal: "package"
mavenPublishJUnitResults: true
- serviceCoreMavenOptions: '-P search-core -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
- mavenOptions: '-P search-azure -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+ serviceCoreMavenOptions: "-P search-core -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)"
+ mavenOptions: "-P search-azure -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)"
copyFileContents: |
pom.xml
provider/search-azure/maven/settings.xml
provider/search-azure/pom.xml
provider/search-azure/target/*-spring-boot.jar
- copyFileContentsToFlatten: ''
- mavenSettingsFile: './maven/settings.xml'
+ copyFileContentsToFlatten: ""
+ mavenSettingsFile: "./maven/settings.xml"
serviceBase: ${{ variables.serviceName }}
- testingRootFolder: 'testing'
+ testingRootFolder: "testing"
chartPath: ${{ variables.chartPath }}
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
- testCoreMavenPomFile: 'testing/integration-tests/search-test-core/pom.xml'
- testCoreMavenOptions: ''
+ testCoreMavenPomFile: "testing/integration-tests/search-test-core/pom.xml"
+ testCoreMavenOptions: ""
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
providers:
- - name: Azure
- environments: ['dev']
+ - name: Azure
+ environments: ["dev"]
diff --git a/devops/azure/pipeline.yml b/devops/azure/pipeline.yml
index 4a914159a..927674f61 100644
--- a/devops/azure/pipeline.yml
+++ b/devops/azure/pipeline.yml
@@ -24,21 +24,21 @@ trigger:
- /docs
- /indexer/search-aws
- /indexer/search-byoc
- - /indexer/search-gcp
+ - /indexer/search-gc
- /indexer/search-ibm
resources:
repositories:
- - repository: FluxRepo
- type: git
- name: k8-gitops-manifests
- - repository: TemplateRepo
- type: git
- name: infra-azure-provisioning
+ - repository: FluxRepo
+ type: git
+ name: k8-gitops-manifests
+ - repository: TemplateRepo
+ type: git
+ name: infra-azure-provisioning
variables:
- - group: 'Azure - OSDU'
- - group: 'Azure - OSDU Secrets'
+ - group: "Azure - OSDU"
+ - group: "Azure - OSDU Secrets"
- name: serviceName
value: "search-service"
@@ -46,39 +46,39 @@ variables:
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- - name: 'MANIFEST_REPO'
+ - name: "MANIFEST_REPO"
value: $[ resources.repositories['FluxRepo'].name ]
- - name: 'MAVEN_CACHE_FOLDER'
+ - name: "MAVEN_CACHE_FOLDER"
value: $(Pipeline.Workspace)/.m2/repository
- name: SKIP_TESTS
- value: 'false'
+ value: "false"
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
- mavenGoal: 'package'
+ mavenGoal: "package"
mavenPublishJUnitResults: true
- serviceCoreMavenOptions: '-P search-core -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
- mavenOptions: '-P search-azure -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+ serviceCoreMavenOptions: "-P search-core -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)"
+ mavenOptions: "-P search-azure -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)"
copyFileContents: |
pom.xml
provider/search-azure/maven/settings.xml
provider/search-azure/pom.xml
provider/search-azure/target/*-spring-boot.jar
- copyFileContentsToFlatten: ''
- mavenSettingsFile: './maven/settings.xml'
+ copyFileContentsToFlatten: ""
+ mavenSettingsFile: "./maven/settings.xml"
serviceBase: ${{ variables.serviceName }}
- testingRootFolder: 'testing'
+ testingRootFolder: "testing"
chartPath: ${{ variables.chartPath }}
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
- testCoreMavenPomFile: 'testing/integration-tests/search-test-core/pom.xml'
- testCoreMavenOptions: ''
+ testCoreMavenPomFile: "testing/integration-tests/search-test-core/pom.xml"
+ testCoreMavenOptions: ""
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
providers:
- - name: Azure
- environments: ['demo']
+ - name: Azure
+ environments: ["demo"]
diff --git a/devops/gcp/deploy/Chart.yaml b/devops/gc/deploy/Chart.yaml
similarity index 97%
rename from devops/gcp/deploy/Chart.yaml
rename to devops/gc/deploy/Chart.yaml
index db0e807f3..4fc3e55a6 100644
--- a/devops/gcp/deploy/Chart.yaml
+++ b/devops/gc/deploy/Chart.yaml
@@ -1,5 +1,5 @@
apiVersion: v2
-name: gcp-search-deploy
+name: gc-search-deploy
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
diff --git a/devops/gcp/deploy/README.md b/devops/gc/deploy/README.md
similarity index 98%
rename from devops/gcp/deploy/README.md
rename to devops/gc/deploy/README.md
index eddd81351..4e5dba421 100644
--- a/devops/gcp/deploy/README.md
+++ b/devops/gc/deploy/README.md
@@ -78,7 +78,7 @@ You need to set variables in **values.yaml** file using any code editor. Some of
Run this command from within this directory:
```console
-helm install gcp-search-deploy .
+helm install gc-search-deploy .
```
## Uninstall the Helm chart
@@ -86,7 +86,7 @@ helm install gcp-search-deploy .
To uninstall the helm deployment:
```console
-helm uninstall gcp-search-deploy
+helm uninstall gc-search-deploy
```
> Do not forget to delete all k8s secrets and PVCs accociated with the Service.
diff --git a/devops/gcp/deploy/templates/configmap.yaml b/devops/gc/deploy/templates/configmap.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/configmap.yaml
rename to devops/gc/deploy/templates/configmap.yaml
diff --git a/devops/gcp/deploy/templates/deploy-redis.yaml b/devops/gc/deploy/templates/deploy-redis.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/deploy-redis.yaml
rename to devops/gc/deploy/templates/deploy-redis.yaml
diff --git a/devops/gcp/deploy/templates/deployment.yaml b/devops/gc/deploy/templates/deployment.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/deployment.yaml
rename to devops/gc/deploy/templates/deployment.yaml
diff --git a/devops/gcp/deploy/templates/service-account.yaml b/devops/gc/deploy/templates/service-account.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/service-account.yaml
rename to devops/gc/deploy/templates/service-account.yaml
diff --git a/devops/gcp/deploy/templates/service-redis.yaml b/devops/gc/deploy/templates/service-redis.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/service-redis.yaml
rename to devops/gc/deploy/templates/service-redis.yaml
diff --git a/devops/gcp/deploy/templates/service.yaml b/devops/gc/deploy/templates/service.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/service.yaml
rename to devops/gc/deploy/templates/service.yaml
diff --git a/devops/gcp/deploy/templates/virtual-service.yaml b/devops/gc/deploy/templates/virtual-service.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/virtual-service.yaml
rename to devops/gc/deploy/templates/virtual-service.yaml
diff --git a/devops/gcp/deploy/values.yaml b/devops/gc/deploy/values.yaml
similarity index 95%
rename from devops/gcp/deploy/values.yaml
rename to devops/gc/deploy/values.yaml
index ffccc25bb..42b7a58fa 100644
--- a/devops/gcp/deploy/values.yaml
+++ b/devops/gc/deploy/values.yaml
@@ -19,7 +19,7 @@ data:
# redis connection. Change it if you want to use external redis
redisSearchHost: ""
redisSearchPort: 6379
- redisImage: "redis:7"
+ redisImage: "docker.io/library/redis:7"
conf:
appName: "search"
elasticSecretName: "search-elastic-secret"
diff --git a/devops/gc/pipeline/override-stages.yml b/devops/gc/pipeline/override-stages.yml
new file mode 100644
index 000000000..ce6501487
--- /dev/null
+++ b/devops/gc/pipeline/override-stages.yml
@@ -0,0 +1,11 @@
+variables:
+ GC_SERVICE: search
+ GC_VENDOR: gc
+ GC_TESTING_DIR: "testing/integration-tests"
+ # FIXME remove when all services are migrated to a single helm
+ GC_ENABLE_HELM_CONFIG: "false"
+
+gc-anthos-test:
+ variables:
+ GC_VENDOR: anthos
+ DEFAULT_DATA_PARTITION_ID_TENANT2: non-exist
diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml
deleted file mode 100644
index 65533dce5..000000000
--- a/devops/gcp/pipeline/override-stages.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-variables:
- OSDU_GCP_SERVICE: search
- OSDU_GCP_VENDOR: gcp
- OSDU_GCP_TESTING_DIR: "testing/integration-tests"
- # FIXME remove when all services are migrated to a single helm
- OSDU_GCP_ENABLE_HELM_CONFIG: "false"
-
-osdu-gcp-anthos-test:
- variables:
- OSDU_GCP_VENDOR: anthos
- DEFAULT_DATA_PARTITION_ID_TENANT2: non-exist
diff --git a/devops/gke-search-cicd.yml b/devops/gke-search-cicd.yml
index 1485fa87e..6fdd61344 100644
--- a/devops/gke-search-cicd.yml
+++ b/devops/gke-search-cicd.yml
@@ -17,17 +17,16 @@ pool:
demands: maven
variables:
- buildMavenModules: search-core,provider/search-gcp
- dockerDir: provider/search-gcp/docker
+ buildMavenModules: search-core,provider/search-gc
+ dockerDir: provider/search-gc/docker
imageName: os-search-app
- deploymentDir: provider/search-gcp/kubernetes/deployments
+ deploymentDir: provider/search-gc/kubernetes/deployments
deploymentFile: deployment-os-search-service.yml
mavenSettings: ./search-core/maven/settings.xml
integrationTestCorePom: testing/integration-tests/search-test-core/pom.xml
- integrationTestGcpPom: testing/integration-tests/search-test-gcp/pom.xml
+ integrationTestGcpPom: testing/integration-tests/search-test-gc/pom.xml
steps:
-
- task: DownloadSecureFile@1
name: gcrKey
inputs:
@@ -40,11 +39,11 @@ steps:
- task: Maven@3
inputs:
- mavenPomFile: 'pom.xml'
- options: '--settings $(mavenSettings) -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -pl $(buildMavenModules) package'
+ mavenPomFile: "pom.xml"
+ options: "--settings $(mavenSettings) -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -pl $(buildMavenModules) package"
publishJUnitResults: false
- javaHomeOption: 'JDKVersion'
- mavenVersionOption: 'Default'
+ javaHomeOption: "JDKVersion"
+ mavenVersionOption: "Default"
mavenAuthenticateFeed: false
effectivePomSkip: false
sonarQubeRunAnalysis: false
@@ -80,22 +79,22 @@ steps:
- task: Maven@3
inputs:
- mavenPomFile: '$(integrationTestCorePom)'
- options: '--settings $(mavenSettings) -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) install'
+ mavenPomFile: "$(integrationTestCorePom)"
+ options: "--settings $(mavenSettings) -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) install"
publishJUnitResults: false
- javaHomeOption: 'JDKVersion'
- mavenVersionOption: 'Default'
+ javaHomeOption: "JDKVersion"
+ mavenVersionOption: "Default"
mavenAuthenticateFeed: false
effectivePomSkip: false
sonarQubeRunAnalysis: false
- task: Maven@3
inputs:
- mavenPomFile: '$(integrationTestGcpPom)'
- options: '--settings $(mavenSettings) -Dsurefire.useFile=false -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -DDEFAULT_DATA_PARTITION_ID_TENANT1=$(DEFAULT_DATA_PARTITION_ID_TENANT1) -DDEFAULT_DATA_PARTITION_ID_TENANT2=$(DEFAULT_DATA_PARTITION_ID_TENANT2) -DELASTIC_HOST=$(ELASTIC_HOST) -DELASTIC_PASSWORD=$(ELASTIC_PASSWORD) -DELASTIC_USER_NAME=$(ELASTIC_USER_NAME) -DENTITLEMENTS_DOMAIN=$(ENTITLEMENTS_DOMAIN) -DINDEXER_HOST=$(INDEXER_HOST) -DLEGAL_TAG=$(LEGAL_TAG) -DOTHER_RELEVANT_DATA_COUNTRIES=$(OTHER_RELEVANT_DATA_COUNTRIES) -DSEARCH_HOST=$(SEARCH_HOST) -DSEARCH_INTEGRATION_TESTER=$(SEARCH_INTEGRATION_TESTER) -DSTORAGE_HOST=$(STORAGE_HOST) compile'
+ mavenPomFile: "$(integrationTestGcpPom)"
+ options: "--settings $(mavenSettings) -Dsurefire.useFile=false -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -DDEFAULT_DATA_PARTITION_ID_TENANT1=$(DEFAULT_DATA_PARTITION_ID_TENANT1) -DDEFAULT_DATA_PARTITION_ID_TENANT2=$(DEFAULT_DATA_PARTITION_ID_TENANT2) -DELASTIC_HOST=$(ELASTIC_HOST) -DELASTIC_PASSWORD=$(ELASTIC_PASSWORD) -DELASTIC_USER_NAME=$(ELASTIC_USER_NAME) -DENTITLEMENTS_DOMAIN=$(ENTITLEMENTS_DOMAIN) -DINDEXER_HOST=$(INDEXER_HOST) -DLEGAL_TAG=$(LEGAL_TAG) -DOTHER_RELEVANT_DATA_COUNTRIES=$(OTHER_RELEVANT_DATA_COUNTRIES) -DSEARCH_HOST=$(SEARCH_HOST) -DSEARCH_INTEGRATION_TESTER=$(SEARCH_INTEGRATION_TESTER) -DSTORAGE_HOST=$(STORAGE_HOST) compile"
publishJUnitResults: false
- javaHomeOption: 'JDKVersion'
- mavenVersionOption: 'Default'
+ javaHomeOption: "JDKVersion"
+ mavenVersionOption: "Default"
mavenAuthenticateFeed: false
effectivePomSkip: false
sonarQubeRunAnalysis: false
diff --git a/devops/service-pipeline.yml b/devops/service-pipeline.yml
index 19dc5f22a..df06e265e 100644
--- a/devops/service-pipeline.yml
+++ b/devops/service-pipeline.yml
@@ -17,7 +17,7 @@ pr:
autoCancel: false
branches:
include:
- - '*'
+ - "*"
exclude:
- master
paths:
@@ -33,17 +33,17 @@ resources:
name: open-data-ecosystem/infrastructure-templates
variables:
- - group: 'Azure Common Secrets'
- - group: 'Azure - Common'
+ - group: "Azure Common Secrets"
+ - group: "Azure - Common"
- name: serviceName
- value: 'search'
+ value: "search"
stages:
- template: devops/service-pipelines/build-stage.yml@infrastructure-templates
parameters:
mavenPublishJUnitResults: true
- mavenOptions: '--settings ./search-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U'
+ mavenOptions: "--settings ./search-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U"
copyFileContents: |
pom.xml
provider/search-azure/maven/settings.xml
@@ -51,23 +51,23 @@ stages:
provider/search-azure/target/*-spring-boot.jar
copyFileContentsToFlatten: |
provider/search-byoc/target/*-spring-boot.jar
- provider/search-gcp/target/*-spring-boot.jar
- provider/search-gcp/src/main/appengine/app.yaml
- provider/search-gcp/src/main/resources/application.properties
- provider/search-gcp/scripts/deploy.sh
+ provider/search-gc/target/*-spring-boot.jar
+ provider/search-gc/src/main/appengine/app.yaml
+ provider/search-gc/src/main/resources/application.properties
+ provider/search-gc/scripts/deploy.sh
provider/search-integration-tests.zip
serviceBase: ${{ variables.serviceName }}
- testingRootFolder: 'testing/integration-tests'
+ testingRootFolder: "testing/integration-tests"
- template: devops/service-pipelines/deploy-stages.yml@infrastructure-templates
parameters:
serviceName: ${{ variables.serviceName }}
- testCoreMavenPomFile: 'integration-tests/search-test-core/pom.xml'
- testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/integration-tests/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
+ testCoreMavenPomFile: "integration-tests/search-test-core/pom.xml"
+ testCoreMavenOptions: "--settings $(System.DefaultWorkingDirectory)/drop/deploy/integration-tests/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)"
providers:
- - name: Azure
+ - name: Azure
# Merges into Master
- ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
- environments: ['devint', 'qa', 'prod']
+ ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
+ environments: ["devint", "qa", "prod"]
# PR updates / creations
- ${{ if ne(variables['Build.SourceBranchName'], 'master') }}:
- environments: ['devint']
+ ${{ if ne(variables['Build.SourceBranchName'], 'master') }}:
+ environments: ["devint"]
diff --git a/docs/tutorial/SearchService.md b/docs/tutorial/SearchService.md
index b6231b8ca..7eea7ba27 100644
--- a/docs/tutorial/SearchService.md
+++ b/docs/tutorial/SearchService.md
@@ -1,28 +1,29 @@
## Search service
### Table of contents <a name="TOC"></a>
+
- [Introduction](#introduction)
- [Search API access](#search-api-access)
- [Permissions](#permissions)
- [Query](#query)
- * [Query by kind](#query-by-kind)
- * [Text Queries](#text-queries)
- + [Examples](#examples)
- + [Grouping](#grouping)
- + [Reserved characters](#reserved-characters)
- + [Wildcards](#wildcards)
- + [Query by `nested` arrays objects](#nested-queries)
- * [Aggregation](#aggregate-queries)
- + [Aggregation by `nested` arrays objects](#nested-aggregation)
- * [Sort](#sort-queries)
- + [Sort by `nested` arrays objects](#nested-sort)
- * [Range Queries](#range-queries)
- * [Geo-Spatial Queries](#geo-spatial-queries)
- + [Geo Distance](#geo-distance)
- - [Distance Units](#distance-units)
- + [Geo Bounding Box](#bounding-box)
- + [Geo Polygon](#geo-polygon)
- * [Cross-Kind Queries](#cross-kind-queries)
+ - [Query by kind](#query-by-kind)
+ - [Text Queries](#text-queries)
+ - [Examples](#examples)
+ - [Grouping](#grouping)
+ - [Reserved characters](#reserved-characters)
+ - [Wildcards](#wildcards)
+ - [Query by `nested` arrays objects](#nested-queries)
+ - [Aggregation](#aggregate-queries)
+ - [Aggregation by `nested` arrays objects](#nested-aggregation)
+ - [Sort](#sort-queries)
+ - [Sort by `nested` arrays objects](#nested-sort)
+ - [Range Queries](#range-queries)
+ - [Geo-Spatial Queries](#geo-spatial-queries)
+ - [Geo Distance](#geo-distance)
+ - [Distance Units](#distance-units)
+ - [Geo Bounding Box](#bounding-box)
+ - [Geo Polygon](#geo-polygon)
+ - [Cross-Kind Queries](#cross-kind-queries)
- [Query With Cursor](#query-with-cursor)
- [Get indexing status](#get-indexing-status)
- [Version info endpoint](#version-info-endpoint)
@@ -35,35 +36,38 @@ The API supports full text search on string fields, range queries on date, numer
## Search API access <a name="search-api-access"></a>
-* Required roles
+- Required roles
Search service requires that users have dedicated roles in order to use it. Users must be a member of `users.datalake.viewers` or `users.datalake.editors` or `users.datalake.admins`, roles can be assigned using the [Entitlements Service](/solutions/osdu/tutorials/core-services/entitlementsservice). Please look at the API documentation for specific requirements.
In addition to service roles, users __must__ be a member of data groups to access the data.
-* Required headers
+- Required headers
The OSDU Data Platform stores data in different partitions, depending on the different accounts in the OSDU system.
A user may belong to more than one account. As a user, after logging into the OSDU portal, you need to select the account you wish to be active.
Likewise, when using the Search APIs, you need to specify the active account in the header called `Data-Partition-Id`. The correct `Data-Partition-Id` can be obtained from the CFS services. The `Data-Partition-Id` enables the search within the mapped partition. e.g.
+
```
Data-Partition-Id: opendes
```
-* Optional headers
+- Optional headers
The Correlation-Id is a traceable ID to track the journey of a single request. The Correlation-Id can be a GUID on the header with a key. It is best practice to provide the Correlation-Id so the request can be tracked through all the services.
+
```
Correlation-Id: 1e0fef08-22fd-49b1-a5cc-dffa21bc0b70
```
+
If the service is initiating the request, an ID should be generated. If the Correlation-Id is not provided, then a new ID will be generated by the service so that the request would be traceable.
[Back to table of contents](#TOC)
## Permissions <a name="permissions"></a>
-| **_Endpoint URL_** | **_Method_** | **_Minimum Permissions Required_** | **_Data Permissions Required _** |
+| **_Endpoint URL_** | **_Method_** | **_Minimum Permissions Required_** | ___Data Permissions Required ___ |
| --- | --- | --- | --- |
| /search/v2/query | POST | users.datalake.viewers | Yes |
| /search/v2/query_with_cursor | POST | users.datalake.viewers | Yes |
@@ -138,6 +142,7 @@ curl --request POST \
"returnedFields": [ "data.Status" ]
}'
```
+
</details>
__Note:__ : It can take a delay of atleast 30 seconds once records are successfully ingested via Storage service to become searchable in OSDU Data Platform. You can check the [index status](#get-indexing-status).
@@ -151,7 +156,7 @@ __Note:__ : It can take a delay of atleast 30 seconds once records are successfu
| offset | The starting offset from which to return results. |
| limit | The maximum number of results to return from the given offset. If no limit is provided, then it will return __10__ items. Max number of items which can be fetched by the query is __1000__. (If you wish to fetch large set of items, please use [query_with_cursor](#query-with-cursor) API). |
| sort | Allows you to add one or more sorts on specific fields. The length of fields and the length of order must match. Order value must be either ASC or DESC (case insensitive). For more details, ability and limitation about this feature, please refer to [Sort](#sort-queries)
-| queryAsOwner | If true, the result only contains the records that the user owns. If false, the result contains all records that the user is entitled to see. Default value is false |
+| queryAsOwner | If true, the result only contains the records that the user owns. If false, the result contains all records that the user is entitled to see. Default value is false |
| spatialFilter | A spatial filter to apply, please see [Geo-Spatial Queries](#geo-spatial-queries). |
| trackTotalCount | Tracks accurate record count matching the query if 'true', partial count otherwise. Partial count queries are more performant. Default is 'false' and returns 10000 if matching records are higher than 10000. |
| aggregateBy | Allows user to get unique value of given field, please see [Aggregate Queries](#aggregate-queries). |
@@ -171,6 +176,7 @@ POST /api/search/v2/query
"kind": "opendes:welldb:wellbore:1.0.0"
}
```
+
<details><summary>**Curl**</summary>
```bash
@@ -184,6 +190,7 @@ curl --request POST \
"kind": "opendes:welldb:wellbore:1.0.0"
}'
```
+
</details>
The query will return 10 (default limit) documents for the kind.
@@ -204,7 +211,7 @@ __Note:__ __kind__ is a required parameter and is omitted for brevity in followi
#### Examples <a name="examples"></a>
-* search all fields which contains text 'well'
+- search all fields which contains text 'well'
```json
{
@@ -214,7 +221,7 @@ __Note:__ __kind__ is a required parameter and is omitted for brevity in followi
__Note:__ In absence of `<field-name>`, the query string will automatically attempt to determine the existing fields in the index’s mapping that are queryable, and perform the search on those fields. Search query will be more performant if field name are specified in the query instead of searching across all queryable attribute. The following examples cover this:
-* where the Basin field contains "Permian"
+- where the Basin field contains "Permian"
```json
{
@@ -222,7 +229,7 @@ __Note:__ In absence of `<field-name>`, the query string will automatically atte
}
```
-* where the Rig_Contractor field contains "Ocean" or "Drilling". OR is the default operator
+- where the Rig_Contractor field contains "Ocean" or "Drilling". OR is the default operator
```json
{
@@ -238,7 +245,7 @@ or
}
```
-* where the Rig_Contractor field contains the exact `phrase` "Ocean Drilling"
+- where the Rig_Contractor field contains the exact `phrase` "Ocean Drilling"
```json
{
@@ -246,7 +253,7 @@ or
}
```
-* where any of the fields ValueList.OriginalValue, ValueList.Value or ValueList.AppDataType contains "PRODUCING" or "DUAINE" (note how we need to escape the * with a backslash)
+- where any of the fields ValueList.OriginalValue, ValueList.Value or ValueList.AppDataType contains "PRODUCING" or "DUAINE" (note how we need to escape the * with a backslash)
```json
{
@@ -254,7 +261,7 @@ or
}
```
-* where the field Status has any non-null value, use the \_exists\_ prefix for a field will search to see if the field exists
+- where the field Status has any non-null value, use the \_exists\_ prefix for a field will search to see if the field exists
```json
{
@@ -307,6 +314,7 @@ If you need to use date in your query, it has to be in one of the following form
Please note that the time element is optional
```
+
```
date-element = std-date-element
@@ -314,6 +322,7 @@ If you need to use date in your query, it has to be in one of the following form
Example: 2017-12-29
```
+
```
time-element = HH [minute-element] | [fraction]
@@ -354,7 +363,7 @@ For more details, ability and limitation about this feature, please refer to [Ar
#### Single-level one condition `nested` query
-* where `work-product-component--WellboreMarkerSet` has any Marker with MarkerMeasuredDepth field value greater than 10000
+- where `work-product-component--WellboreMarkerSet` has any Marker with MarkerMeasuredDepth field value greater than 10000
```json
{
@@ -365,7 +374,7 @@ For more details, ability and limitation about this feature, please refer to [Ar
#### Single-level several conditions `nested` query
-* where `work-product-component--WellboreMarkerSet` has any Marker with VerticalMeasurement field value greater than 100 and VerticalMeasurementPathID field value is "osdu-openness:reference-data--VerticalMeasurementPath:ELEV:"
+- where `work-product-component--WellboreMarkerSet` has any Marker with VerticalMeasurement field value greater than 100 and VerticalMeasurementPathID field value is "osdu-openness:reference-data--VerticalMeasurementPath:ELEV:"
```json
{
@@ -376,7 +385,7 @@ For more details, ability and limitation about this feature, please refer to [Ar
#### Combination of single-level `nested` queries
-* where `work-product-component--WellboreMarkerSet` has any Marker with MarkerMeasuredDepth field value greater 10000 or SurfaceDipAzimuth field value less than 360
+- where `work-product-component--WellboreMarkerSet` has any Marker with MarkerMeasuredDepth field value greater 10000 or SurfaceDipAzimuth field value less than 360
```json
{
@@ -437,7 +446,7 @@ Allows user to get unique value of field specified by `aggregateBy` request para
"kind": "opendes:welldb:*:*",
"aggregateBy": "kind"
}
-```
+```
### Aggregation by `nested` arrays objects <a name="nested-aggregation"></a>
@@ -453,6 +462,7 @@ Allows user to get unique value of field specified by `aggregateBy` request para
For more details, ability and limitation about this feature, please refer to [ArrayOfObjects](#docs/tutorial/ArrayOfObjects.md).
## Sort <a name="sort-queries"></a>
+
Sort query allows to add one or more sorts on specific fields. Each sort can be reversed as well.
The sort feature supports string, int, float, double, long, datetime, nested object & nested array of objects. Sorting on array of string, geo-point & geo-shape type is not supported.
@@ -474,7 +484,8 @@ E.g. Consider following scenarios:
"order": ["ASC"]
}
}
-```
+```
+
The above request payload asks Search service to sort on "data.Id" in an ascending order, and the expected response will have "totalCount: 10" (instead of 20, please note that the 10 returned records are only from opendes:welldb:wellbore:1.0.0 because the data.Id in opendes:welldb:well:1.0.0 is of data type string, which is not currently supported - and therefore, will not be returned) and should list the 5 records which have empty data.Id value at last.
Search results are by default ordered by relevancy `_score` in descending order. Users are not required to provide any sort query for this. Users can also make request to query record in reverse relevancy order.
@@ -490,9 +501,9 @@ Search results are by default ordered by relevancy `_score` in descending order.
}
```
-**NOTE:** Search service does not validate the provided sort field, whether it exists or is of the supported data types. Different kinds may have attributes with the same names, but are different data types. Therefore, it is the user's responsibility to be aware and validate this in one's own workflow.
+__NOTE:__ Search service does not validate the provided sort field, whether it exists or is of the supported data types. Different kinds may have attributes with the same names, but are different data types. Therefore, it is the user's responsibility to be aware and validate this in one's own workflow.
-The sort query could be very expensive, especially if the given kind is too broad (e.g. "kind": "*:*:*:*"). The current time-out threshold is 60 seconds; a 504 error ("Request timed out after waiting for 1m") will be returned if the request times out. The suggestion is to make the kind parameter as narrow as possible while using the sort feature.
+The sort query could be very expensive, especially if the given kind is too broad (e.g. "kind": "*:_:_:*"). The current time-out threshold is 60 seconds; a 504 error ("Request timed out after waiting for 1m") will be returned if the request times out. The suggestion is to make the kind parameter as narrow as possible while using the sort feature.
### Sort by `nested` arrays objects <a name="nested-sort"></a>
@@ -516,7 +527,7 @@ For more details, ability and limitation about this feature, please refer to [Ar
Ranges can be specified for date, numeric or string fields. Inclusive ranges are specified with square brackets `[min TO max]` and exclusive ranges with curly brackets `{min TO max}`. Here are some of the examples:
-* All SpudDate in 2012
+- All SpudDate in 2012
```json
{
@@ -524,7 +535,7 @@ Ranges can be specified for date, numeric or string fields. Inclusive ranges are
}
```
-* Count 1..5
+- Count 1..5
```json
{
@@ -532,7 +543,7 @@ Ranges can be specified for date, numeric or string fields. Inclusive ranges are
}
```
-* Count from 10 upwards
+- Count from 10 upwards
```json
{
@@ -540,7 +551,7 @@ Ranges can be specified for date, numeric or string fields. Inclusive ranges are
}
```
-* Ranges with one side unbounded can use the following syntax
+- Ranges with one side unbounded can use the following syntax
```json
{
@@ -548,7 +559,7 @@ Ranges can be specified for date, numeric or string fields. Inclusive ranges are
}
```
-* combine an upper and lower bound with the simplified syntax, you would need to join two clauses with an AND operator
+- combine an upper and lower bound with the simplified syntax, you would need to join two clauses with an AND operator
```json
{
@@ -556,7 +567,7 @@ Ranges can be specified for date, numeric or string fields. Inclusive ranges are
}
```
-* jobStatus tags between IN_PROGRESS & SUCCESS
+- jobStatus tags between IN_PROGRESS & SUCCESS
```json
{
@@ -566,7 +577,6 @@ Ranges can be specified for date, numeric or string fields. Inclusive ranges are
[Back to table of contents](#TOC)
-
## Geo-Spatial Queries <a name="geo-spatial-queries"></a>
OSDU Data Platform supports geo-point geo data which supports lat/lon pairs. `spatialFilter` and `query` group in the request have AND relationship. If both of the criteria are defined in the query, then the search service will return results which match both clauses.
@@ -623,6 +633,7 @@ curl --request POST \
"limit": 30
}'
```
+
</details>
| Parameter | Description |
@@ -692,9 +703,10 @@ curl --request POST \
"limit": 30
}'
```
+
</details>
-| Parameter | Description |
+| Parameter | Description |
| :--- | :--- |
| field | `geo-point` field in the index on which filtering will be performed. |
| topLeft.latitude | latitude of top left corner of bounding box. |
@@ -754,22 +766,21 @@ curl --request POST \
"limit": 30
}'
```
+
</details>
-| Parameter | Description |
+| Parameter | Description |
| :--- | :--- |
| field | `geo-point` field in the index on which filtering will be performed. |
| points | list of `geo-point` describing polygon. |
-
[Back to table of contents](#TOC)
-
## Cross-Kind Queries <a name="cross-kind-queries"></a>
OSDU Data Platform search supports cross-kind queries. A typical kind can be formatted as authority/data-partition-id:data-source-id:entity-type:schema-version. Each of the text partitioned by ':' can be replaced with wildcard characters to support cross-kind search.
-* search across all data-source, types & versions for opendes
+- search across all data-source, types & versions for opendes
```json
{
@@ -777,7 +788,7 @@ OSDU Data Platform search supports cross-kind queries. A typical kind can be for
}
```
-* search across all data-source, type well with schema version 1.0.0
+- search across all data-source, type well with schema version 1.0.0
```json
{
@@ -785,7 +796,7 @@ OSDU Data Platform search supports cross-kind queries. A typical kind can be for
}
```
-* search across all types and versions for welldb namespace in opendes
+- search across all types and versions for welldb namespace in opendes
```json
{
@@ -795,7 +806,6 @@ OSDU Data Platform search supports cross-kind queries. A typical kind can be for
[Back to table of contents](#TOC)
-
## Query With Cursor <a name="query-with-cursor"></a>
While a search request returns a single “page†of results, the `query_with_cursor` API can be used to retrieve large numbers of results (or even all results) from a single search request, in much the same way as you would use a cursor on a traditional database.
@@ -860,6 +870,7 @@ curl --request POST \
"returnedFields": [ "data.Status" ]
}'
```
+
</details>
The successful response from the above request will include a "cursor", which should be passed to next call of `query_with_cursor` API in order to retrieve the next batch of results.
@@ -886,6 +897,7 @@ curl --request POST \
"cursor": "cursor-key"
}'
```
+
</details>
__Caution:__ As next batches of results are retrieved by `query_with_cursor` API, cursor value may or may not change. API users should not expect different cursor value in each `query_with_cursor` response.
@@ -894,7 +906,6 @@ __Note:__ To process the next `query_with_cursor` request, the search service ke
[Back to table of contents](#TOC)
-
## Get indexing status <a name="get-indexing-status"></a>
Indexer service adds internal meta data to each record which registers the status of the indexing. The meta data includes the status and the last indexing date and time. This additional meta block helps to see the details of indexing. The format of the index meta block is as follows:
@@ -909,7 +920,9 @@ Indexer service adds internal meta data to each record which registers the statu
"lastUpdateTime": Datetime
}
```
+
Example:
+
```json
{
"results": [
@@ -929,11 +942,12 @@ Example:
```
Details of the index block:
+
1) trace: This field collects all the issues related to the indexing and concatinates using '|'. This is a String field.
2) statusCode: This field determines the category of the error. This is integer field. It can have the following values:
- * 200 - All OK
- * 404 - Schema is missing in Schema and Storage service
- * 400 - Some fields were not properly mapped with the schema defined, e.g. schema defined as `int` for field but input record had attribute value as `text`
+ - 200 - All OK
+ - 404 - Schema is missing in Schema and Storage service
+ - 400 - Some fields were not properly mapped with the schema defined, e.g. schema defined as `int` for field but input record had attribute value as `text`
3) lastUpdateTime: This field captures the last time the record was updated by by the indexer service. This is datetime field so you can do range queries on this field.
You can query the index status using the following example query:
@@ -948,6 +962,7 @@ curl --request POST \
NOTE: By default, the API response excludes the 'index' attribute block. The user must specify 'index' as the 'returnedFields" in order to see it in the response.
```
+
The above query will return all records which had problems due to fields mismatch.
[Back to table of contents](#TOC)
@@ -956,13 +971,13 @@ The above query will return all records which had problems due to fields mismatc
Provides build and git related information for Search service.
-#### Example response:
+#### Example response
```json
GET /api/search/v2/info
{
"groupId": "org.opengroup.osdu",
- "artifactId": "search-gcp",
+ "artifactId": "search-gc",
"version": "0.10.0-SNAPSHOT",
"buildTime": "2021-07-09T14:29:51.584Z",
"branch": "feature/GONRG-2681_Build_info",
@@ -982,7 +997,8 @@ GET /api/search/v2/info
```
This endpoint takes information from files generated by `spring-boot-maven-plugin`, `git-commit-id-plugin` plugins. Need to specify paths for generated files to matching properties:
+
- `version.info.buildPropertiesPath`
- `version.info.gitPropertiesPath`
-[Back to table of contents](#TOC)
\ No newline at end of file
+[Back to table of contents](#TOC)
diff --git a/pom.xml b/pom.xml
index 0fd44516f..bfdf089ef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,7 +57,7 @@
<modules>
<module>search-core</module>
<module>provider/search-byoc</module>
- <module>provider/search-gcp</module>
+ <module>provider/search-gc</module>
<module>provider/search-aws</module>
<module>provider/search-azure</module>
<module>provider/search-ibm</module>
diff --git a/provider/search-gcp/README.md b/provider/search-gc/README.md
similarity index 93%
rename from provider/search-gcp/README.md
rename to provider/search-gc/README.md
index 80a1f519b..f948391d6 100644
--- a/provider/search-gcp/README.md
+++ b/provider/search-gc/README.md
@@ -1,5 +1,5 @@
# Search and Indexer Service
-os-search-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) service that hosts CRUD APIs that enable the execution of OSDU R2 domain searches against Elasticsearch.
+os-search-gc is a [Spring Boot](https://spring.io/projects/spring-boot) service that hosts CRUD APIs that enable the execution of OSDU R2 domain searches against Elasticsearch.
## Getting Started
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
@@ -18,7 +18,7 @@ Pre-requisites
### Anthos:
[Anthos service configuration ](docs/anthos/README.md)
### Google Cloud:
-[Google Cloud service configuration ](docs/gcp/README.md)
+[Google Cloud service configuration ](docs/gc/README.md)
### Run Locally
Check that maven is installed:
@@ -96,7 +96,7 @@ mvn clean install -DskipTests
After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
```bash
-cd provider/search-gcp/ && mvn spring-boot:run
+cd provider/search-gc/ && mvn spring-boot:run
```
## Testing
@@ -113,7 +113,7 @@ This section describes how to run cloud OSDU E2E tests.
### Anthos test configuration:
[Anthos service configuration ](docs/anthos/README.md)
### Google Cloud test configuration:
-[Google Cloud service configuration ](docs/gcp/README.md)
+[Google Cloud service configuration ](docs/gc/README.md)
## Deployment
diff --git a/provider/search-gcp/cloudbuild/Dockerfile.cloudbuild b/provider/search-gc/cloudbuild/Dockerfile.cloudbuild
similarity index 100%
rename from provider/search-gcp/cloudbuild/Dockerfile.cloudbuild
rename to provider/search-gc/cloudbuild/Dockerfile.cloudbuild
diff --git a/provider/search-gcp/cloudbuild/cloudbuild.yaml b/provider/search-gc/cloudbuild/cloudbuild.yaml
similarity index 51%
rename from provider/search-gcp/cloudbuild/cloudbuild.yaml
rename to provider/search-gc/cloudbuild/cloudbuild.yaml
index a0839623e..61563cdfa 100644
--- a/provider/search-gcp/cloudbuild/cloudbuild.yaml
+++ b/provider/search-gc/cloudbuild/cloudbuild.yaml
@@ -15,17 +15,22 @@
# limitations under the License.
steps:
- - name: 'gcr.io/cloud-builders/docker'
- args: [
- 'build',
- '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
- '--build-arg', 'PORT=${_PORT}',
- '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}',
- '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest',
- '-f', 'provider/${_GCP_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild',
- '.'
- ]
+ - name: "gcr.io/cloud-builders/docker"
+ args:
+ [
+ "build",
+ "--build-arg",
+ "PROVIDER_NAME=${_PROVIDER_NAME}",
+ "--build-arg",
+ "PORT=${_PORT}",
+ "-t",
+ "gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GC_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}",
+ "-t",
+ "gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GC_SERVICE}-${_PROVIDER_NAME}:latest",
+ "-f",
+ "provider/${_GC_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild",
+ ".",
+ ]
images:
- - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}'
-
+ - "gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GC_SERVICE}-${_PROVIDER_NAME}"
diff --git a/provider/search-gcp/docs/anthos/README.md b/provider/search-gc/docs/anthos/README.md
similarity index 95%
rename from provider/search-gcp/docs/anthos/README.md
rename to provider/search-gc/docs/anthos/README.md
index 34b6fe25d..dd7ebb885 100644
--- a/provider/search-gcp/docs/anthos/README.md
+++ b/provider/search-gc/docs/anthos/README.md
@@ -1,6 +1,6 @@
## Service Configuration for Anthos
-## Environment variables:
+## Environment variables
Define the following environment variables.
@@ -9,8 +9,8 @@ Must have:
| name | value | description | sensitive? | source |
|----------------------------------------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|--------|
| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for Google Cloud environment | false | - |
-| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | yes | - |
-| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | false | - |
+| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#properties-set-in-partition-service) | yes | - |
+| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#properties-set-in-partition-service) | false | - |
Defined in default application property file but possible to override:
@@ -28,7 +28,7 @@ Defined in default application property file but possible to override:
| `REDIS_SEARCH_PASSWORD` | ex `127.0.0.1` | Redis search host password | yes | |
| `REDIS_SEARCH_WITH_SSL` | ex `true` or `false` | Redis search host ssl config | no | |
| `REDIS_SEARCH_EXPIRATION` | ex `30` | Redis search cache expiration in seconds | no | |
-| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
| `PARTITION_API` | ex `http://localhost:8080/api/partition/v1` | Partition service endpoint | no | output of infrastructure deployment |
| `POLICY_API` | ex `http://localhost:8080/api/policy/v1/` | Policy service endpoint | no | output of infrastructure deployment |
@@ -36,7 +36,7 @@ Defined in default application property file but possible to override:
| `SERVICE_POLICY_ENABLED` | ex `false` | Enable or Disable an integration with Policy Service | no | output of infrastructure deployment |
| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+These variables define service behavior, and are used to switch between `Reference` or `Google Cloud` environments, their overriding and usage in mixed mode was not tested.
Usage of spring profiles is preferred.
| name | value | description | sensitive? | source |
@@ -44,12 +44,13 @@ Usage of spring profiles is preferred.
| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` | Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
-### Properties set in Partition service:
+### Properties set in Partition service
Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name.
This variable should be present in environment of service that need that variable.
Example:
+
```
"elasticsearch.port": {
"sensitive": false, <- value not sensitive
@@ -61,8 +62,6 @@ Example:
}
```
-
-
## Elasticsearch configuration
**prefix:** `elasticsearch`
@@ -89,7 +88,7 @@ curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H '
"properties": {
"elasticsearch.host": {
"sensitive": false,
- "value": "elastic.us-central1.gcp.cloud.es.io"
+ "value": "elastic.us-central1.gc.cloud.es.io"
},
"elasticsearch.port": {
"sensitive": false,
@@ -120,7 +119,7 @@ You will need to have the following environment variables defined.
| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
| `DATA_GROUP` | `opendes` | The service account to this group and substitute | no | - |
-| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
+| `ENTITLEMENTS_DOMAIN` | ex `opendes-gc.projects.com` | OSDU R2 to run tests under | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `not-exist` | HTTP Header 'Data-Partition-ID' with not existing tenant | no | - |
| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
@@ -132,7 +131,7 @@ You will need to have the following environment variables defined.
**Entitlements configuration for integration accounts**
-| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
+| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
| --- | --- |
| users<br/>service.entitlements.user<br/>service.search.user<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
diff --git a/provider/search-gcp/docs/gcp/README.md b/provider/search-gc/docs/gc/README.md
similarity index 95%
rename from provider/search-gcp/docs/gcp/README.md
rename to provider/search-gc/docs/gc/README.md
index 283dd87c2..18ce3bb10 100644
--- a/provider/search-gcp/docs/gcp/README.md
+++ b/provider/search-gc/docs/gc/README.md
@@ -1,6 +1,6 @@
## Service Configuration for Google Cloud
-## Environment variables:
+## Environment variables
Define the following environment variables.
@@ -9,8 +9,8 @@ Must have:
| name | value | description | sensitive? | source |
|----------------------------------------------|---------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|---------------------------------------------------|
| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for Google Cloud environment | false | - |
-| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | yes | - |
-| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | false | - |
+| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#properties-set-in-partition-service) | yes | - |
+| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#properties-set-in-partition-service) | false | - |
Defined in default application property file but possible to override:
@@ -28,7 +28,7 @@ Defined in default application property file but possible to override:
| `REDIS_SEARCH_PASSWORD` | ex `*****` | Redis search host password | yes | |
| `REDIS_SEARCH_WITH_SSL` | ex `true` or `false` | Redis search host ssl config | no | |
| `REDIS_SEARCH_EXPIRATION` | ex `30` | Redis search cache expiration in seconds | no | |
-| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
| `PARTITION_API` | ex `http://localhost:8080/api/partition/v1` | Partition service endpoint | no | output of infrastructure deployment |
| `POLICY_API` | ex `http://localhost:8080/api/policy/v1/` | Policy service endpoint | no | output of infrastructure deployment |
@@ -36,7 +36,7 @@ Defined in default application property file but possible to override:
| `SERVICE_POLICY_ENABLED` | ex `false` | Enable or Disable an integration with Policy Service | no | output of infrastructure deployment |
| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+These variables define service behavior, and are used to switch between `Reference` or `Google Cloud` environments, their overriding and usage in mixed mode was not tested.
Usage of spring profiles is preferred.
| name | value | description | sensitive? | source |
@@ -44,12 +44,13 @@ Usage of spring profiles is preferred.
| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` | Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
-### Properties set in Partition service:
+### Properties set in Partition service
Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name.
This variable should be present in environment of service that need that variable.
Example:
+
```
"elasticsearch.port": {
"sensitive": false, <- value not sensitive
@@ -61,8 +62,6 @@ Example:
}
```
-
-
## Elasticsearch configuration
**prefix:** `elasticsearch`
@@ -89,7 +88,7 @@ curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H '
"properties": {
"elasticsearch.host": {
"sensitive": false,
- "value": "elastic.us-central1.gcp.cloud.es.io"
+ "value": "elastic.us-central1.gc.cloud.es.io"
},
"elasticsearch.port": {
"sensitive": false,
@@ -125,15 +124,14 @@ You will need to have the following environment variables defined.
| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `not-exist` | HTTP Header 'Data-Partition-ID' with not existing tenant | no | - |
-| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
| `LEGAL_TAG` | ex `my-legal-tag` | a valid legal tag | no | - |
| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-
**Entitlements configuration for integration accounts**
-| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
+| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
| --- | --- |
| users<br/>service.entitlements.user<br/>service.search.user<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
@@ -142,5 +140,5 @@ Execute following command to build code and run all the integration tests:
```bash
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
-$ (cd testing/search-test-gcp/ && mvn clean test)
+$ (cd testing/integration-tests/search-test-gc/ && mvn clean test)
```
diff --git a/provider/search-gcp/pom.xml b/provider/search-gc/pom.xml
similarity index 99%
rename from provider/search-gcp/pom.xml
rename to provider/search-gc/pom.xml
index bb902653b..5f32c7417 100644
--- a/provider/search-gcp/pom.xml
+++ b/provider/search-gc/pom.xml
@@ -17,7 +17,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
- <artifactId>search-gcp</artifactId>
+ <artifactId>search-gc</artifactId>
<version>0.19.0-SNAPSHOT</version>
<description>Google cloud implementation of Search service APIs</description>
<packaging>jar</packaging>
diff --git a/provider/search-gcp/src/main/appengine/app.yaml b/provider/search-gc/src/main/appengine/app.yaml
similarity index 100%
rename from provider/search-gcp/src/main/appengine/app.yaml
rename to provider/search-gc/src/main/appengine/app.yaml
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/SearchGcpApplication.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/SearchGcpApplication.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/SearchGcpApplication.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/SearchGcpApplication.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/auth/GcpAuthorizationService.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/auth/GcpAuthorizationService.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/auth/GcpAuthorizationService.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/auth/GcpAuthorizationService.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CacheConfig.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CacheConfig.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CacheConfig.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CacheConfig.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CursorCacheImpl.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CursorCacheImpl.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CursorCacheImpl.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/CursorCacheImpl.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/ElasticCredentialsCache.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/ElasticCredentialsCache.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/ElasticCredentialsCache.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/ElasticCredentialsCache.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/FieldTypeMappingCache.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/FieldTypeMappingCache.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/FieldTypeMappingCache.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/cache/FieldTypeMappingCache.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/config/GcpSearchConfigurationProperties.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/config/GcpSearchConfigurationProperties.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/config/GcpSearchConfigurationProperties.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/config/GcpSearchConfigurationProperties.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfig.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfig.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfig.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfig.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfigurationProperties.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfigurationProperties.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfigurationProperties.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/di/ElasticSearchConfigurationProperties.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/di/EntitlementsClientFactory.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/di/EntitlementsClientFactory.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/di/EntitlementsClientFactory.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/di/EntitlementsClientFactory.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/di/GcpServiceAccountJwtClient.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/di/GcpServiceAccountJwtClient.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/di/GcpServiceAccountJwtClient.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/di/GcpServiceAccountJwtClient.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceImpl.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceImpl.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceImpl.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceImpl.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryBase.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryBase.java
similarity index 98%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryBase.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryBase.java
index b23250cd8..2cc8b4e36 100644
--- a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryBase.java
+++ b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryBase.java
@@ -1,470 +1,470 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.search.provider.gcp.provider.impl;
-
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.geoBoundingBoxQuery;
-import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery;
-import static org.elasticsearch.index.query.QueryBuilders.geoIntersectionQuery;
-import static org.elasticsearch.index.query.QueryBuilders.geoPolygonQuery;
-import static org.elasticsearch.index.query.QueryBuilders.geoWithinQuery;
-import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
-
-import com.google.common.base.Strings;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import javax.inject.Inject;
-import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.lang3.StringUtils;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.geo.builders.CircleBuilder;
-import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
-import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
-import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
-import org.elasticsearch.common.geo.builders.MultiPointBuilder;
-import org.elasticsearch.common.geo.builders.MultiPolygonBuilder;
-import org.elasticsearch.common.geo.builders.PolygonBuilder;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.geometry.Geometry;
-import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.WrapperQueryBuilder;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.aggregations.Aggregation;
-import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.locationtech.jts.geom.Coordinate;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.entitlements.AclRole;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.AggregationResponse;
-import org.opengroup.osdu.core.common.model.search.Point;
-import org.opengroup.osdu.core.common.model.search.Polygon;
-import org.opengroup.osdu.core.common.model.search.Query;
-import org.opengroup.osdu.core.common.model.search.QueryUtils;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.model.search.SpatialFilter;
-import org.opengroup.osdu.search.policy.service.IPolicyService;
-import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
-import org.opengroup.osdu.search.service.IFieldMappingTypeService;
-import org.opengroup.osdu.search.util.AggregationParserUtil;
-import org.opengroup.osdu.search.util.CrossTenantUtils;
-import org.opengroup.osdu.search.util.IDetailedBadRequestMessageUtil;
-import org.opengroup.osdu.search.util.IQueryParserUtil;
-import org.opengroup.osdu.search.util.ISortParserUtil;
-import org.springframework.beans.factory.annotation.Autowired;
-
-abstract class QueryBase {
-
- public static final String SEARCH_ERROR_MSG = "Search error";
- public static final String ERROR_PROCESSING_SEARCH_REQUEST_MSG = "Error processing search request";
- @Inject
- DpsHeaders dpsHeaders;
- @Inject
- private JaxRsDpsLog log;
- @Inject
- private IProviderHeaderService providerHeaderService;
- @Inject
- private CrossTenantUtils crossTenantUtils;
- @Inject
- private IFieldMappingTypeService fieldMappingTypeService;
- @Autowired(required = false)
- private IPolicyService iPolicyService;
- @Autowired
- private IQueryParserUtil queryParserUtil;
- @Autowired
- private ISortParserUtil sortParserUtil;
- @Autowired
- private IDetailedBadRequestMessageUtil detailedBadRequestMessageUtil;
-
- private static final String GEO_SHAPE_INDEXED_TYPE = "geo_shape";
- private static final int MINIMUM_POLYGON_POINTS_SIZE = 4;
-
- // if returnedField contains property matching from excludes than query result will NOT include that property
- private final Set<String> excludes = new HashSet<>(Arrays.asList(RecordMetaAttribute.X_ACL.getValue()));
-
- // queryableExcludes properties can be returned by query results
- private final Set<String> queryableExcludes = new HashSet<>(Arrays.asList(RecordMetaAttribute.INDEX_STATUS.getValue()));
-
- private final TimeValue requestTimeout = TimeValue.timeValueMinutes(1);
-
- private boolean useGeoShapeQuery = false;
-
- QueryBuilder buildQuery(String simpleQuery, SpatialFilter spatialFilter, boolean asOwner) throws IOException {
- QueryBuilder textQueryBuilder = null;
- QueryBuilder spatialQueryBuilder = null;
- QueryBuilder queryBuilder = null;
-
- if (!Strings.isNullOrEmpty(simpleQuery)) {
- textQueryBuilder = queryParserUtil.buildQueryBuilderFromQueryString(simpleQuery);
- }
-
- // use only one of the spatial request
- if (spatialFilter != null) {
- if (useGeoShapeQuery) {
- if (spatialFilter.getByBoundingBox() != null) {
- spatialQueryBuilder = getGeoShapeBoundingBoxQuery(spatialFilter);
- } else if (spatialFilter.getByDistance() != null) {
- spatialQueryBuilder = getGeoShapeDistanceQuery(spatialFilter);
- } else if (spatialFilter.getByGeoPolygon() != null) {
- spatialQueryBuilder = getGeoShapePolygonQuery(spatialFilter);
- } else if (spatialFilter.getByIntersection() != null) {
- spatialQueryBuilder = getGeoShapeIntersectionQuery(spatialFilter);
- } else if (spatialFilter.getByWithinPolygon() != null) {
- spatialQueryBuilder = getWithinPolygonQuery(spatialFilter);
- }
- } else {
- if (spatialFilter.getByBoundingBox() != null) {
- spatialQueryBuilder = getBoundingBoxQuery(spatialFilter);
- } else if (spatialFilter.getByDistance() != null) {
- spatialQueryBuilder = getDistanceQuery(spatialFilter);
- } else if (spatialFilter.getByGeoPolygon() != null) {
- spatialQueryBuilder = getGeoPolygonQuery(spatialFilter);
- }
- }
- }
-
- if (textQueryBuilder != null) {
- queryBuilder = boolQuery().must(textQueryBuilder);
- }
- if (spatialQueryBuilder != null) {
- queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(spatialQueryBuilder) : boolQuery().must(spatialQueryBuilder);
- }
-
- if(this.iPolicyService != null) {
- String compiledESPolicy = this.iPolicyService.getCompiledPolicy(providerHeaderService);
- WrapperQueryBuilder wrapperQueryBuilder = new WrapperQueryBuilder(compiledESPolicy);
- return queryBuilder != null ? boolQuery().must(queryBuilder).must(wrapperQueryBuilder) : boolQuery().must(wrapperQueryBuilder);
- } else {
- return getQueryBuilderWithAuthorization(queryBuilder, asOwner);
- }
- }
-
- private QueryBuilder getQueryBuilderWithAuthorization(QueryBuilder queryBuilder, boolean asOwner) {
- if (userHasFullDataAccess()) {
- return queryBuilder;
- }
-
- QueryBuilder authorizationQueryBuilder = null;
- String groups = dpsHeaders.getHeaders().get(providerHeaderService.getDataGroupsHeader());
- if (groups != null) {
- String[] groupArray = groups.trim().split("\\s*,\\s*");
- if (asOwner) {
- authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(AclRole.OWNERS.getPath(), groupArray));
- } else {
- authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(RecordMetaAttribute.X_ACL.getValue(), groupArray));
- }
- }
- if (authorizationQueryBuilder != null) {
- queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(authorizationQueryBuilder) : boolQuery().must(authorizationQueryBuilder);
- }
- return queryBuilder;
- }
-
- private QueryBuilder getSimpleQuery(String searchQuery) {
-
- // if query is empty , then put *
- String query = StringUtils.isNotBlank(searchQuery) ? searchQuery : "*";
- return queryStringQuery(query).allowLeadingWildcard(false);
- }
-
- private QueryBuilder getBoundingBoxQuery(SpatialFilter spatialFilter) {
-
- GeoPoint topLeft = new GeoPoint(spatialFilter.getByBoundingBox().getTopLeft().getLatitude(), spatialFilter.getByBoundingBox().getTopLeft().getLongitude());
- GeoPoint bottomRight = new GeoPoint(spatialFilter.getByBoundingBox().getBottomRight().getLatitude(), spatialFilter.getByBoundingBox().getBottomRight().getLongitude());
- return geoBoundingBoxQuery(spatialFilter.getField()).setCorners(topLeft, bottomRight);
- }
-
- private QueryBuilder getDistanceQuery(SpatialFilter spatialFilter) {
-
- return geoDistanceQuery(spatialFilter.getField())
- .point(spatialFilter.getByDistance().getPoint().getLatitude(), spatialFilter.getByDistance().getPoint().getLongitude())
- .distance(spatialFilter.getByDistance().getDistance(), DistanceUnit.METERS);
- }
-
- private QueryBuilder getGeoPolygonQuery(SpatialFilter spatialFilter) {
-
- List<GeoPoint> points = new ArrayList<>();
- for (Point point : spatialFilter.getByGeoPolygon().getPoints()) {
- points.add(new GeoPoint(point.getLatitude(), point.getLongitude()));
- }
- return geoPolygonQuery(spatialFilter.getField(), points);
- }
-
- private QueryBuilder getGeoShapePolygonQuery(SpatialFilter spatialFilter) throws IOException {
-
- List<Coordinate> points = new ArrayList<>();
- for (Point point : spatialFilter.getByGeoPolygon().getPoints()) {
- points.add(new Coordinate(point.getLongitude(), point.getLatitude()));
- }
- CoordinatesBuilder cb = new CoordinatesBuilder().coordinates(points);
- Geometry geometry = new PolygonBuilder(cb).buildGeometry();
- return geoWithinQuery(spatialFilter.getField(), geometry);
- }
-
- private QueryBuilder getGeoShapeBoundingBoxQuery(SpatialFilter spatialFilter) throws IOException {
-
- Coordinate topLeft = new Coordinate(spatialFilter.getByBoundingBox().getTopLeft().getLongitude(), spatialFilter.getByBoundingBox().getTopLeft().getLatitude());
- Coordinate bottomRight = new Coordinate(spatialFilter.getByBoundingBox().getBottomRight().getLongitude(), spatialFilter.getByBoundingBox().getBottomRight().getLatitude());
- Rectangle rectangle = new EnvelopeBuilder(topLeft, bottomRight).buildGeometry();
- return geoWithinQuery(spatialFilter.getField(), rectangle);
- }
-
- private QueryBuilder getGeoShapeDistanceQuery(SpatialFilter spatialFilter) throws IOException {
-
- Coordinate center = new Coordinate(spatialFilter.getByDistance().getPoint().getLongitude(), spatialFilter.getByDistance().getPoint().getLatitude());
- CircleBuilder circleBuilder = new CircleBuilder().center(center).radius(spatialFilter.getByDistance().getDistance(), DistanceUnit.METERS);
- return geoWithinQuery(spatialFilter.getField(), circleBuilder);
- }
-
- private QueryBuilder getGeoShapeIntersectionQuery(SpatialFilter spatialFilter)
- throws IOException {
- MultiPolygonBuilder multiPolygonBuilder = new MultiPolygonBuilder();
- for (Polygon polygon : spatialFilter.getByIntersection().getPolygons()) {
- List<Coordinate> coordinates = new ArrayList<>();
- for (Point point : polygon.getPoints()) {
- coordinates.add(new Coordinate(point.getLongitude(), point.getLatitude()));
- }
-
- checkPolygon(coordinates);
-
- CoordinatesBuilder cb = new CoordinatesBuilder().coordinates(coordinates);
- multiPolygonBuilder.polygon(new PolygonBuilder(cb));
- }
-
- GeometryCollectionBuilder geometryCollection = new GeometryCollectionBuilder();
- geometryCollection.shape(multiPolygonBuilder);
- return geoIntersectionQuery(spatialFilter.getField(), geometryCollection.buildGeometry());
- }
-
- private void checkPolygon(List<Coordinate> coordinates) {
- if (coordinates.size() < MINIMUM_POLYGON_POINTS_SIZE ||
- (
- coordinates.get(0).x != coordinates.get(coordinates.size() - 1).x
- || coordinates.get(0).y != coordinates.get(coordinates.size() - 1).y
- )
- ) {
- throw new AppException(HttpServletResponse.SC_BAD_REQUEST, "Bad Request",
- String.format(
- "Polygons must have at least %s points and the first point must match the last point",
- MINIMUM_POLYGON_POINTS_SIZE));
- }
- }
-
- private QueryBuilder getWithinPolygonQuery(SpatialFilter spatialFilter) throws IOException {
- MultiPointBuilder multiPointBuilder = new MultiPointBuilder();
- for (Point point : spatialFilter.getByWithinPolygon().getPoints()) {
- multiPointBuilder.coordinate(new Coordinate(point.getLongitude(), point.getLatitude()));
- }
-
- GeometryCollectionBuilder geometryCollection = new GeometryCollectionBuilder();
- geometryCollection.multiPoint(multiPointBuilder);
- // geoWithinQuery doesn't work with a polygon as the field to search on
- return geoIntersectionQuery(spatialFilter.getField(), geometryCollection.buildGeometry());
- }
-
- String getIndex(Query request) {
- return this.crossTenantUtils.getIndexName(request);
- }
-
- List<Map<String, Object>> getHitsFromSearchResponse(SearchResponse searchResponse) {
- List<Map<String, Object>> results = new ArrayList<>();
- SearchHits searchHits = searchResponse.getHits();
- if (searchHits.getHits().length != 0) {
- for (SearchHit searchHitFields : searchHits.getHits()) {
- Map<String, Object> hitFields = searchHitFields.getSourceAsMap();
- if (!searchHitFields.getHighlightFields().isEmpty()) {
- for (HighlightField hf : searchHitFields.getHighlightFields().values()) {
- if (!hf.getName().equalsIgnoreCase(RecordMetaAttribute.X_ACL.getValue())) {
- Text[] fragments = hf.getFragments();
- if (fragments.length > 0) {
- hitFields.put(hf.getName(), fragments[0].toString());
- }
- }
- }
- }
- results.add(hitFields);
- }
- return results;
- }
-
- return null;
- }
-
- List<AggregationResponse> getAggregationFromSearchResponse(SearchResponse searchResponse) {
- List<AggregationResponse> results = null;
-
- if (searchResponse.getAggregations() != null) {
- Terms kindAgg = null;
- ParsedNested nested = searchResponse.getAggregations().get(AggregationParserUtil.NESTED_AGGREGATION_NAME);
- if(nested != null){
- kindAgg = (Terms) getTermsAggregationFromNested(nested);
- }else {
- kindAgg = searchResponse.getAggregations().get(AggregationParserUtil.TERM_AGGREGATION_NAME);
- }
- if (kindAgg.getBuckets() != null) {
- results = new ArrayList<>();
- for (Terms.Bucket bucket : kindAgg.getBuckets()) {
- results.add(AggregationResponse.builder().key(bucket.getKeyAsString()).count(bucket.getDocCount()).build());
- }
- }
- }
-
- return results;
- }
-
- private Aggregation getTermsAggregationFromNested(ParsedNested parsedNested){
- ParsedNested nested = parsedNested.getAggregations().get(AggregationParserUtil.NESTED_AGGREGATION_NAME);
- if(nested != null){
- return getTermsAggregationFromNested(nested);
- }else {
- return parsedNested.getAggregations().get(AggregationParserUtil.TERM_AGGREGATION_NAME);
- }
- }
-
- SearchSourceBuilder createSearchSourceBuilder(Query request) throws IOException {
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-
- // build query: set query options and query
- QueryBuilder queryBuilder = buildQuery(request.getQuery(), request.getSpatialFilter(), request.isQueryAsOwner());
- sourceBuilder.size(QueryUtils.getResultSizeForQuery(request.getLimit()));
- sourceBuilder.query(queryBuilder);
- sourceBuilder.timeout(requestTimeout);
- if (request.isTrackTotalCount()) {
- sourceBuilder.trackTotalHits(request.isTrackTotalCount());
- }
-
- // set highlighter
- if (request.isReturnHighlightedFields()) {
- HighlightBuilder highlightBuilder = new HighlightBuilder().field("*", 200, 5);
- sourceBuilder.highlighter(highlightBuilder);
- }
-
- // set the return fields
- List<String> returnedFields = request.getReturnedFields();
- if (returnedFields == null) {
- returnedFields = new ArrayList<>();
- }
- Set<String> returnedFieldsSet = new HashSet<>(returnedFields);
- String[] includesArr = returnedFieldsSet.toArray(new String[returnedFieldsSet.size()]);
-
- // remove all matching returnedField and queryable from excludes
- Set<String> requestQueryableExcludes = new HashSet<>(queryableExcludes);
- Set<String> requestExcludes = new HashSet<>(excludes);
- requestQueryableExcludes.removeAll(returnedFields);
- requestExcludes.addAll(requestQueryableExcludes);
- String[] excludesArr = requestExcludes.toArray(new String[requestExcludes.size()]);
-
- sourceBuilder.fetchSource(includesArr, excludesArr);
- return sourceBuilder;
- }
-
- SearchResponse makeSearchRequest(Query searchRequest, RestHighLevelClient client) {
- Long startTime = 0L;
- SearchRequest elasticSearchRequest = null;
- SearchResponse searchResponse = null;
- try {
- String index = this.getIndex(searchRequest);
- if (searchRequest.getSpatialFilter() != null) {
- useGeoShapeQuery = this.useGeoShapeQuery(client, searchRequest, index);
- }
- elasticSearchRequest = createElasticRequest(searchRequest);
- if (searchRequest.getSort() != null) {
- List<FieldSortBuilder> sortBuilders = this.sortParserUtil.getSortQuery(client, searchRequest.getSort(), index);
- for (FieldSortBuilder fieldSortBuilder : sortBuilders) {
- elasticSearchRequest.source().sort(fieldSortBuilder);
- }
- }
-
- startTime = System.currentTimeMillis();
- searchResponse = client.search(elasticSearchRequest, RequestOptions.DEFAULT);
- return searchResponse;
- } catch (ElasticsearchStatusException e) {
- switch (e.status()) {
- case NOT_FOUND:
- throw new AppException(HttpServletResponse.SC_NOT_FOUND, "Not Found", "Resource you are trying to find does not exists", e);
- case BAD_REQUEST:
- throw new AppException(HttpServletResponse.SC_BAD_REQUEST, "Bad Request", detailedBadRequestMessageUtil.getDetailedBadRequestMessage(elasticSearchRequest, e), e);
- case SERVICE_UNAVAILABLE:
- throw new AppException(HttpServletResponse.SC_SERVICE_UNAVAILABLE, SEARCH_ERROR_MSG, "Please re-try search after some time.", e);
- default:
- throw new AppException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, SEARCH_ERROR_MSG, ERROR_PROCESSING_SEARCH_REQUEST_MSG, e);
- }
- } catch (AppException e){
- throw e;
- } catch (IOException e) {
- if (e.getMessage().startsWith("listener timeout after waiting for")) {
- throw new AppException(HttpServletResponse.SC_GATEWAY_TIMEOUT, SEARCH_ERROR_MSG, String.format("Request timed out after waiting for %sm", requestTimeout.getMinutes()), e);
- }
- throw new AppException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, SEARCH_ERROR_MSG, ERROR_PROCESSING_SEARCH_REQUEST_MSG, e);
- } catch (Exception e) {
- if(e instanceof java.net.SocketTimeoutException){
- throw new AppException(HttpServletResponse.SC_REQUEST_TIMEOUT, SEARCH_ERROR_MSG, String.format("Request timed out after waiting for %sm", requestTimeout.getMinutes()), e);
- }
- throw new AppException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, SEARCH_ERROR_MSG, ERROR_PROCESSING_SEARCH_REQUEST_MSG, e);
- } finally {
- Long latency = System.currentTimeMillis() - startTime;
- String request = elasticSearchRequest != null ? elasticSearchRequest.source().toString() : searchRequest.toString();
- this.log.info(String.format("elastic latency: %s | elastic request-payload: %s", latency, request));
- this.auditLog(searchRequest, searchResponse);
- }
- }
-
- private boolean useGeoShapeQuery(RestHighLevelClient client, Query searchRequest, String index) throws IOException {
- Set<String> indexedTypes = this.fieldMappingTypeService.getFieldTypes(client, searchRequest.getSpatialFilter().getField(), index);
- // fallback to geo_point search if mixed type found for spatialFilter.field
- if (indexedTypes.isEmpty() || indexedTypes.size() > 1) return false;
- return indexedTypes.contains(GEO_SHAPE_INDEXED_TYPE);
- }
-
- abstract SearchRequest createElasticRequest(Query request) throws IOException;
-
- abstract void querySuccessAuditLogger(Query request);
-
- abstract void queryFailedAuditLogger(Query request);
-
- private void auditLog(Query searchRequest, SearchResponse searchResponse) {
- if (searchResponse != null && searchResponse.status() == RestStatus.OK) {
- this.querySuccessAuditLogger(searchRequest);
- return;
- }
- this.queryFailedAuditLogger(searchRequest);
- }
-
- private boolean userHasFullDataAccess() {
- String dataRootUser = dpsHeaders.getHeaders().getOrDefault(providerHeaderService.getDataRootUserHeader(), "false");
- return Boolean.parseBoolean(dataRootUser);
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.search.provider.gcp.provider.impl;
+
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoBoundingBoxQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoIntersectionQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoPolygonQuery;
+import static org.elasticsearch.index.query.QueryBuilders.geoWithinQuery;
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
+
+import com.google.common.base.Strings;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.builders.CircleBuilder;
+import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
+import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
+import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
+import org.elasticsearch.common.geo.builders.MultiPointBuilder;
+import org.elasticsearch.common.geo.builders.MultiPolygonBuilder;
+import org.elasticsearch.common.geo.builders.PolygonBuilder;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.locationtech.jts.geom.Coordinate;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.entitlements.AclRole;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.AggregationResponse;
+import org.opengroup.osdu.core.common.model.search.Point;
+import org.opengroup.osdu.core.common.model.search.Polygon;
+import org.opengroup.osdu.core.common.model.search.Query;
+import org.opengroup.osdu.core.common.model.search.QueryUtils;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.model.search.SpatialFilter;
+import org.opengroup.osdu.search.policy.service.IPolicyService;
+import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
+import org.opengroup.osdu.search.service.IFieldMappingTypeService;
+import org.opengroup.osdu.search.util.AggregationParserUtil;
+import org.opengroup.osdu.search.util.CrossTenantUtils;
+import org.opengroup.osdu.search.util.IDetailedBadRequestMessageUtil;
+import org.opengroup.osdu.search.util.IQueryParserUtil;
+import org.opengroup.osdu.search.util.ISortParserUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+
+abstract class QueryBase {
+
+ public static final String SEARCH_ERROR_MSG = "Search error";
+ public static final String ERROR_PROCESSING_SEARCH_REQUEST_MSG = "Error processing search request";
+ @Inject
+ DpsHeaders dpsHeaders;
+ @Inject
+ private JaxRsDpsLog log;
+ @Inject
+ private IProviderHeaderService providerHeaderService;
+ @Inject
+ private CrossTenantUtils crossTenantUtils;
+ @Inject
+ private IFieldMappingTypeService fieldMappingTypeService;
+ @Autowired(required = false)
+ private IPolicyService iPolicyService;
+ @Autowired
+ private IQueryParserUtil queryParserUtil;
+ @Autowired
+ private ISortParserUtil sortParserUtil;
+ @Autowired
+ private IDetailedBadRequestMessageUtil detailedBadRequestMessageUtil;
+
+ private static final String GEO_SHAPE_INDEXED_TYPE = "geo_shape";
+ private static final int MINIMUM_POLYGON_POINTS_SIZE = 4;
+
+ // if returnedField contains property matching from excludes than query result will NOT include that property
+ private final Set<String> excludes = new HashSet<>(Arrays.asList(RecordMetaAttribute.X_ACL.getValue()));
+
+ // queryableExcludes properties can be returned by query results
+ private final Set<String> queryableExcludes = new HashSet<>(Arrays.asList(RecordMetaAttribute.INDEX_STATUS.getValue()));
+
+ private final TimeValue requestTimeout = TimeValue.timeValueMinutes(1);
+
+ private boolean useGeoShapeQuery = false;
+
+ QueryBuilder buildQuery(String simpleQuery, SpatialFilter spatialFilter, boolean asOwner) throws IOException {
+ QueryBuilder textQueryBuilder = null;
+ QueryBuilder spatialQueryBuilder = null;
+ QueryBuilder queryBuilder = null;
+
+ if (!Strings.isNullOrEmpty(simpleQuery)) {
+ textQueryBuilder = queryParserUtil.buildQueryBuilderFromQueryString(simpleQuery);
+ }
+
+ // use only one of the spatial request
+ if (spatialFilter != null) {
+ if (useGeoShapeQuery) {
+ if (spatialFilter.getByBoundingBox() != null) {
+ spatialQueryBuilder = getGeoShapeBoundingBoxQuery(spatialFilter);
+ } else if (spatialFilter.getByDistance() != null) {
+ spatialQueryBuilder = getGeoShapeDistanceQuery(spatialFilter);
+ } else if (spatialFilter.getByGeoPolygon() != null) {
+ spatialQueryBuilder = getGeoShapePolygonQuery(spatialFilter);
+ } else if (spatialFilter.getByIntersection() != null) {
+ spatialQueryBuilder = getGeoShapeIntersectionQuery(spatialFilter);
+ } else if (spatialFilter.getByWithinPolygon() != null) {
+ spatialQueryBuilder = getWithinPolygonQuery(spatialFilter);
+ }
+ } else {
+ if (spatialFilter.getByBoundingBox() != null) {
+ spatialQueryBuilder = getBoundingBoxQuery(spatialFilter);
+ } else if (spatialFilter.getByDistance() != null) {
+ spatialQueryBuilder = getDistanceQuery(spatialFilter);
+ } else if (spatialFilter.getByGeoPolygon() != null) {
+ spatialQueryBuilder = getGeoPolygonQuery(spatialFilter);
+ }
+ }
+ }
+
+ if (textQueryBuilder != null) {
+ queryBuilder = boolQuery().must(textQueryBuilder);
+ }
+ if (spatialQueryBuilder != null) {
+ queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(spatialQueryBuilder) : boolQuery().must(spatialQueryBuilder);
+ }
+
+ if(this.iPolicyService != null) {
+ String compiledESPolicy = this.iPolicyService.getCompiledPolicy(providerHeaderService);
+ WrapperQueryBuilder wrapperQueryBuilder = new WrapperQueryBuilder(compiledESPolicy);
+ return queryBuilder != null ? boolQuery().must(queryBuilder).must(wrapperQueryBuilder) : boolQuery().must(wrapperQueryBuilder);
+ } else {
+ return getQueryBuilderWithAuthorization(queryBuilder, asOwner);
+ }
+ }
+
+ private QueryBuilder getQueryBuilderWithAuthorization(QueryBuilder queryBuilder, boolean asOwner) {
+ if (userHasFullDataAccess()) {
+ return queryBuilder;
+ }
+
+ QueryBuilder authorizationQueryBuilder = null;
+ String groups = dpsHeaders.getHeaders().get(providerHeaderService.getDataGroupsHeader());
+ if (groups != null) {
+ String[] groupArray = groups.trim().split("\\s*,\\s*");
+ if (asOwner) {
+ authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(AclRole.OWNERS.getPath(), groupArray));
+ } else {
+ authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(RecordMetaAttribute.X_ACL.getValue(), groupArray));
+ }
+ }
+ if (authorizationQueryBuilder != null) {
+ queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(authorizationQueryBuilder) : boolQuery().must(authorizationQueryBuilder);
+ }
+ return queryBuilder;
+ }
+
+ private QueryBuilder getSimpleQuery(String searchQuery) {
+
+ // if query is empty , then put *
+ String query = StringUtils.isNotBlank(searchQuery) ? searchQuery : "*";
+ return queryStringQuery(query).allowLeadingWildcard(false);
+ }
+
+ private QueryBuilder getBoundingBoxQuery(SpatialFilter spatialFilter) {
+
+ GeoPoint topLeft = new GeoPoint(spatialFilter.getByBoundingBox().getTopLeft().getLatitude(), spatialFilter.getByBoundingBox().getTopLeft().getLongitude());
+ GeoPoint bottomRight = new GeoPoint(spatialFilter.getByBoundingBox().getBottomRight().getLatitude(), spatialFilter.getByBoundingBox().getBottomRight().getLongitude());
+ return geoBoundingBoxQuery(spatialFilter.getField()).setCorners(topLeft, bottomRight);
+ }
+
+ private QueryBuilder getDistanceQuery(SpatialFilter spatialFilter) {
+
+ return geoDistanceQuery(spatialFilter.getField())
+ .point(spatialFilter.getByDistance().getPoint().getLatitude(), spatialFilter.getByDistance().getPoint().getLongitude())
+ .distance(spatialFilter.getByDistance().getDistance(), DistanceUnit.METERS);
+ }
+
+ private QueryBuilder getGeoPolygonQuery(SpatialFilter spatialFilter) {
+
+ List<GeoPoint> points = new ArrayList<>();
+ for (Point point : spatialFilter.getByGeoPolygon().getPoints()) {
+ points.add(new GeoPoint(point.getLatitude(), point.getLongitude()));
+ }
+ return geoPolygonQuery(spatialFilter.getField(), points);
+ }
+
+ private QueryBuilder getGeoShapePolygonQuery(SpatialFilter spatialFilter) throws IOException {
+
+ List<Coordinate> points = new ArrayList<>();
+ for (Point point : spatialFilter.getByGeoPolygon().getPoints()) {
+ points.add(new Coordinate(point.getLongitude(), point.getLatitude()));
+ }
+ CoordinatesBuilder cb = new CoordinatesBuilder().coordinates(points);
+ Geometry geometry = new PolygonBuilder(cb).buildGeometry();
+ return geoWithinQuery(spatialFilter.getField(), geometry);
+ }
+
+ private QueryBuilder getGeoShapeBoundingBoxQuery(SpatialFilter spatialFilter) throws IOException {
+
+ Coordinate topLeft = new Coordinate(spatialFilter.getByBoundingBox().getTopLeft().getLongitude(), spatialFilter.getByBoundingBox().getTopLeft().getLatitude());
+ Coordinate bottomRight = new Coordinate(spatialFilter.getByBoundingBox().getBottomRight().getLongitude(), spatialFilter.getByBoundingBox().getBottomRight().getLatitude());
+ Rectangle rectangle = new EnvelopeBuilder(topLeft, bottomRight).buildGeometry();
+ return geoWithinQuery(spatialFilter.getField(), rectangle);
+ }
+
+ private QueryBuilder getGeoShapeDistanceQuery(SpatialFilter spatialFilter) throws IOException {
+
+ Coordinate center = new Coordinate(spatialFilter.getByDistance().getPoint().getLongitude(), spatialFilter.getByDistance().getPoint().getLatitude());
+ CircleBuilder circleBuilder = new CircleBuilder().center(center).radius(spatialFilter.getByDistance().getDistance(), DistanceUnit.METERS);
+ return geoWithinQuery(spatialFilter.getField(), circleBuilder);
+ }
+
+ private QueryBuilder getGeoShapeIntersectionQuery(SpatialFilter spatialFilter)
+ throws IOException {
+ MultiPolygonBuilder multiPolygonBuilder = new MultiPolygonBuilder();
+ for (Polygon polygon : spatialFilter.getByIntersection().getPolygons()) {
+ List<Coordinate> coordinates = new ArrayList<>();
+ for (Point point : polygon.getPoints()) {
+ coordinates.add(new Coordinate(point.getLongitude(), point.getLatitude()));
+ }
+
+ checkPolygon(coordinates);
+
+ CoordinatesBuilder cb = new CoordinatesBuilder().coordinates(coordinates);
+ multiPolygonBuilder.polygon(new PolygonBuilder(cb));
+ }
+
+ GeometryCollectionBuilder geometryCollection = new GeometryCollectionBuilder();
+ geometryCollection.shape(multiPolygonBuilder);
+ return geoIntersectionQuery(spatialFilter.getField(), geometryCollection.buildGeometry());
+ }
+
+ private void checkPolygon(List<Coordinate> coordinates) {
+ if (coordinates.size() < MINIMUM_POLYGON_POINTS_SIZE ||
+ (
+ coordinates.get(0).x != coordinates.get(coordinates.size() - 1).x
+ || coordinates.get(0).y != coordinates.get(coordinates.size() - 1).y
+ )
+ ) {
+ throw new AppException(HttpServletResponse.SC_BAD_REQUEST, "Bad Request",
+ String.format(
+ "Polygons must have at least %s points and the first point must match the last point",
+ MINIMUM_POLYGON_POINTS_SIZE));
+ }
+ }
+
+ private QueryBuilder getWithinPolygonQuery(SpatialFilter spatialFilter) throws IOException {
+ MultiPointBuilder multiPointBuilder = new MultiPointBuilder();
+ for (Point point : spatialFilter.getByWithinPolygon().getPoints()) {
+ multiPointBuilder.coordinate(new Coordinate(point.getLongitude(), point.getLatitude()));
+ }
+
+ GeometryCollectionBuilder geometryCollection = new GeometryCollectionBuilder();
+ geometryCollection.multiPoint(multiPointBuilder);
+ // geoWithinQuery doesn't work with a polygon as the field to search on
+ return geoIntersectionQuery(spatialFilter.getField(), geometryCollection.buildGeometry());
+ }
+
+ String getIndex(Query request) {
+ return this.crossTenantUtils.getIndexName(request);
+ }
+
+ List<Map<String, Object>> getHitsFromSearchResponse(SearchResponse searchResponse) {
+ List<Map<String, Object>> results = new ArrayList<>();
+ SearchHits searchHits = searchResponse.getHits();
+ if (searchHits.getHits().length != 0) {
+ for (SearchHit searchHitFields : searchHits.getHits()) {
+ Map<String, Object> hitFields = searchHitFields.getSourceAsMap();
+ if (!searchHitFields.getHighlightFields().isEmpty()) {
+ for (HighlightField hf : searchHitFields.getHighlightFields().values()) {
+ if (!hf.getName().equalsIgnoreCase(RecordMetaAttribute.X_ACL.getValue())) {
+ Text[] fragments = hf.getFragments();
+ if (fragments.length > 0) {
+ hitFields.put(hf.getName(), fragments[0].toString());
+ }
+ }
+ }
+ }
+ results.add(hitFields);
+ }
+ return results;
+ }
+
+ return null;
+ }
+
+ List<AggregationResponse> getAggregationFromSearchResponse(SearchResponse searchResponse) {
+ List<AggregationResponse> results = null;
+
+ if (searchResponse.getAggregations() != null) {
+ Terms kindAgg = null;
+ ParsedNested nested = searchResponse.getAggregations().get(AggregationParserUtil.NESTED_AGGREGATION_NAME);
+ if(nested != null){
+ kindAgg = (Terms) getTermsAggregationFromNested(nested);
+ }else {
+ kindAgg = searchResponse.getAggregations().get(AggregationParserUtil.TERM_AGGREGATION_NAME);
+ }
+ if (kindAgg.getBuckets() != null) {
+ results = new ArrayList<>();
+ for (Terms.Bucket bucket : kindAgg.getBuckets()) {
+ results.add(AggregationResponse.builder().key(bucket.getKeyAsString()).count(bucket.getDocCount()).build());
+ }
+ }
+ }
+
+ return results;
+ }
+
+ private Aggregation getTermsAggregationFromNested(ParsedNested parsedNested){
+ ParsedNested nested = parsedNested.getAggregations().get(AggregationParserUtil.NESTED_AGGREGATION_NAME);
+ if(nested != null){
+ return getTermsAggregationFromNested(nested);
+ }else {
+ return parsedNested.getAggregations().get(AggregationParserUtil.TERM_AGGREGATION_NAME);
+ }
+ }
+
+ SearchSourceBuilder createSearchSourceBuilder(Query request) throws IOException {
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+
+ // build query: set query options and query
+ QueryBuilder queryBuilder = buildQuery(request.getQuery(), request.getSpatialFilter(), request.isQueryAsOwner());
+ sourceBuilder.size(QueryUtils.getResultSizeForQuery(request.getLimit()));
+ sourceBuilder.query(queryBuilder);
+ sourceBuilder.timeout(requestTimeout);
+ if (request.isTrackTotalCount()) {
+ sourceBuilder.trackTotalHits(request.isTrackTotalCount());
+ }
+
+ // set highlighter
+ if (request.isReturnHighlightedFields()) {
+ HighlightBuilder highlightBuilder = new HighlightBuilder().field("*", 200, 5);
+ sourceBuilder.highlighter(highlightBuilder);
+ }
+
+ // set the return fields
+ List<String> returnedFields = request.getReturnedFields();
+ if (returnedFields == null) {
+ returnedFields = new ArrayList<>();
+ }
+ Set<String> returnedFieldsSet = new HashSet<>(returnedFields);
+ String[] includesArr = returnedFieldsSet.toArray(new String[returnedFieldsSet.size()]);
+
+ // remove all matching returnedField and queryable from excludes
+ Set<String> requestQueryableExcludes = new HashSet<>(queryableExcludes);
+ Set<String> requestExcludes = new HashSet<>(excludes);
+ requestQueryableExcludes.removeAll(returnedFields);
+ requestExcludes.addAll(requestQueryableExcludes);
+ String[] excludesArr = requestExcludes.toArray(new String[requestExcludes.size()]);
+
+ sourceBuilder.fetchSource(includesArr, excludesArr);
+ return sourceBuilder;
+ }
+
+ SearchResponse makeSearchRequest(Query searchRequest, RestHighLevelClient client) {
+ Long startTime = 0L;
+ SearchRequest elasticSearchRequest = null;
+ SearchResponse searchResponse = null;
+ try {
+ String index = this.getIndex(searchRequest);
+ if (searchRequest.getSpatialFilter() != null) {
+ useGeoShapeQuery = this.useGeoShapeQuery(client, searchRequest, index);
+ }
+ elasticSearchRequest = createElasticRequest(searchRequest);
+ if (searchRequest.getSort() != null) {
+ List<FieldSortBuilder> sortBuilders = this.sortParserUtil.getSortQuery(client, searchRequest.getSort(), index);
+ for (FieldSortBuilder fieldSortBuilder : sortBuilders) {
+ elasticSearchRequest.source().sort(fieldSortBuilder);
+ }
+ }
+
+ startTime = System.currentTimeMillis();
+ searchResponse = client.search(elasticSearchRequest, RequestOptions.DEFAULT);
+ return searchResponse;
+ } catch (ElasticsearchStatusException e) {
+ switch (e.status()) {
+ case NOT_FOUND:
+ throw new AppException(HttpServletResponse.SC_NOT_FOUND, "Not Found", "Resource you are trying to find does not exists", e);
+ case BAD_REQUEST:
+ throw new AppException(HttpServletResponse.SC_BAD_REQUEST, "Bad Request", detailedBadRequestMessageUtil.getDetailedBadRequestMessage(elasticSearchRequest, e), e);
+ case SERVICE_UNAVAILABLE:
+ throw new AppException(HttpServletResponse.SC_SERVICE_UNAVAILABLE, SEARCH_ERROR_MSG, "Please re-try search after some time.", e);
+ default:
+ throw new AppException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, SEARCH_ERROR_MSG, ERROR_PROCESSING_SEARCH_REQUEST_MSG, e);
+ }
+ } catch (AppException e){
+ throw e;
+ } catch (IOException e) {
+ if (e.getMessage().startsWith("listener timeout after waiting for")) {
+ throw new AppException(HttpServletResponse.SC_GATEWAY_TIMEOUT, SEARCH_ERROR_MSG, String.format("Request timed out after waiting for %sm", requestTimeout.getMinutes()), e);
+ }
+ throw new AppException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, SEARCH_ERROR_MSG, ERROR_PROCESSING_SEARCH_REQUEST_MSG, e);
+ } catch (Exception e) {
+ if(e instanceof java.net.SocketTimeoutException){
+ throw new AppException(HttpServletResponse.SC_REQUEST_TIMEOUT, SEARCH_ERROR_MSG, String.format("Request timed out after waiting for %sm", requestTimeout.getMinutes()), e);
+ }
+ throw new AppException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, SEARCH_ERROR_MSG, ERROR_PROCESSING_SEARCH_REQUEST_MSG, e);
+ } finally {
+ Long latency = System.currentTimeMillis() - startTime;
+ String request = elasticSearchRequest != null ? elasticSearchRequest.source().toString() : searchRequest.toString();
+ this.log.info(String.format("elastic latency: %s | elastic request-payload: %s", latency, request));
+ this.auditLog(searchRequest, searchResponse);
+ }
+ }
+
+ private boolean useGeoShapeQuery(RestHighLevelClient client, Query searchRequest, String index) throws IOException {
+ Set<String> indexedTypes = this.fieldMappingTypeService.getFieldTypes(client, searchRequest.getSpatialFilter().getField(), index);
+ // fallback to geo_point search if mixed type found for spatialFilter.field
+ if (indexedTypes.isEmpty() || indexedTypes.size() > 1) return false;
+ return indexedTypes.contains(GEO_SHAPE_INDEXED_TYPE);
+ }
+
+ abstract SearchRequest createElasticRequest(Query request) throws IOException;
+
+ abstract void querySuccessAuditLogger(Query request);
+
+ abstract void queryFailedAuditLogger(Query request);
+
+ private void auditLog(Query searchRequest, SearchResponse searchResponse) {
+ if (searchResponse != null && searchResponse.status() == RestStatus.OK) {
+ this.querySuccessAuditLogger(searchRequest);
+ return;
+ }
+ this.queryFailedAuditLogger(searchRequest);
+ }
+
+ private boolean userHasFullDataAccess() {
+ String dataRootUser = dpsHeaders.getHeaders().getOrDefault(providerHeaderService.getDataRootUserHeader(), "false");
+ return Boolean.parseBoolean(dataRootUser);
+ }
+}
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceImpl.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceImpl.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceImpl.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceImpl.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceImpl.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceImpl.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceImpl.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceImpl.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/security/GcpSecurityConfig.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/security/GcpSecurityConfig.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/security/GcpSecurityConfig.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/security/GcpSecurityConfig.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/service/ElasticSettingServiceImpl.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/service/ElasticSettingServiceImpl.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/service/ElasticSettingServiceImpl.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/service/ElasticSettingServiceImpl.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/ConfigGcpModule.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/ConfigGcpModule.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/ConfigGcpModule.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/ConfigGcpModule.java
diff --git a/provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/TraceIdExtractor.java b/provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/TraceIdExtractor.java
similarity index 100%
rename from provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/TraceIdExtractor.java
rename to provider/search-gc/src/main/java/org/opengroup/osdu/search/provider/gcp/utils/TraceIdExtractor.java
diff --git a/provider/search-gcp/src/main/resources/application-anthos.properties b/provider/search-gc/src/main/resources/application-anthos.properties
similarity index 100%
rename from provider/search-gcp/src/main/resources/application-anthos.properties
rename to provider/search-gc/src/main/resources/application-anthos.properties
diff --git a/provider/search-gcp/src/main/resources/application-gcp.properties b/provider/search-gc/src/main/resources/application-gcp.properties
similarity index 100%
rename from provider/search-gcp/src/main/resources/application-gcp.properties
rename to provider/search-gc/src/main/resources/application-gcp.properties
diff --git a/provider/search-gcp/src/main/resources/application.properties b/provider/search-gc/src/main/resources/application.properties
similarity index 100%
rename from provider/search-gcp/src/main/resources/application.properties
rename to provider/search-gc/src/main/resources/application.properties
diff --git a/provider/search-gcp/src/main/resources/logback.xml b/provider/search-gc/src/main/resources/logback.xml
similarity index 100%
rename from provider/search-gcp/src/main/resources/logback.xml
rename to provider/search-gc/src/main/resources/logback.xml
diff --git a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/middleware/RedirectHttpRequestsHandlerTest.java b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/middleware/RedirectHttpRequestsHandlerTest.java
similarity index 100%
rename from provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/middleware/RedirectHttpRequestsHandlerTest.java
rename to provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/middleware/RedirectHttpRequestsHandlerTest.java
diff --git a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceTest.java b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceTest.java
similarity index 100%
rename from provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceTest.java
rename to provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/CrossTenantInfoServiceTest.java
diff --git a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceTest.java b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceTest.java
similarity index 97%
rename from provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceTest.java
rename to provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceTest.java
index a030a87ac..3b8aed1fc 100644
--- a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceTest.java
+++ b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/QueryServiceTest.java
@@ -1,882 +1,882 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.opengroup.osdu.search.provider.gcp.provider.impl;
-
-import static org.junit.Assert.*;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import javax.servlet.http.HttpServletResponse;
-import org.apache.lucene.search.TotalHits;
-import org.apache.lucene.search.TotalHits.Relation;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchResponseSections;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder;
-import org.elasticsearch.index.query.GeoDistanceQueryBuilder;
-import org.elasticsearch.index.query.GeoPolygonQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryStringQueryBuilder;
-import org.elasticsearch.index.query.TermsQueryBuilder;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentCaptor;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.Spy;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.Point;
-import org.opengroup.osdu.core.common.model.search.Polygon;
-import org.opengroup.osdu.core.common.model.search.QueryRequest;
-import org.opengroup.osdu.core.common.model.search.QueryResponse;
-import org.opengroup.osdu.core.common.model.search.SortOrder;
-import org.opengroup.osdu.core.common.model.search.SortQuery;
-import org.opengroup.osdu.core.common.model.search.SpatialFilter;
-import org.opengroup.osdu.core.common.model.search.SpatialFilter.ByBoundingBox;
-import org.opengroup.osdu.core.common.model.search.SpatialFilter.ByDistance;
-import org.opengroup.osdu.core.common.model.search.SpatialFilter.ByGeoPolygon;
-import org.opengroup.osdu.search.config.SearchConfigurationProperties;
-import org.opengroup.osdu.search.logging.AuditLogger;
-import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
-import org.opengroup.osdu.search.service.IFieldMappingTypeService;
-import org.opengroup.osdu.search.util.AggregationParserUtil;
-import org.opengroup.osdu.search.util.CrossTenantUtils;
-import org.opengroup.osdu.search.util.DetailedBadRequestMessageUtil;
-import org.opengroup.osdu.search.util.ElasticClientHandler;
-import org.opengroup.osdu.search.util.IAggregationParserUtil;
-import org.opengroup.osdu.search.util.IDetailedBadRequestMessageUtil;
-import org.opengroup.osdu.search.util.IQueryParserUtil;
-import org.opengroup.osdu.search.util.ISortParserUtil;
-import org.opengroup.osdu.search.util.QueryParserUtil;
-import org.opengroup.osdu.search.util.SortParserUtil;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({SearchRequest.class, SearchHits.class, RestHighLevelClient.class})
-public class QueryServiceTest {
-
- private final ObjectMapper objectMapper = new ObjectMapper();
- @Mock
- private SpatialFilter spatialFilter;
- @Mock
- private ByBoundingBox byBoundingBox;
- @Mock
- private ByDistance byDistance;
- @Mock
- private ByGeoPolygon byGeoPolygon;
- @Mock
- private ElasticClientHandler elasticClientHandler;
- @Mock
- private QueryRequest searchRequest;
- @Mock
- private DpsHeaders dpsHeaders;
- @Mock
- private CrossTenantUtils crossTenantUtils;
- @Mock
- private IFieldMappingTypeService fieldMappingTypeService;
- @Mock
- private AuditLogger auditLogger;
- @Mock
- private JaxRsDpsLog log;
- @Mock
- private IProviderHeaderService providerHeaderService;
- @Spy
- private SearchConfigurationProperties properties = new SearchConfigurationProperties();
- @Spy
- private IQueryParserUtil parserService = new QueryParserUtil();
- @Spy
- private ISortParserUtil sortParserUtil = new SortParserUtil();
- @Spy
- private IAggregationParserUtil aggregationParserUtil = new AggregationParserUtil(properties);
- @Spy
- private IDetailedBadRequestMessageUtil detailedBadRequestMessageUtil = new DetailedBadRequestMessageUtil(
- objectMapper);
-
- @Mock
- private RestHighLevelClient restHighLevelClient;
-
- private SearchRequest elasticSearchRequest;
-
- @InjectMocks
- @Spy
- private QueryServiceImpl sut = new QueryServiceImpl();
-
- private final String DATA_GROUPS = "X-Data-Groups";
- private final String DATA_GROUP_1 = "data.welldb.viewers@common.evd.cloud.slb-ds.com";
-
- private final String DATA_GROUP_2 = "data.npd.viewers@common.evd.cloud.slb-ds.com";
-
- @Before
- public void setup() {
- initMocks(this);
-
- mockStatic(RestHighLevelClient.class);
- mockStatic(SearchRequest.class);
- mockStatic(SearchHits.class);
-
- when(properties.getEnvironment()).thenReturn("evt");
- restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
-
- Map<String, String> HEADERS = new HashMap<>();
- HEADERS.put(DpsHeaders.ACCOUNT_ID, "tenant1");
- HEADERS.put(DpsHeaders.AUTHORIZATION, "Bearer blah");
- HEADERS.put(DATA_GROUPS, String.format("%s,%s", DATA_GROUP_1, DATA_GROUP_2));
-
- when(providerHeaderService.getDataGroupsHeader()).thenReturn(DATA_GROUPS);
- when(dpsHeaders.getHeaders()).thenReturn(HEADERS);
- }
-
- @Test
- public void should_returnSimpleQuery_when_request_queryIsSpecified() throws Exception {
-
- Map<String, Object> hit = new HashMap<>();
- hit.put("_id", "tenant1:welldb:wellbore-33fe05e1-df20-49d9-bd63-74cf750a206f");
- hit.put("type", "wellbore");
-
- List<Map<String, Object>> results = new ArrayList<>();
- results.add(hit);
-
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- TotalHits totalHits = new TotalHits(1, Relation.EQUAL_TO);
- SearchHits searchHits = new SearchHits(new SearchHit[0], totalHits, 2);
- SearchResponse mockSearchResponse = new SearchResponse(
- new SearchResponseSections(searchHits, null,
- null, false, false, null, 1), "2",
- 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
- SearchResponse.Clusters.EMPTY);
-
- doReturn(mockSearchResponse).when(this.sut).makeSearchRequest(any(), any());
- doReturn(results).when(this.sut).getHitsFromSearchResponse(any());
-
- QueryResponse queryResponse = this.sut.queryIndex(searchRequest);
- assertNotNull(queryResponse);
- assertEquals(1, queryResponse.getTotalCount());
- }
-
- @Test
- public void should_returnRightTotalCount_when_queryResponseResultsIsNull() throws Exception {
- List<Map<String, Object>> results = null;
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- TotalHits totalHits = new TotalHits(1, Relation.EQUAL_TO);
- SearchHits searchHits = new SearchHits(new SearchHit[0], totalHits, 2);
- SearchResponse mockSearchResponse = new SearchResponse(
- new SearchResponseSections(searchHits, null,
- null, false, false, null, 1), "2",
- 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
- SearchResponse.Clusters.EMPTY);
-
- doReturn(mockSearchResponse).when(this.sut).makeSearchRequest(any(), any());
- doReturn(results).when(this.sut).getHitsFromSearchResponse(any());
-
- QueryResponse queryResponse = this.sut.queryIndex(searchRequest);
- assertNotNull(queryResponse);
- assertEquals(1, queryResponse.getTotalCount());
- }
-
- @Test
- public void should_throwElasticException_when_indexNotFound() throws Exception {
-
- ElasticsearchStatusException notFound = new ElasticsearchStatusException("blah",
- RestStatus.NOT_FOUND);
-
- doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
- PowerMockito.when(restHighLevelClient.search(any(), any(RequestOptions.class)))
- .thenThrow(notFound);
-
- try {
- this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
- fail("Should not succeed!");
- } catch (AppException e) {
- assertEquals("Resource you are trying to find does not exists", e.getError().getMessage());
- }
- }
-
- @Test
- public void should_throwElasticException_given_badRequest() throws Exception {
-
- ElasticsearchStatusException badRequest = new ElasticsearchStatusException("blah",
- RestStatus.BAD_REQUEST);
-
- doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
- when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(badRequest);
-
- try {
- this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
- fail("Should not succeed!");
- } catch (AppException e) {
- assertEquals("Invalid parameters were given on search request", e.getError().getMessage());
- }
- }
-
- @Test
- public void should_throwElasticException_given_searchError() throws Exception {
-
- ElasticsearchStatusException generic = new ElasticsearchStatusException("blah",
- RestStatus.BAD_GATEWAY);
-
- doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
- when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(generic);
-
- try {
- this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
- fail("Should not succeed!");
- } catch (AppException e) {
- assertEquals("Error processing search request", e.getError().getMessage());
- }
- }
-
- @Test
- public void should_throwSystemException_given_genericError() throws Exception {
-
- IllegalArgumentException exception = new IllegalArgumentException("search on fire");
-
- doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
- when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(exception);
-
- try {
- this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
- fail("Should not succeed!");
- } catch (AppException e) {
- assertEquals("Error processing search request", e.getError().getMessage());
- }
- }
-
- @Test
- public void should_throwTimeoutException_given_timeoutError() throws Exception {
-
- IOException exception = new IOException("listener timeout after waiting for [60000] ms");
-
- doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
- when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(exception);
-
- try {
- this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
- fail("Should not succeed!");
- } catch (AppException e) {
- assertEquals(HttpServletResponse.SC_GATEWAY_TIMEOUT, e.getError().getCode());
- assertEquals("Request timed out after waiting for 1m", e.getError().getMessage());
- }
- }
-
- @Test
- public void should_throwSystemException_given_genericIOError() throws Exception {
-
- IOException exception = new IOException("search on fire");
-
- doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
- when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(exception);
-
- try {
- this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
- fail("Should not succeed!");
- } catch (AppException e) {
- assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
- assertEquals("Error processing search request", e.getError().getMessage());
- }
- }
-
- @Test
- public void should_return_textQuery_when_requestHasIt() throws IOException {
-
- String simpleQuery = "find-me";
- BoolQueryBuilder builder = (BoolQueryBuilder) this.sut.buildQuery(simpleQuery, null, true);
- assertNotNull(builder);
-
- List<QueryBuilder> topLevelMustClause = builder.must();
- assertEquals(2, topLevelMustClause.size());
-
- BoolQueryBuilder queryLevelBuilder = (BoolQueryBuilder) topLevelMustClause.get(0);
- assertNotNull(queryLevelBuilder);
-
- List<QueryBuilder> queryLevelMustClause = queryLevelBuilder.must();
- assertEquals(1, queryLevelMustClause.size());
-
- BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryLevelMustClause.get(0);
- List<QueryBuilder> must = boolQueryBuilder.must();
- QueryStringQueryBuilder queryBuilder = (QueryStringQueryBuilder) must.get(0);
-
- assertNotNull(boolQueryBuilder);
- assertEquals(simpleQuery, queryBuilder.queryString());
-
- verifyAcls(topLevelMustClause.get(1), true);
- }
-
- @Test
- public void should_searchAll_when_requestHas_noQueryString() throws IOException {
-
- BoolQueryBuilder builder = (BoolQueryBuilder) this.sut.buildQuery(null, null, true);
- assertNotNull(builder);
-
- List<QueryBuilder> topLevelMustClause = builder.must();
- assertEquals(1, topLevelMustClause.size());
-
- verifyAcls(topLevelMustClause.get(0), true);
- }
-
- @Test
- public void should_return_ownerOnlyMustClause_when_searchAsOwners() throws IOException {
-
- BoolQueryBuilder builder = (BoolQueryBuilder) this.sut.buildQuery(null, null, false);
- assertNotNull(builder);
-
- List<QueryBuilder> topLevelMustClause = builder.must();
- assertEquals(1, topLevelMustClause.size());
-
- verifyAcls(topLevelMustClause.get(0), false);
- }
-
- @Test
- public void should_return_nullQuery_when_searchAsDataRootUser() throws IOException {
- Map<String, String> HEADERS = new HashMap<>();
- HEADERS.put(DpsHeaders.ACCOUNT_ID, "tenant1");
- HEADERS.put(DpsHeaders.AUTHORIZATION, "Bearer blah");
- HEADERS.put(DATA_GROUPS, String.format("%s,%s", DATA_GROUP_1, DATA_GROUP_2));
- HEADERS.put(providerHeaderService.getDataRootUserHeader(), "true");
- when(dpsHeaders.getHeaders()).thenReturn(HEADERS);
-
- QueryBuilder builder = this.sut.buildQuery(null, null, false);
- assertNull(builder);
- }
-
- @Test(expected = AppException.class)
- public void testQueryBase_whenUnsupportedSortRequested_statusBadRequest_throwsException()
- throws IOException {
- String fieldName = "field";
- String indexName = "index";
- String dummySortError = "Text fields are not optimised for operations that require per-document field data like aggregations and sorting, so these operations are disabled by default. Please use a keyword field instead";
- ElasticsearchStatusException exception = new ElasticsearchStatusException("blah",
- RestStatus.BAD_REQUEST, new ElasticsearchException(dummySortError));
-
- doThrow(exception).when(restHighLevelClient).search(any(), any(RequestOptions.class));
- doReturn(new HashSet<>()).when(fieldMappingTypeService)
- .getFieldTypes(eq(restHighLevelClient), eq(fieldName), eq(indexName));
- SortQuery sortQuery = new SortQuery();
- sortQuery.setField(Collections.singletonList("name"));
- sortQuery.setOrder(Collections.singletonList(SortOrder.DESC));
- when(searchRequest.getSort()).thenReturn(sortQuery);
- when(sortParserUtil.getSortQuery(restHighLevelClient, sortQuery, indexName))
- .thenReturn(Collections.singletonList(
- new FieldSortBuilder("name").order(org.elasticsearch.search.sort.SortOrder.DESC)));
-
- try {
- this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
- } catch (AppException e) {
- int errorCode = 400;
- String errorMessage = "Sort is not supported for one or more of the requested fields";
- assertEquals(e.getError().getCode(), errorCode);
- assertEquals(e.getError().getMessage(), errorMessage);
-
- throw (e);
- }
- }
-
- @Test
- public void should_return_boundingBoxQuery_given_spatialCriteria() throws IOException {
-
- String simpleQuery = "dare-find-me";
-
- String field = "LonLat";
- Double bottomRightLon = 124.174762;
- Double bottomRightLat = 36.450727;
- Double topLeftLon = 37.450727;
- Double topLeftLat = 70.174762;
-
- when(this.spatialFilter.getField()).thenReturn(field);
- when(this.spatialFilter.getByBoundingBox()).thenReturn(byBoundingBox);
- when(this.spatialFilter.getByBoundingBox().getBottomRight()).thenReturn(mock(Point.class));
- when(this.spatialFilter.getByBoundingBox().getBottomRight().getLongitude())
- .thenReturn(bottomRightLon);
- when(this.spatialFilter.getByBoundingBox().getBottomRight().getLatitude())
- .thenReturn(bottomRightLat);
- when(this.spatialFilter.getByBoundingBox().getTopLeft()).thenReturn(mock(Point.class));
- when(this.spatialFilter.getByBoundingBox().getTopLeft().getLongitude()).thenReturn(37.450727);
- when(this.spatialFilter.getByBoundingBox().getTopLeft().getLatitude()).thenReturn(70.174762);
-
- BoolQueryBuilder builder = (BoolQueryBuilder) this.sut
- .buildQuery(simpleQuery, this.spatialFilter, true);
- assertNotNull(builder);
-
- List<QueryBuilder> topLevelMustClause = builder.must();
- assertEquals(2, topLevelMustClause.size());
-
- BoolQueryBuilder queryLevelBuilder = (BoolQueryBuilder) topLevelMustClause.get(0);
- assertNotNull(queryLevelBuilder);
-
- List<QueryBuilder> queryLevelMustClause = queryLevelBuilder.must();
- assertEquals(2, queryLevelMustClause.size());
-
- BoolQueryBuilder queryStringBoolQueryBuilder = (BoolQueryBuilder) queryLevelMustClause.get(0);
- assertNotNull(queryStringBoolQueryBuilder);
-
- GeoBoundingBoxQueryBuilder geoBoundingBoxQueryBuilder = (GeoBoundingBoxQueryBuilder) queryLevelMustClause
- .get(1);
- assertNotNull(geoBoundingBoxQueryBuilder);
- assertEquals(field, geoBoundingBoxQueryBuilder.fieldName());
- assertEquals(topLeftLon, geoBoundingBoxQueryBuilder.topLeft().getLon(), .001);
- assertEquals(topLeftLat, geoBoundingBoxQueryBuilder.topLeft().getLat(), .001);
- assertEquals(bottomRightLon, geoBoundingBoxQueryBuilder.bottomRight().getLon(), .001);
- assertEquals(bottomRightLat, geoBoundingBoxQueryBuilder.bottomRight().getLat(), .001);
-
- verifyAcls(topLevelMustClause.get(1), true);
- }
-
- @Test
- public void should_return_distanceQuery_given_spatialCriteria() throws IOException {
-
- String simpleQuery = "oh no you found me";
-
- String field = "LonLat";
- Double pointLon = 124.174762;
- Double pointLat = 36.450727;
- Double distance = 10.0;
-
- when(this.spatialFilter.getField()).thenReturn(field);
- when(this.spatialFilter.getByDistance()).thenReturn(byDistance);
- when(this.spatialFilter.getByDistance().getPoint()).thenReturn(mock(Point.class));
- when(this.spatialFilter.getByDistance().getPoint().getLongitude()).thenReturn(pointLon);
- when(this.spatialFilter.getByDistance().getPoint().getLatitude()).thenReturn(pointLat);
- when(this.spatialFilter.getByDistance().getDistance()).thenReturn(distance);
-
- BoolQueryBuilder builder = (BoolQueryBuilder) this.sut
- .buildQuery(simpleQuery, this.spatialFilter, true);
- assertNotNull(builder);
-
- List<QueryBuilder> topLevelMustClause = builder.must();
- assertEquals(2, topLevelMustClause.size());
-
- BoolQueryBuilder queryLevelBuilder = (BoolQueryBuilder) topLevelMustClause.get(0);
- assertNotNull(queryLevelBuilder);
-
- List<QueryBuilder> queryLevelMustClause = queryLevelBuilder.must();
- assertEquals(2, queryLevelMustClause.size());
-
- BoolQueryBuilder queryStringBoolQueryBuilder = (BoolQueryBuilder) queryLevelMustClause.get(0);
- assertNotNull(queryStringBoolQueryBuilder);
-
- GeoDistanceQueryBuilder geoDistanceQueryBuilder = (GeoDistanceQueryBuilder) queryLevelMustClause
- .get(1);
- assertNotNull(geoDistanceQueryBuilder);
- assertEquals(field, geoDistanceQueryBuilder.fieldName());
- assertEquals(distance, geoDistanceQueryBuilder.distance(), .001);
- assertEquals(pointLon, geoDistanceQueryBuilder.point().getLon(), .001);
- assertEquals(pointLat, geoDistanceQueryBuilder.point().getLat(), .001);
-
- verifyAcls(topLevelMustClause.get(1), true);
- }
-
- @Test
- public void should_return_polygonBoxQuery_given_spatialCriteria() throws IOException {
-
- String simpleQuery = "polygons are fun";
-
- String field = "LonLat";
- List<Point> points = new ArrayList<>();
- points.add(new Point(124.174762, 36.450727));
- points.add(new Point(126.174762, 36.450727));
- points.add(new Point(128.174762, 46.450727));
- points.add(new Point(124.174762, 36.450727));
-
- when(this.spatialFilter.getField()).thenReturn(field);
- when(this.spatialFilter.getByGeoPolygon()).thenReturn(byGeoPolygon);
- when(this.spatialFilter.getByGeoPolygon().getPoints()).thenReturn(points);
-
- BoolQueryBuilder builder = (BoolQueryBuilder) this.sut
- .buildQuery(simpleQuery, this.spatialFilter, true);
- assertNotNull(builder);
-
- List<QueryBuilder> topLevelMustClause = builder.must();
- assertEquals(2, topLevelMustClause.size());
-
- BoolQueryBuilder queryLevelBuilder = (BoolQueryBuilder) topLevelMustClause.get(0);
- assertNotNull(queryLevelBuilder);
-
- List<QueryBuilder> queryLevelMustClause = queryLevelBuilder.must();
- assertEquals(2, queryLevelMustClause.size());
-
- BoolQueryBuilder queryStringBoolQueryBuilder = (BoolQueryBuilder) queryLevelMustClause.get(0);
- assertNotNull(queryStringBoolQueryBuilder);
-
- GeoPolygonQueryBuilder geoPolygonQueryBuilder = (GeoPolygonQueryBuilder) queryLevelMustClause
- .get(1);
- assertNotNull(geoPolygonQueryBuilder);
- assertEquals(field, geoPolygonQueryBuilder.fieldName());
- assertEquals(points.size(), geoPolygonQueryBuilder.points().size());
- List<GeoPoint> points1 = geoPolygonQueryBuilder.points();
- for (int i = 0; i < points1.size(); i++) {
- GeoPoint geoPoint = points1.get(i);
- Point requestPoint = points.get(i);
- assertEquals(requestPoint.getLongitude(), geoPoint.getLon(), .001);
- assertEquals(requestPoint.getLatitude(), geoPoint.getLat(), .001);
- }
-
- verifyAcls(topLevelMustClause.get(1), true);
- }
-
- @Test
- public void should_return_correctElasticRequest_given_requestQuery() throws IOException {
- int limit = 5;
- int from = 2;
- String kind = "tenant1:welldb:well:1.0.0";
-
- List<String> returnedFields = new ArrayList<>();
- returnedFields.add("id");
- when(searchRequest.getKind()).thenReturn(kind);
- when(searchRequest.getLimit()).thenReturn(limit);
- when(searchRequest.getFrom()).thenReturn(from);
- when(searchRequest.getReturnedFields()).thenReturn(returnedFields);
-
- when(crossTenantUtils.getIndexName(any())).thenReturn("tenant1-welldb-well-1.0.0,-.*");
-
- SearchRequest elasticRequest = this.sut.createElasticRequest(searchRequest);
- assertNotNull(elasticRequest);
-
- String[] indices = elasticRequest.indices();
- assertEquals(1, indices.length);
- assertEquals("tenant1-welldb-well-1.0.0,-.*", indices[0]);
-
- SearchSourceBuilder elasticSearchSourceBuilder = elasticRequest.source();
- assertNotNull(elasticSearchSourceBuilder);
- assertEquals(limit, elasticSearchSourceBuilder.size());
- assertEquals(from, elasticSearchSourceBuilder.from());
- assertEquals(1, elasticSearchSourceBuilder.timeout().getMinutes());
-
- FetchSourceContext elasticFetchSourceContext = elasticSearchSourceBuilder.fetchSource();
- assertNotNull(elasticFetchSourceContext);
-
- String[] elasticExcludes = elasticFetchSourceContext.excludes();
- assertEquals(2, elasticExcludes.length);
- assertEquals("x-acl", elasticExcludes[0]);
- assertEquals("index", elasticExcludes[1]);
-
- String[] elasticIncludes = elasticFetchSourceContext.includes();
- assertEquals(1, elasticIncludes.length);
- assertEquals("id", elasticIncludes[0]);
-
- QueryBuilder elasticQueryBuilder = elasticSearchSourceBuilder.query();
- assertNotNull(elasticQueryBuilder);
- }
-
- @Test
- public void should_return_correctElasticRequest_given_returnedFieldContainsQueryableExcludes()
- throws IOException {
-
- List<String> returnedFields = new ArrayList<>(Arrays.asList("id", "index"));
- when(searchRequest.getKind()).thenReturn("tenant1:welldb:well:1.0.0");
- when(searchRequest.getReturnedFields()).thenReturn(returnedFields);
-
- when(crossTenantUtils.getIndexName(any())).thenReturn("tenant1-welldb-well-1.0.0,-.*");
-
- SearchRequest elasticRequest = this.sut.createElasticRequest(searchRequest);
- assertNotNull(elasticRequest);
-
- String[] indices = elasticRequest.indices();
- assertEquals(1, indices.length);
- assertEquals("tenant1-welldb-well-1.0.0,-.*", indices[0]);
-
- SearchSourceBuilder elasticSearchSourceBuilder = elasticRequest.source();
- assertNotNull(elasticSearchSourceBuilder);
-
- FetchSourceContext elasticFetchSourceContext = elasticSearchSourceBuilder.fetchSource();
- assertNotNull(elasticFetchSourceContext);
-
- String[] elasticExcludes = elasticFetchSourceContext.excludes();
- assertEquals(1, elasticExcludes.length);
- assertEquals("x-acl", elasticExcludes[0]);
-
- List<String> elasticIncludes = Arrays.asList(elasticFetchSourceContext.includes());
- assertEquals(2, elasticIncludes.size());
- assertTrue(elasticIncludes.contains("index"));
- assertTrue(elasticIncludes.contains("id"));
- }
-
- @Test
- public void should_return_correctElasticRequest_given_noReturnedField() throws IOException {
-
- when(searchRequest.getKind()).thenReturn("tenant1:welldb:well:1.0.0");
- when(crossTenantUtils.getIndexName(any())).thenReturn("tenant1-welldb-well-1.0.0,-.*");
-
- SearchRequest elasticRequest = this.sut.createElasticRequest(searchRequest);
- assertNotNull(elasticRequest);
-
- String[] indices = elasticRequest.indices();
- assertEquals(1, indices.length);
- assertEquals("tenant1-welldb-well-1.0.0,-.*", indices[0]);
-
- SearchSourceBuilder elasticSearchSourceBuilder = elasticRequest.source();
- assertNotNull(elasticSearchSourceBuilder);
-
- FetchSourceContext elasticFetchSourceContext = elasticSearchSourceBuilder.fetchSource();
- assertNotNull(elasticFetchSourceContext);
-
- List<String> elasticExcludes = Arrays.asList(elasticFetchSourceContext.excludes());
- assertEquals(2, elasticExcludes.size());
- assertTrue(elasticExcludes.contains("index"));
- assertTrue(elasticExcludes.contains("x-acl"));
-
- List<String> elasticIncludes = Arrays.asList(elasticFetchSourceContext.includes());
- assertEquals(0, elasticIncludes.size());
- }
-
- @Test
- public void should_return_correctElasticRequest_given_groupByField() throws IOException {
- when(searchRequest.getKind()).thenReturn("tenant1:welldb:well:1.0.0");
- when(searchRequest.getAggregateBy()).thenReturn("namespace");
- when(crossTenantUtils.getIndexName(any())).thenReturn("tenant1-welldb-well-1.0.0,-.*");
-
- SearchRequest elasticRequest = this.sut.createElasticRequest(searchRequest);
- assertNotNull(elasticRequest);
- assertNotNull(elasticRequest.source().aggregations());
- assertEquals(1, elasticRequest.source().aggregations().count());
- }
-
- private void verifyAcls(QueryBuilder aclMustClause, boolean asOwner) {
- BoolQueryBuilder aclLevelBuilder = (BoolQueryBuilder) aclMustClause;
- assertNotNull(aclLevelBuilder);
- assertEquals("1", aclLevelBuilder.minimumShouldMatch());
-
- List<QueryBuilder> aclShouldClause = aclLevelBuilder.should();
- assertEquals(1, aclShouldClause.size());
-
- TermsQueryBuilder aclQuery = (TermsQueryBuilder) aclShouldClause.get(0);
- assertNotNull(aclQuery);
- if (asOwner) {
- assertEquals("acl.owners", aclQuery.fieldName());
- } else {
- assertEquals("x-acl", aclQuery.fieldName());
- }
- assertEquals(2, aclQuery.values().size());
-
- List<Object> acls = aclQuery.values();
- assertEquals(2, acls.size());
- assertTrue(acls.contains(DATA_GROUP_1));
- assertTrue(acls.contains(DATA_GROUP_2));
- }
-
- /* arrange
- create query request according to this example query:
- {
- "kind": "osdu:wks:reference-data--CoordinateTransformation:1.0.0",
- "query": "data.ID:\"EPSG::1078\"",
- "spatialFilter": {
- "field": "data.Wgs84Coordinates",
- "byIntersection": {
- "polygons": [
- {
- "points": [
- {
- "latitude": 10.75,
- "longitude": -8.61
- }
- ]
- }
- ]
- }
- }
- }*/
- @Test
- public void should_return_CorrectQueryResponseForIntersectionSpatialFilter() throws Exception {
- QueryRequest queryRequest = new QueryRequest();
- queryRequest.setQuery("data.ID:\"EPSG::1078\"");
-
- SpatialFilter spatialFilter = new SpatialFilter();
- spatialFilter.setField("data.Wgs84Coordinates");
-
- SpatialFilter.ByIntersection byIntersection = new SpatialFilter.ByIntersection();
- Polygon polygon = new Polygon();
- Point point = new Point(1.02, -8.61);
- Point point1 = new Point(1.02, -2.48);
- Point point2 = new Point(10.74, -2.48);
- Point point3 = new Point(10.74, -8.61);
- Point point4 = new Point(1.02, -8.61);
-
- List<Point> points = new ArrayList<>();
- points.add(point);
- points.add(point1);
- points.add(point2);
- points.add(point3);
- points.add(point4);
- polygon.setPoints(points);
-
- List<Polygon> polygons = new ArrayList<>();
- polygons.add(polygon);
- byIntersection.setPolygons(polygons);
- spatialFilter.setByIntersection(byIntersection);
- queryRequest.setSpatialFilter(spatialFilter);
-
- SearchResponse searchResponse = Mockito.mock(SearchResponse.class);
-
- when(searchResponse.status()).thenReturn(RestStatus.OK);
-
- SearchHits searchHits = mock(SearchHits.class);
- when(searchHits.getHits()).thenReturn(new SearchHit[]{});
- when(searchResponse.getHits()).thenReturn(searchHits);
-
- when(restHighLevelClient.search(any(SearchRequest.class),
- eq(RequestOptions.DEFAULT))).thenReturn(searchResponse);
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- String index = "some-index";
- when(crossTenantUtils.getIndexName(any())).thenReturn(index);
-
- Set<String> indexedTypes = new HashSet<>();
- indexedTypes.add("geo_shape");
- when(fieldMappingTypeService.getFieldTypes(eq(restHighLevelClient), anyString(),
- eq(index))).thenReturn(indexedTypes);
-
- when(providerHeaderService.getDataGroupsHeader()).thenReturn("groups");
-
- Map<String, String> headers = new HashMap<>();
- headers.put("groups", "[]");
- when(dpsHeaders.getHeaders()).thenReturn(headers);
-
- String expectedSource = toString(getContent("1.json"));
-
- QueryResponse response = sut.queryIndex(queryRequest);
-
- ArgumentCaptor<SearchRequest> searchRequestArg = ArgumentCaptor.forClass(SearchRequest.class);
- verify(restHighLevelClient, times(1)).search(searchRequestArg.capture(), any());
- SearchRequest searchRequest = searchRequestArg.getValue();
- String actualSource = searchRequest.source().toString();
- assertEquals(expectedSource, actualSource);
- }
-
- /* arrange
- create query request according to this example query:
- {
- "kind": "osdu:wks:reference-data--CoordinateTransformation:1.0.1",
- "query": "data.ID:\"EPSG::blahblah8\"",
- "spatialFilter": {
- "field": "data.SpatialLocation.Wgs84Coordinates",
- "byWithinPolygon": {
- "points": [
- {
- "latitude": 10.71,
- "longitude": -8.60
- }
- ]
- }
-
- }
- }*/
- @Test
- public void should_return_CorrectQueryResponseForWithinSpatialFilter() throws Exception {
- QueryRequest queryRequest = new QueryRequest();
- queryRequest.setQuery("data.ID:\"EPSG::1078\"");
-
- SpatialFilter spatialFilter = new SpatialFilter();
- spatialFilter.setField("data.Wgs84Coordinates");
-
- SpatialFilter.ByWithinPolygon byWithinPolygon = new SpatialFilter.ByWithinPolygon();
- Point point = new Point(1.02, -8.61);
- List<Point> points = new ArrayList<>();
- points.add(point);
- byWithinPolygon.setPoints(points);
- spatialFilter.setByWithinPolygon(byWithinPolygon);
- queryRequest.setSpatialFilter(spatialFilter);
-
- SearchResponse searchResponse = mock(SearchResponse.class);
- when(searchResponse.status()).thenReturn(RestStatus.OK);
-
- SearchHits searchHits = mock(SearchHits.class);
- when(searchHits.getHits()).thenReturn(new SearchHit[]{});
- when(searchResponse.getHits()).thenReturn(searchHits);
-
- when(restHighLevelClient.search(any(SearchRequest.class),
- eq(RequestOptions.DEFAULT))).thenReturn(searchResponse);
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- when(restHighLevelClient.search(any(SearchRequest.class),
- eq(RequestOptions.DEFAULT))).thenReturn(searchResponse);
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- String index = "some-index";
- when(crossTenantUtils.getIndexName(any())).thenReturn(index);
-
- Set<String> indexedTypes = new HashSet<>();
- indexedTypes.add("geo_shape");
-
- when(fieldMappingTypeService.getFieldTypes(eq(restHighLevelClient), anyString(),
- eq(index))).thenReturn(indexedTypes);
- when(providerHeaderService.getDataGroupsHeader()).thenReturn("groups");
-
- Map<String, String> headers = new HashMap<>();
- headers.put("groups", "[]");
- when(dpsHeaders.getHeaders()).thenReturn(headers);
-
- String expectedSource = toString(getContent("2.json"));
-
- QueryResponse response = sut.queryIndex(queryRequest);
-
- ArgumentCaptor<SearchRequest> searchRequestArg = ArgumentCaptor.forClass(SearchRequest.class);
- Mockito.verify(restHighLevelClient, times(1)).search(searchRequestArg.capture(), any());
- SearchRequest searchRequest = searchRequestArg.getValue();
- String actualSource = searchRequest.source().toString();
- assertEquals(expectedSource, actualSource);
- }
-
- private InputStream getContent(String fileName) {
- return this.getClass().getResourceAsStream("/elasticresponses/spatialfilters/" + fileName);
- }
-
- private String toString(InputStream inputStream) {
- return new BufferedReader(new InputStreamReader(inputStream)).lines()
- .collect(Collectors.joining("")).replace(" ", "");
- }
-}
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.opengroup.osdu.search.provider.gcp.provider.impl;
+
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.search.TotalHits.Relation;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchResponseSections;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder;
+import org.elasticsearch.index.query.GeoDistanceQueryBuilder;
+import org.elasticsearch.index.query.GeoPolygonQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryStringQueryBuilder;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.Point;
+import org.opengroup.osdu.core.common.model.search.Polygon;
+import org.opengroup.osdu.core.common.model.search.QueryRequest;
+import org.opengroup.osdu.core.common.model.search.QueryResponse;
+import org.opengroup.osdu.core.common.model.search.SortOrder;
+import org.opengroup.osdu.core.common.model.search.SortQuery;
+import org.opengroup.osdu.core.common.model.search.SpatialFilter;
+import org.opengroup.osdu.core.common.model.search.SpatialFilter.ByBoundingBox;
+import org.opengroup.osdu.core.common.model.search.SpatialFilter.ByDistance;
+import org.opengroup.osdu.core.common.model.search.SpatialFilter.ByGeoPolygon;
+import org.opengroup.osdu.search.config.SearchConfigurationProperties;
+import org.opengroup.osdu.search.logging.AuditLogger;
+import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
+import org.opengroup.osdu.search.service.IFieldMappingTypeService;
+import org.opengroup.osdu.search.util.AggregationParserUtil;
+import org.opengroup.osdu.search.util.CrossTenantUtils;
+import org.opengroup.osdu.search.util.DetailedBadRequestMessageUtil;
+import org.opengroup.osdu.search.util.ElasticClientHandler;
+import org.opengroup.osdu.search.util.IAggregationParserUtil;
+import org.opengroup.osdu.search.util.IDetailedBadRequestMessageUtil;
+import org.opengroup.osdu.search.util.IQueryParserUtil;
+import org.opengroup.osdu.search.util.ISortParserUtil;
+import org.opengroup.osdu.search.util.QueryParserUtil;
+import org.opengroup.osdu.search.util.SortParserUtil;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({SearchRequest.class, SearchHits.class, RestHighLevelClient.class})
+public class QueryServiceTest {
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ @Mock
+ private SpatialFilter spatialFilter;
+ @Mock
+ private ByBoundingBox byBoundingBox;
+ @Mock
+ private ByDistance byDistance;
+ @Mock
+ private ByGeoPolygon byGeoPolygon;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private QueryRequest searchRequest;
+ @Mock
+ private DpsHeaders dpsHeaders;
+ @Mock
+ private CrossTenantUtils crossTenantUtils;
+ @Mock
+ private IFieldMappingTypeService fieldMappingTypeService;
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IProviderHeaderService providerHeaderService;
+ @Spy
+ private SearchConfigurationProperties properties = new SearchConfigurationProperties();
+ @Spy
+ private IQueryParserUtil parserService = new QueryParserUtil();
+ @Spy
+ private ISortParserUtil sortParserUtil = new SortParserUtil();
+ @Spy
+ private IAggregationParserUtil aggregationParserUtil = new AggregationParserUtil(properties);
+ @Spy
+ private IDetailedBadRequestMessageUtil detailedBadRequestMessageUtil = new DetailedBadRequestMessageUtil(
+ objectMapper);
+
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+
+ private SearchRequest elasticSearchRequest;
+
+ @InjectMocks
+ @Spy
+ private QueryServiceImpl sut = new QueryServiceImpl();
+
+ private final String DATA_GROUPS = "X-Data-Groups";
+ private final String DATA_GROUP_1 = "data.welldb.viewers@common.evd.cloud.slb-ds.com";
+
+ private final String DATA_GROUP_2 = "data.npd.viewers@common.evd.cloud.slb-ds.com";
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(RestHighLevelClient.class);
+ mockStatic(SearchRequest.class);
+ mockStatic(SearchHits.class);
+
+ when(properties.getEnvironment()).thenReturn("evt");
+ restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+
+ Map<String, String> HEADERS = new HashMap<>();
+ HEADERS.put(DpsHeaders.ACCOUNT_ID, "tenant1");
+ HEADERS.put(DpsHeaders.AUTHORIZATION, "Bearer blah");
+ HEADERS.put(DATA_GROUPS, String.format("%s,%s", DATA_GROUP_1, DATA_GROUP_2));
+
+ when(providerHeaderService.getDataGroupsHeader()).thenReturn(DATA_GROUPS);
+ when(dpsHeaders.getHeaders()).thenReturn(HEADERS);
+ }
+
+ @Test
+ public void should_returnSimpleQuery_when_request_queryIsSpecified() throws Exception {
+
+ Map<String, Object> hit = new HashMap<>();
+ hit.put("_id", "tenant1:welldb:wellbore-33fe05e1-df20-49d9-bd63-74cf750a206f");
+ hit.put("type", "wellbore");
+
+ List<Map<String, Object>> results = new ArrayList<>();
+ results.add(hit);
+
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ TotalHits totalHits = new TotalHits(1, Relation.EQUAL_TO);
+ SearchHits searchHits = new SearchHits(new SearchHit[0], totalHits, 2);
+ SearchResponse mockSearchResponse = new SearchResponse(
+ new SearchResponseSections(searchHits, null,
+ null, false, false, null, 1), "2",
+ 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
+ SearchResponse.Clusters.EMPTY);
+
+ doReturn(mockSearchResponse).when(this.sut).makeSearchRequest(any(), any());
+ doReturn(results).when(this.sut).getHitsFromSearchResponse(any());
+
+ QueryResponse queryResponse = this.sut.queryIndex(searchRequest);
+ assertNotNull(queryResponse);
+ assertEquals(1, queryResponse.getTotalCount());
+ }
+
+ @Test
+ public void should_returnRightTotalCount_when_queryResponseResultsIsNull() throws Exception {
+ List<Map<String, Object>> results = null;
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ TotalHits totalHits = new TotalHits(1, Relation.EQUAL_TO);
+ SearchHits searchHits = new SearchHits(new SearchHit[0], totalHits, 2);
+ SearchResponse mockSearchResponse = new SearchResponse(
+ new SearchResponseSections(searchHits, null,
+ null, false, false, null, 1), "2",
+ 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
+ SearchResponse.Clusters.EMPTY);
+
+ doReturn(mockSearchResponse).when(this.sut).makeSearchRequest(any(), any());
+ doReturn(results).when(this.sut).getHitsFromSearchResponse(any());
+
+ QueryResponse queryResponse = this.sut.queryIndex(searchRequest);
+ assertNotNull(queryResponse);
+ assertEquals(1, queryResponse.getTotalCount());
+ }
+
+ @Test
+ public void should_throwElasticException_when_indexNotFound() throws Exception {
+
+ ElasticsearchStatusException notFound = new ElasticsearchStatusException("blah",
+ RestStatus.NOT_FOUND);
+
+ doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
+ PowerMockito.when(restHighLevelClient.search(any(), any(RequestOptions.class)))
+ .thenThrow(notFound);
+
+ try {
+ this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
+ fail("Should not succeed!");
+ } catch (AppException e) {
+ assertEquals("Resource you are trying to find does not exists", e.getError().getMessage());
+ }
+ }
+
+ @Test
+ public void should_throwElasticException_given_badRequest() throws Exception {
+
+ ElasticsearchStatusException badRequest = new ElasticsearchStatusException("blah",
+ RestStatus.BAD_REQUEST);
+
+ doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
+ when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(badRequest);
+
+ try {
+ this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
+ fail("Should not succeed!");
+ } catch (AppException e) {
+ assertEquals("Invalid parameters were given on search request", e.getError().getMessage());
+ }
+ }
+
+ @Test
+ public void should_throwElasticException_given_searchError() throws Exception {
+
+ ElasticsearchStatusException generic = new ElasticsearchStatusException("blah",
+ RestStatus.BAD_GATEWAY);
+
+ doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
+ when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(generic);
+
+ try {
+ this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
+ fail("Should not succeed!");
+ } catch (AppException e) {
+ assertEquals("Error processing search request", e.getError().getMessage());
+ }
+ }
+
+ @Test
+ public void should_throwSystemException_given_genericError() throws Exception {
+
+ IllegalArgumentException exception = new IllegalArgumentException("search on fire");
+
+ doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
+ when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(exception);
+
+ try {
+ this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
+ fail("Should not succeed!");
+ } catch (AppException e) {
+ assertEquals("Error processing search request", e.getError().getMessage());
+ }
+ }
+
+ @Test
+ public void should_throwTimeoutException_given_timeoutError() throws Exception {
+
+ IOException exception = new IOException("listener timeout after waiting for [60000] ms");
+
+ doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
+ when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(exception);
+
+ try {
+ this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
+ fail("Should not succeed!");
+ } catch (AppException e) {
+ assertEquals(HttpServletResponse.SC_GATEWAY_TIMEOUT, e.getError().getCode());
+ assertEquals("Request timed out after waiting for 1m", e.getError().getMessage());
+ }
+ }
+
+ @Test
+ public void should_throwSystemException_given_genericIOError() throws Exception {
+
+ IOException exception = new IOException("search on fire");
+
+ doReturn(elasticSearchRequest).when(this.sut).createElasticRequest(any());
+ when(restHighLevelClient.search(any(), any(RequestOptions.class))).thenThrow(exception);
+
+ try {
+ this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
+ fail("Should not succeed!");
+ } catch (AppException e) {
+ assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ assertEquals("Error processing search request", e.getError().getMessage());
+ }
+ }
+
+ @Test
+ public void should_return_textQuery_when_requestHasIt() throws IOException {
+
+ String simpleQuery = "find-me";
+ BoolQueryBuilder builder = (BoolQueryBuilder) this.sut.buildQuery(simpleQuery, null, true);
+ assertNotNull(builder);
+
+ List<QueryBuilder> topLevelMustClause = builder.must();
+ assertEquals(2, topLevelMustClause.size());
+
+ BoolQueryBuilder queryLevelBuilder = (BoolQueryBuilder) topLevelMustClause.get(0);
+ assertNotNull(queryLevelBuilder);
+
+ List<QueryBuilder> queryLevelMustClause = queryLevelBuilder.must();
+ assertEquals(1, queryLevelMustClause.size());
+
+ BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryLevelMustClause.get(0);
+ List<QueryBuilder> must = boolQueryBuilder.must();
+ QueryStringQueryBuilder queryBuilder = (QueryStringQueryBuilder) must.get(0);
+
+ assertNotNull(boolQueryBuilder);
+ assertEquals(simpleQuery, queryBuilder.queryString());
+
+ verifyAcls(topLevelMustClause.get(1), true);
+ }
+
+ @Test
+ public void should_searchAll_when_requestHas_noQueryString() throws IOException {
+
+ BoolQueryBuilder builder = (BoolQueryBuilder) this.sut.buildQuery(null, null, true);
+ assertNotNull(builder);
+
+ List<QueryBuilder> topLevelMustClause = builder.must();
+ assertEquals(1, topLevelMustClause.size());
+
+ verifyAcls(topLevelMustClause.get(0), true);
+ }
+
+ @Test
+ public void should_return_ownerOnlyMustClause_when_searchAsOwners() throws IOException {
+
+ BoolQueryBuilder builder = (BoolQueryBuilder) this.sut.buildQuery(null, null, false);
+ assertNotNull(builder);
+
+ List<QueryBuilder> topLevelMustClause = builder.must();
+ assertEquals(1, topLevelMustClause.size());
+
+ verifyAcls(topLevelMustClause.get(0), false);
+ }
+
+ @Test
+ public void should_return_nullQuery_when_searchAsDataRootUser() throws IOException {
+ Map<String, String> HEADERS = new HashMap<>();
+ HEADERS.put(DpsHeaders.ACCOUNT_ID, "tenant1");
+ HEADERS.put(DpsHeaders.AUTHORIZATION, "Bearer blah");
+ HEADERS.put(DATA_GROUPS, String.format("%s,%s", DATA_GROUP_1, DATA_GROUP_2));
+ HEADERS.put(providerHeaderService.getDataRootUserHeader(), "true");
+ when(dpsHeaders.getHeaders()).thenReturn(HEADERS);
+
+ QueryBuilder builder = this.sut.buildQuery(null, null, false);
+ assertNull(builder);
+ }
+
+ @Test(expected = AppException.class)
+ public void testQueryBase_whenUnsupportedSortRequested_statusBadRequest_throwsException()
+ throws IOException {
+ String fieldName = "field";
+ String indexName = "index";
+ String dummySortError = "Text fields are not optimised for operations that require per-document field data like aggregations and sorting, so these operations are disabled by default. Please use a keyword field instead";
+ ElasticsearchStatusException exception = new ElasticsearchStatusException("blah",
+ RestStatus.BAD_REQUEST, new ElasticsearchException(dummySortError));
+
+ doThrow(exception).when(restHighLevelClient).search(any(), any(RequestOptions.class));
+ doReturn(new HashSet<>()).when(fieldMappingTypeService)
+ .getFieldTypes(eq(restHighLevelClient), eq(fieldName), eq(indexName));
+ SortQuery sortQuery = new SortQuery();
+ sortQuery.setField(Collections.singletonList("name"));
+ sortQuery.setOrder(Collections.singletonList(SortOrder.DESC));
+ when(searchRequest.getSort()).thenReturn(sortQuery);
+ when(sortParserUtil.getSortQuery(restHighLevelClient, sortQuery, indexName))
+ .thenReturn(Collections.singletonList(
+ new FieldSortBuilder("name").order(org.elasticsearch.search.sort.SortOrder.DESC)));
+
+ try {
+ this.sut.makeSearchRequest(searchRequest, restHighLevelClient);
+ } catch (AppException e) {
+ int errorCode = 400;
+ String errorMessage = "Sort is not supported for one or more of the requested fields";
+ assertEquals(e.getError().getCode(), errorCode);
+ assertEquals(e.getError().getMessage(), errorMessage);
+
+ throw (e);
+ }
+ }
+
+ @Test
+ public void should_return_boundingBoxQuery_given_spatialCriteria() throws IOException {
+
+ String simpleQuery = "dare-find-me";
+
+ String field = "LonLat";
+ Double bottomRightLon = 124.174762;
+ Double bottomRightLat = 36.450727;
+ Double topLeftLon = 37.450727;
+ Double topLeftLat = 70.174762;
+
+ when(this.spatialFilter.getField()).thenReturn(field);
+ when(this.spatialFilter.getByBoundingBox()).thenReturn(byBoundingBox);
+ when(this.spatialFilter.getByBoundingBox().getBottomRight()).thenReturn(mock(Point.class));
+ when(this.spatialFilter.getByBoundingBox().getBottomRight().getLongitude())
+ .thenReturn(bottomRightLon);
+ when(this.spatialFilter.getByBoundingBox().getBottomRight().getLatitude())
+ .thenReturn(bottomRightLat);
+ when(this.spatialFilter.getByBoundingBox().getTopLeft()).thenReturn(mock(Point.class));
+ when(this.spatialFilter.getByBoundingBox().getTopLeft().getLongitude()).thenReturn(37.450727);
+ when(this.spatialFilter.getByBoundingBox().getTopLeft().getLatitude()).thenReturn(70.174762);
+
+ BoolQueryBuilder builder = (BoolQueryBuilder) this.sut
+ .buildQuery(simpleQuery, this.spatialFilter, true);
+ assertNotNull(builder);
+
+ List<QueryBuilder> topLevelMustClause = builder.must();
+ assertEquals(2, topLevelMustClause.size());
+
+ BoolQueryBuilder queryLevelBuilder = (BoolQueryBuilder) topLevelMustClause.get(0);
+ assertNotNull(queryLevelBuilder);
+
+ List<QueryBuilder> queryLevelMustClause = queryLevelBuilder.must();
+ assertEquals(2, queryLevelMustClause.size());
+
+ BoolQueryBuilder queryStringBoolQueryBuilder = (BoolQueryBuilder) queryLevelMustClause.get(0);
+ assertNotNull(queryStringBoolQueryBuilder);
+
+ GeoBoundingBoxQueryBuilder geoBoundingBoxQueryBuilder = (GeoBoundingBoxQueryBuilder) queryLevelMustClause
+ .get(1);
+ assertNotNull(geoBoundingBoxQueryBuilder);
+ assertEquals(field, geoBoundingBoxQueryBuilder.fieldName());
+ assertEquals(topLeftLon, geoBoundingBoxQueryBuilder.topLeft().getLon(), .001);
+ assertEquals(topLeftLat, geoBoundingBoxQueryBuilder.topLeft().getLat(), .001);
+ assertEquals(bottomRightLon, geoBoundingBoxQueryBuilder.bottomRight().getLon(), .001);
+ assertEquals(bottomRightLat, geoBoundingBoxQueryBuilder.bottomRight().getLat(), .001);
+
+ verifyAcls(topLevelMustClause.get(1), true);
+ }
+
+ @Test
+ public void should_return_distanceQuery_given_spatialCriteria() throws IOException {
+
+ String simpleQuery = "oh no you found me";
+
+ String field = "LonLat";
+ Double pointLon = 124.174762;
+ Double pointLat = 36.450727;
+ Double distance = 10.0;
+
+ when(this.spatialFilter.getField()).thenReturn(field);
+ when(this.spatialFilter.getByDistance()).thenReturn(byDistance);
+ when(this.spatialFilter.getByDistance().getPoint()).thenReturn(mock(Point.class));
+ when(this.spatialFilter.getByDistance().getPoint().getLongitude()).thenReturn(pointLon);
+ when(this.spatialFilter.getByDistance().getPoint().getLatitude()).thenReturn(pointLat);
+ when(this.spatialFilter.getByDistance().getDistance()).thenReturn(distance);
+
+ BoolQueryBuilder builder = (BoolQueryBuilder) this.sut
+ .buildQuery(simpleQuery, this.spatialFilter, true);
+ assertNotNull(builder);
+
+ List<QueryBuilder> topLevelMustClause = builder.must();
+ assertEquals(2, topLevelMustClause.size());
+
+ BoolQueryBuilder queryLevelBuilder = (BoolQueryBuilder) topLevelMustClause.get(0);
+ assertNotNull(queryLevelBuilder);
+
+ List<QueryBuilder> queryLevelMustClause = queryLevelBuilder.must();
+ assertEquals(2, queryLevelMustClause.size());
+
+ BoolQueryBuilder queryStringBoolQueryBuilder = (BoolQueryBuilder) queryLevelMustClause.get(0);
+ assertNotNull(queryStringBoolQueryBuilder);
+
+ GeoDistanceQueryBuilder geoDistanceQueryBuilder = (GeoDistanceQueryBuilder) queryLevelMustClause
+ .get(1);
+ assertNotNull(geoDistanceQueryBuilder);
+ assertEquals(field, geoDistanceQueryBuilder.fieldName());
+ assertEquals(distance, geoDistanceQueryBuilder.distance(), .001);
+ assertEquals(pointLon, geoDistanceQueryBuilder.point().getLon(), .001);
+ assertEquals(pointLat, geoDistanceQueryBuilder.point().getLat(), .001);
+
+ verifyAcls(topLevelMustClause.get(1), true);
+ }
+
+ @Test
+ public void should_return_polygonBoxQuery_given_spatialCriteria() throws IOException {
+
+ String simpleQuery = "polygons are fun";
+
+ String field = "LonLat";
+ List<Point> points = new ArrayList<>();
+ points.add(new Point(124.174762, 36.450727));
+ points.add(new Point(126.174762, 36.450727));
+ points.add(new Point(128.174762, 46.450727));
+ points.add(new Point(124.174762, 36.450727));
+
+ when(this.spatialFilter.getField()).thenReturn(field);
+ when(this.spatialFilter.getByGeoPolygon()).thenReturn(byGeoPolygon);
+ when(this.spatialFilter.getByGeoPolygon().getPoints()).thenReturn(points);
+
+ BoolQueryBuilder builder = (BoolQueryBuilder) this.sut
+ .buildQuery(simpleQuery, this.spatialFilter, true);
+ assertNotNull(builder);
+
+ List<QueryBuilder> topLevelMustClause = builder.must();
+ assertEquals(2, topLevelMustClause.size());
+
+ BoolQueryBuilder queryLevelBuilder = (BoolQueryBuilder) topLevelMustClause.get(0);
+ assertNotNull(queryLevelBuilder);
+
+ List<QueryBuilder> queryLevelMustClause = queryLevelBuilder.must();
+ assertEquals(2, queryLevelMustClause.size());
+
+ BoolQueryBuilder queryStringBoolQueryBuilder = (BoolQueryBuilder) queryLevelMustClause.get(0);
+ assertNotNull(queryStringBoolQueryBuilder);
+
+ GeoPolygonQueryBuilder geoPolygonQueryBuilder = (GeoPolygonQueryBuilder) queryLevelMustClause
+ .get(1);
+ assertNotNull(geoPolygonQueryBuilder);
+ assertEquals(field, geoPolygonQueryBuilder.fieldName());
+ assertEquals(points.size(), geoPolygonQueryBuilder.points().size());
+ List<GeoPoint> points1 = geoPolygonQueryBuilder.points();
+ for (int i = 0; i < points1.size(); i++) {
+ GeoPoint geoPoint = points1.get(i);
+ Point requestPoint = points.get(i);
+ assertEquals(requestPoint.getLongitude(), geoPoint.getLon(), .001);
+ assertEquals(requestPoint.getLatitude(), geoPoint.getLat(), .001);
+ }
+
+ verifyAcls(topLevelMustClause.get(1), true);
+ }
+
+ @Test
+ public void should_return_correctElasticRequest_given_requestQuery() throws IOException {
+ int limit = 5;
+ int from = 2;
+ String kind = "tenant1:welldb:well:1.0.0";
+
+ List<String> returnedFields = new ArrayList<>();
+ returnedFields.add("id");
+ when(searchRequest.getKind()).thenReturn(kind);
+ when(searchRequest.getLimit()).thenReturn(limit);
+ when(searchRequest.getFrom()).thenReturn(from);
+ when(searchRequest.getReturnedFields()).thenReturn(returnedFields);
+
+ when(crossTenantUtils.getIndexName(any())).thenReturn("tenant1-welldb-well-1.0.0,-.*");
+
+ SearchRequest elasticRequest = this.sut.createElasticRequest(searchRequest);
+ assertNotNull(elasticRequest);
+
+ String[] indices = elasticRequest.indices();
+ assertEquals(1, indices.length);
+ assertEquals("tenant1-welldb-well-1.0.0,-.*", indices[0]);
+
+ SearchSourceBuilder elasticSearchSourceBuilder = elasticRequest.source();
+ assertNotNull(elasticSearchSourceBuilder);
+ assertEquals(limit, elasticSearchSourceBuilder.size());
+ assertEquals(from, elasticSearchSourceBuilder.from());
+ assertEquals(1, elasticSearchSourceBuilder.timeout().getMinutes());
+
+ FetchSourceContext elasticFetchSourceContext = elasticSearchSourceBuilder.fetchSource();
+ assertNotNull(elasticFetchSourceContext);
+
+ String[] elasticExcludes = elasticFetchSourceContext.excludes();
+ assertEquals(2, elasticExcludes.length);
+ assertEquals("x-acl", elasticExcludes[0]);
+ assertEquals("index", elasticExcludes[1]);
+
+ String[] elasticIncludes = elasticFetchSourceContext.includes();
+ assertEquals(1, elasticIncludes.length);
+ assertEquals("id", elasticIncludes[0]);
+
+ QueryBuilder elasticQueryBuilder = elasticSearchSourceBuilder.query();
+ assertNotNull(elasticQueryBuilder);
+ }
+
+ @Test
+ public void should_return_correctElasticRequest_given_returnedFieldContainsQueryableExcludes()
+ throws IOException {
+
+ List<String> returnedFields = new ArrayList<>(Arrays.asList("id", "index"));
+ when(searchRequest.getKind()).thenReturn("tenant1:welldb:well:1.0.0");
+ when(searchRequest.getReturnedFields()).thenReturn(returnedFields);
+
+ when(crossTenantUtils.getIndexName(any())).thenReturn("tenant1-welldb-well-1.0.0,-.*");
+
+ SearchRequest elasticRequest = this.sut.createElasticRequest(searchRequest);
+ assertNotNull(elasticRequest);
+
+ String[] indices = elasticRequest.indices();
+ assertEquals(1, indices.length);
+ assertEquals("tenant1-welldb-well-1.0.0,-.*", indices[0]);
+
+ SearchSourceBuilder elasticSearchSourceBuilder = elasticRequest.source();
+ assertNotNull(elasticSearchSourceBuilder);
+
+ FetchSourceContext elasticFetchSourceContext = elasticSearchSourceBuilder.fetchSource();
+ assertNotNull(elasticFetchSourceContext);
+
+ String[] elasticExcludes = elasticFetchSourceContext.excludes();
+ assertEquals(1, elasticExcludes.length);
+ assertEquals("x-acl", elasticExcludes[0]);
+
+ List<String> elasticIncludes = Arrays.asList(elasticFetchSourceContext.includes());
+ assertEquals(2, elasticIncludes.size());
+ assertTrue(elasticIncludes.contains("index"));
+ assertTrue(elasticIncludes.contains("id"));
+ }
+
+ @Test
+ public void should_return_correctElasticRequest_given_noReturnedField() throws IOException {
+
+ when(searchRequest.getKind()).thenReturn("tenant1:welldb:well:1.0.0");
+ when(crossTenantUtils.getIndexName(any())).thenReturn("tenant1-welldb-well-1.0.0,-.*");
+
+ SearchRequest elasticRequest = this.sut.createElasticRequest(searchRequest);
+ assertNotNull(elasticRequest);
+
+ String[] indices = elasticRequest.indices();
+ assertEquals(1, indices.length);
+ assertEquals("tenant1-welldb-well-1.0.0,-.*", indices[0]);
+
+ SearchSourceBuilder elasticSearchSourceBuilder = elasticRequest.source();
+ assertNotNull(elasticSearchSourceBuilder);
+
+ FetchSourceContext elasticFetchSourceContext = elasticSearchSourceBuilder.fetchSource();
+ assertNotNull(elasticFetchSourceContext);
+
+ List<String> elasticExcludes = Arrays.asList(elasticFetchSourceContext.excludes());
+ assertEquals(2, elasticExcludes.size());
+ assertTrue(elasticExcludes.contains("index"));
+ assertTrue(elasticExcludes.contains("x-acl"));
+
+ List<String> elasticIncludes = Arrays.asList(elasticFetchSourceContext.includes());
+ assertEquals(0, elasticIncludes.size());
+ }
+
+ @Test
+ public void should_return_correctElasticRequest_given_groupByField() throws IOException {
+ when(searchRequest.getKind()).thenReturn("tenant1:welldb:well:1.0.0");
+ when(searchRequest.getAggregateBy()).thenReturn("namespace");
+ when(crossTenantUtils.getIndexName(any())).thenReturn("tenant1-welldb-well-1.0.0,-.*");
+
+ SearchRequest elasticRequest = this.sut.createElasticRequest(searchRequest);
+ assertNotNull(elasticRequest);
+ assertNotNull(elasticRequest.source().aggregations());
+ assertEquals(1, elasticRequest.source().aggregations().count());
+ }
+
+ private void verifyAcls(QueryBuilder aclMustClause, boolean asOwner) {
+ BoolQueryBuilder aclLevelBuilder = (BoolQueryBuilder) aclMustClause;
+ assertNotNull(aclLevelBuilder);
+ assertEquals("1", aclLevelBuilder.minimumShouldMatch());
+
+ List<QueryBuilder> aclShouldClause = aclLevelBuilder.should();
+ assertEquals(1, aclShouldClause.size());
+
+ TermsQueryBuilder aclQuery = (TermsQueryBuilder) aclShouldClause.get(0);
+ assertNotNull(aclQuery);
+ if (asOwner) {
+ assertEquals("acl.owners", aclQuery.fieldName());
+ } else {
+ assertEquals("x-acl", aclQuery.fieldName());
+ }
+ assertEquals(2, aclQuery.values().size());
+
+ List<Object> acls = aclQuery.values();
+ assertEquals(2, acls.size());
+ assertTrue(acls.contains(DATA_GROUP_1));
+ assertTrue(acls.contains(DATA_GROUP_2));
+ }
+
+ /* arrange
+ create query request according to this example query:
+ {
+ "kind": "osdu:wks:reference-data--CoordinateTransformation:1.0.0",
+ "query": "data.ID:\"EPSG::1078\"",
+ "spatialFilter": {
+ "field": "data.Wgs84Coordinates",
+ "byIntersection": {
+ "polygons": [
+ {
+ "points": [
+ {
+ "latitude": 10.75,
+ "longitude": -8.61
+ }
+ ]
+ }
+ ]
+ }
+ }
+ }*/
+ @Test
+ public void should_return_CorrectQueryResponseForIntersectionSpatialFilter() throws Exception {
+ QueryRequest queryRequest = new QueryRequest();
+ queryRequest.setQuery("data.ID:\"EPSG::1078\"");
+
+ SpatialFilter spatialFilter = new SpatialFilter();
+ spatialFilter.setField("data.Wgs84Coordinates");
+
+ SpatialFilter.ByIntersection byIntersection = new SpatialFilter.ByIntersection();
+ Polygon polygon = new Polygon();
+ Point point = new Point(1.02, -8.61);
+ Point point1 = new Point(1.02, -2.48);
+ Point point2 = new Point(10.74, -2.48);
+ Point point3 = new Point(10.74, -8.61);
+ Point point4 = new Point(1.02, -8.61);
+
+ List<Point> points = new ArrayList<>();
+ points.add(point);
+ points.add(point1);
+ points.add(point2);
+ points.add(point3);
+ points.add(point4);
+ polygon.setPoints(points);
+
+ List<Polygon> polygons = new ArrayList<>();
+ polygons.add(polygon);
+ byIntersection.setPolygons(polygons);
+ spatialFilter.setByIntersection(byIntersection);
+ queryRequest.setSpatialFilter(spatialFilter);
+
+ SearchResponse searchResponse = Mockito.mock(SearchResponse.class);
+
+ when(searchResponse.status()).thenReturn(RestStatus.OK);
+
+ SearchHits searchHits = mock(SearchHits.class);
+ when(searchHits.getHits()).thenReturn(new SearchHit[]{});
+ when(searchResponse.getHits()).thenReturn(searchHits);
+
+ when(restHighLevelClient.search(any(SearchRequest.class),
+ eq(RequestOptions.DEFAULT))).thenReturn(searchResponse);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ String index = "some-index";
+ when(crossTenantUtils.getIndexName(any())).thenReturn(index);
+
+ Set<String> indexedTypes = new HashSet<>();
+ indexedTypes.add("geo_shape");
+ when(fieldMappingTypeService.getFieldTypes(eq(restHighLevelClient), anyString(),
+ eq(index))).thenReturn(indexedTypes);
+
+ when(providerHeaderService.getDataGroupsHeader()).thenReturn("groups");
+
+ Map<String, String> headers = new HashMap<>();
+ headers.put("groups", "[]");
+ when(dpsHeaders.getHeaders()).thenReturn(headers);
+
+ String expectedSource = toString(getContent("1.json"));
+
+ QueryResponse response = sut.queryIndex(queryRequest);
+
+ ArgumentCaptor<SearchRequest> searchRequestArg = ArgumentCaptor.forClass(SearchRequest.class);
+ verify(restHighLevelClient, times(1)).search(searchRequestArg.capture(), any());
+ SearchRequest searchRequest = searchRequestArg.getValue();
+ String actualSource = searchRequest.source().toString();
+ assertEquals(expectedSource, actualSource);
+ }
+
+ /* arrange
+ create query request according to this example query:
+ {
+ "kind": "osdu:wks:reference-data--CoordinateTransformation:1.0.1",
+ "query": "data.ID:\"EPSG::blahblah8\"",
+ "spatialFilter": {
+ "field": "data.SpatialLocation.Wgs84Coordinates",
+ "byWithinPolygon": {
+ "points": [
+ {
+ "latitude": 10.71,
+ "longitude": -8.60
+ }
+ ]
+ }
+
+ }
+ }*/
+ @Test
+ public void should_return_CorrectQueryResponseForWithinSpatialFilter() throws Exception {
+ QueryRequest queryRequest = new QueryRequest();
+ queryRequest.setQuery("data.ID:\"EPSG::1078\"");
+
+ SpatialFilter spatialFilter = new SpatialFilter();
+ spatialFilter.setField("data.Wgs84Coordinates");
+
+ SpatialFilter.ByWithinPolygon byWithinPolygon = new SpatialFilter.ByWithinPolygon();
+ Point point = new Point(1.02, -8.61);
+ List<Point> points = new ArrayList<>();
+ points.add(point);
+ byWithinPolygon.setPoints(points);
+ spatialFilter.setByWithinPolygon(byWithinPolygon);
+ queryRequest.setSpatialFilter(spatialFilter);
+
+ SearchResponse searchResponse = mock(SearchResponse.class);
+ when(searchResponse.status()).thenReturn(RestStatus.OK);
+
+ SearchHits searchHits = mock(SearchHits.class);
+ when(searchHits.getHits()).thenReturn(new SearchHit[]{});
+ when(searchResponse.getHits()).thenReturn(searchHits);
+
+ when(restHighLevelClient.search(any(SearchRequest.class),
+ eq(RequestOptions.DEFAULT))).thenReturn(searchResponse);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(restHighLevelClient.search(any(SearchRequest.class),
+ eq(RequestOptions.DEFAULT))).thenReturn(searchResponse);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ String index = "some-index";
+ when(crossTenantUtils.getIndexName(any())).thenReturn(index);
+
+ Set<String> indexedTypes = new HashSet<>();
+ indexedTypes.add("geo_shape");
+
+ when(fieldMappingTypeService.getFieldTypes(eq(restHighLevelClient), anyString(),
+ eq(index))).thenReturn(indexedTypes);
+ when(providerHeaderService.getDataGroupsHeader()).thenReturn("groups");
+
+ Map<String, String> headers = new HashMap<>();
+ headers.put("groups", "[]");
+ when(dpsHeaders.getHeaders()).thenReturn(headers);
+
+ String expectedSource = toString(getContent("2.json"));
+
+ QueryResponse response = sut.queryIndex(queryRequest);
+
+ ArgumentCaptor<SearchRequest> searchRequestArg = ArgumentCaptor.forClass(SearchRequest.class);
+ Mockito.verify(restHighLevelClient, times(1)).search(searchRequestArg.capture(), any());
+ SearchRequest searchRequest = searchRequestArg.getValue();
+ String actualSource = searchRequest.source().toString();
+ assertEquals(expectedSource, actualSource);
+ }
+
+ private InputStream getContent(String fileName) {
+ return this.getClass().getResourceAsStream("/elasticresponses/spatialfilters/" + fileName);
+ }
+
+ private String toString(InputStream inputStream) {
+ return new BufferedReader(new InputStreamReader(inputStream)).lines()
+ .collect(Collectors.joining("")).replace(" ", "");
+ }
+}
diff --git a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceTest.java b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceTest.java
similarity index 100%
rename from provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceTest.java
rename to provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceTest.java
diff --git a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AllFiltersTest.java b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AllFiltersTest.java
similarity index 100%
rename from provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AllFiltersTest.java
rename to provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AllFiltersTest.java
diff --git a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AttributeFilterTest.java b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AttributeFilterTest.java
similarity index 100%
rename from provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AttributeFilterTest.java
rename to provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/AttributeFilterTest.java
diff --git a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/SourceFilterTest.java b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/SourceFilterTest.java
similarity index 100%
rename from provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/SourceFilterTest.java
rename to provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/SourceFilterTest.java
diff --git a/provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/TypeFilterTest.java b/provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/TypeFilterTest.java
similarity index 100%
rename from provider/search-gcp/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/TypeFilterTest.java
rename to provider/search-gc/src/test/java/org/opengroup/osdu/search/provider/gcp/smart/filters/TypeFilterTest.java
diff --git a/provider/search-gcp/src/test/resources/elasticresponses/spatialfilters/1.json b/provider/search-gc/src/test/resources/elasticresponses/spatialfilters/1.json
similarity index 100%
rename from provider/search-gcp/src/test/resources/elasticresponses/spatialfilters/1.json
rename to provider/search-gc/src/test/resources/elasticresponses/spatialfilters/1.json
diff --git a/provider/search-gcp/src/test/resources/elasticresponses/spatialfilters/2.json b/provider/search-gc/src/test/resources/elasticresponses/spatialfilters/2.json
similarity index 100%
rename from provider/search-gcp/src/test/resources/elasticresponses/spatialfilters/2.json
rename to provider/search-gc/src/test/resources/elasticresponses/spatialfilters/2.json
diff --git a/provider/search-gcp/kubernetes/deployments/deployment-os-search-service.yml b/provider/search-gcp/kubernetes/deployments/deployment-os-search-service.yml
deleted file mode 100644
index c8dcb9ee1..000000000
--- a/provider/search-gcp/kubernetes/deployments/deployment-os-search-service.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: os-search-service
- labels:
- app: os-search-service
-spec:
- ports:
- - name: http
- port: 80
- targetPort: 8080
- selector:
- app: os-search-service
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
- name: os-search-service
-spec:
- replicas: 1
- template:
- metadata:
- labels:
- app: os-search-service
- version: v1
- spec:
- containers:
- - name: os-search-app
- envFrom:
- - configMapRef:
- name: os-service-config
- image: gcr.io/opendes/os-search-app
- imagePullPolicy: Always
- ports:
- - containerPort: 8080
diff --git a/provider/search-gcp/scripts/deploy.sh b/provider/search-gcp/scripts/deploy.sh
deleted file mode 100644
index e7d3ec29d..000000000
--- a/provider/search-gcp/scripts/deploy.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-# THIS SCRIPT MIGHT CHANGE CONSIDERABLY LATER
-
-#!/bin/bash
-
-# Exit as soon as a command fails
-set -e
-
-echo " Environment = $ENVIRONMENT"
-echo " Project = $GCLOUD_PROJECT"
-echo " Source branch = $BUILD_SOURCEBRANCHNAME"
-echo " Build definition = $BUILD_DEFINITIONNAME"
-echo " Build number = $BUILD_BUILDNUMBER"
-
-echo "current working directory"
-pwd
-
-echo "working directory contents"
-ls
-
-SERVICE_BASE_NAME="search"
-
-DEPLOY_DIR="deploy_dir"
-mkdir -p $DEPLOY_DIR
-
-cp app.yaml $DEPLOY_DIR
-cp $SERVICE_BASE_NAME-gcp-*.jar $DEPLOY_DIR
-
-# Go to deploy directory
-cd $DEPLOY_DIR
-
-# apply sed to replace ENVIRONMENT with its value in app.yaml
-sed -i -e "s|ENVIRONMENT|$ENVIRONMENT|g" app.yaml
-sed -i -e "s|AUTHORIZE_API_VAR|$AUTHORIZE_API|g" app.yaml
-sed -i -e "s|CRON_JOB_IP_VAR|$CRON_JOB_IP|g" app.yaml
-sed -i -e "s|LEGAL_HOSTNAME_VAR|$LEGAL_HOSTNAME|g" app.yaml
-sed -i -e "s|REGION_VAR|$REGION|g" app.yaml
-
-cat app.yaml
-
-# check the current deployed version of the same service
-SERVICE_NAME="os-$SERVICE_BASE_NAME"
-count=$(gcloud app services list --project $GCLOUD_PROJECT | grep $SERVICE_NAME | wc -l)
-if [ $count -gt 0 ]; then
- CURRENT_VERSION=$(gcloud app services describe $SERVICE_NAME --project $GCLOUD_PROJECT --format=json | jq --raw-output '.split.allocations | keys[0]')
-else
- CURRENT_VERSION=""
-fi
-
-NEW_VERSION=$BUILD_BUILDNUMBER #to keep it unique
-NEW_VERSION=$(echo "$NEW_VERSION" | tr _ - | tr . - | tr '[:upper:]' '[:lower:]')
-
-echo "Current version = $CURRENT_VERSION"
-echo "Version to be deployed = $NEW_VERSION"
-
-if [ "$NEW_VERSION" != "$CURRENT_VERSION" -o "$BUILD_FORCE_DEPLOY" = "true" ]
-then
- gcloud app deploy --quiet --version=$NEW_VERSION --project=$GCLOUD_PROJECT app.yaml
-else
- echo "Not deploying the application because $NEW_VERSION is already deployed and force deploy flag is not set to true"
-fi
-
diff --git a/testing/integration-tests/pom.xml b/testing/integration-tests/pom.xml
index fe3e5bedf..9453a659e 100644
--- a/testing/integration-tests/pom.xml
+++ b/testing/integration-tests/pom.xml
@@ -34,7 +34,7 @@
<module>search-test-core</module>
<module>search-test-aws</module>
<module>search-test-azure</module>
- <module>search-test-gcp</module>
+ <module>search-test-gc</module>
<module>search-test-ibm</module>
<module>search-test-anthos</module>
</modules>
diff --git a/testing/integration-tests/search-test-gcp/pom.xml b/testing/integration-tests/search-test-gc/pom.xml
similarity index 99%
rename from testing/integration-tests/search-test-gcp/pom.xml
rename to testing/integration-tests/search-test-gc/pom.xml
index 066a2e572..d2d6f8fdf 100644
--- a/testing/integration-tests/search-test-gcp/pom.xml
+++ b/testing/integration-tests/search-test-gc/pom.xml
@@ -10,7 +10,7 @@
</parent>
<groupId>org.opengroup.osdu.search</groupId>
- <artifactId>search-test-gcp</artifactId>
+ <artifactId>search-test-gc</artifactId>
<version>0.19.0-SNAPSHOT</version>
<properties>
diff --git a/testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java b/testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
similarity index 100%
rename from testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
rename to testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
diff --git a/testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
similarity index 100%
rename from testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
rename to testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
diff --git a/testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/RunTest.java b/testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/RunTest.java
similarity index 100%
rename from testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/RunTest.java
rename to testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/RunTest.java
diff --git a/testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/Steps.java b/testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/Steps.java
similarity index 100%
rename from testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/Steps.java
rename to testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/Steps.java
diff --git a/testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/RunTest.java b/testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/RunTest.java
similarity index 100%
rename from testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/RunTest.java
rename to testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/RunTest.java
diff --git a/testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/Steps.java b/testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/Steps.java
similarity index 100%
rename from testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/Steps.java
rename to testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/step_definitions/querybycursor/singlecluster/Steps.java
diff --git a/testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/util/GCPHTTPClient.java b/testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/util/GCPHTTPClient.java
similarity index 100%
rename from testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/util/GCPHTTPClient.java
rename to testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/util/GCPHTTPClient.java
diff --git a/testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java b/testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
similarity index 100%
rename from testing/integration-tests/search-test-gcp/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
rename to testing/integration-tests/search-test-gc/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
--
GitLab