Commit 92488303 authored by Alok Joshi's avatar Alok Joshi
Browse files

pullback

parents b33bfcb7 503a4372
Pipeline #100053 failed with stages
in 57 minutes and 46 seconds
This diff is collapsed.
......@@ -16,7 +16,7 @@
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.14.0-rc4</osdu.oscorecommon.version>
<spring-security-web.version>5.4.9</spring-security-web.version>
<gson.version>2.8.9</gson.version>
<netty.version>4.1.70.Final</netty.version>
......
......@@ -400,7 +400,7 @@ public class IndexerServiceImpl implements IndexerService {
IndexRequest indexRequest = new IndexRequest(index).id(record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
bulkRequest.add(indexRequest);
} else if (operation == OperationType.update) {
UpdateRequest updateRequest = new UpdateRequest(index, record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
UpdateRequest updateRequest = new UpdateRequest(index, "_doc", record.getId()).doc(this.gson.toJson(sourceMap), XContentType.JSON).docAsUpsert(true);
bulkRequest.add(updateRequest);
}
}
......
......@@ -13,7 +13,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
<os-core-common.version>0.13.0</os-core-common.version>
<os-core-common.version>0.14.0-rc4</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.4</jackson.version>
......
......@@ -14,6 +14,14 @@ os-indexer-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) servic
## Getting Started
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
# Configuration
## Service Configuration
### Anthos:
[Anthos service configuration ](docs/anthos/README.md)
### GCP:
[Gcp service configuration ](docs/gcp/README.md)
## Mappers
This is a universal solution created using EPAM OQM mappers technology. It allows you to work with various
......
......@@ -12,6 +12,8 @@ Must have:
| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` |yes| - |
| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens| yes | - |
| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - |
| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | yes | - |
| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | false | - |
Defined in default application property file but possible to override:
......@@ -39,10 +41,26 @@ Usage of spring profiles is preferred.
| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
## Elasticsearch configuration
### Properties set in Partition service:
Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name.
This variable should be present in environment of service that need that variable.
Example:
```
"elasticsearch.port": {
"sensitive": false, <- value not sensitive
"value": "9243" <- will be used as is.
},
"elasticsearch.password": {
"sensitive": true, <- value is sensitive
"value": "ELASTIC_SEARCH_PASSWORD_OSDU" <- service consumer should have env variable ELASTIC_SEARCH_PASSWORD_OSDU with elastic search password
}
```
## Elasticsearch configuration
**prefix:** `elasticsearch`
It can be overridden by:
......@@ -56,7 +74,8 @@ It can be overridden by:
| --- | --- |
| elasticsearch.host | server URL |
| elasticsearch.port | server port |
| elasticsearch.configuration | username and password |
| elasticsearch.user | username |
| elasticsearch.password | password |
<details><summary>Example of a definition for a single tenant</summary></details>
......@@ -72,9 +91,13 @@ curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H '
"sensitive": false,
"value": "9243"
},
"elasticsearch.configuration": {
"elasticsearch.user": {
"sensitive": true,
"value": "<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
},
"elasticsearch.password": {
"sensitive": true,
"value": "elasticuser:elasticpassword"
"value": "<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
}
}
}'
......
......@@ -10,6 +10,8 @@ Must have:
| --- | --- | --- | --- | --- |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - |
| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | yes | - |
| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | false | - |
Defined in default application property file but possible to override:
......@@ -44,10 +46,27 @@ At Pubsub should be created topic with name:
**name:** `indexing-progress`
## Elasticsearch configuration
### Properties set in Partition service:
Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name.
This variable should be present in environment of service that need that variable.
Example:
```
"elasticsearch.port": {
"sensitive": false, <- value not sensitive
"value": "9243" <- will be used as is.
},
"elasticsearch.password": {
"sensitive": true, <- value is sensitive
"value": "ELASTIC_SEARCH_PASSWORD_OSDU" <- service consumer should have env variable ELASTIC_SEARCH_PASSWORD_OSDU with elastic search password
}
```
There is no hardcode in services, all behaviour defined by sensitivity of property.
## Elasticsearch configuration
**prefix:** `elasticsearch`
It can be overridden by:
......@@ -61,7 +80,8 @@ It can be overridden by:
| --- | --- |
| elasticsearch.host | server URL |
| elasticsearch.port | server port |
| elasticsearch.configuration | username and password |
| elasticsearch.user | username |
| elasticsearch.password | password |
<details><summary>Example of a definition for a single tenant</summary></details>
......@@ -77,9 +97,13 @@ curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H '
"sensitive": false,
"value": "9243"
},
"elasticsearch.configuration": {
"elasticsearch.user": {
"sensitive": true,
"value": "<USER_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
},
"elasticsearch.password": {
"sensitive": true,
"value": "elasticuser:elasticpassword"
"value": "<PASSWORD_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
}
}
}'
......
......@@ -19,7 +19,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.14.0-rc1</version>
<version>0.14.0-rc2</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
......
......@@ -20,6 +20,7 @@ package org.opengroup.osdu.indexer.di;
import org.opengroup.osdu.core.common.partition.IPartitionProvider;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.destination.elastic.ElasticSearchDestinationResolver;
import org.opengroup.osdu.core.destination.util.IPartitionPropertyResolver;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
......@@ -27,7 +28,8 @@ import org.springframework.context.annotation.Configuration;
public class ElasticSearchConfig {
@Bean
public IElasticRepository elasticRepository(ElasticSearchConfigurationProperties properties, IPartitionProvider partitionProvider) {
return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider);
public IElasticRepository elasticRepository(ElasticSearchConfigurationProperties properties,
IPartitionProvider partitionProvider, IPartitionPropertyResolver propertyResolver) {
return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider, propertyResolver);
}
}
......@@ -90,6 +90,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
@Then("^I should not get any documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
super.i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(index);
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
......
......@@ -36,7 +36,7 @@
<cucumber.version>1.2.5</cucumber.version>
<junit.jupiter.version>5.6.0</junit.jupiter.version>
<elasticsearch.version>7.8.1</elasticsearch.version>
<os-core-common.version>0.12.0-rc3</os-core-common.version>
<os-core-common.version>0.14.0-rc4</os-core-common.version>
</properties>
<dependencies>
......
......@@ -58,6 +58,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
@Then("^I should not get any documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
super.i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(index);
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
......
......@@ -17,7 +17,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
<os-core-common.version>0.12.0-rc1</os-core-common.version>
<os-core-common.version>0.14.0-rc4</os-core-common.version>
</properties>
<dependencies>
......
......@@ -134,6 +134,11 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedCount, numOfIndexedDocuments);
}
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
index = generateActualName(index, timeStamp);
getRecordsInIndex(index, 0);
}
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
index = generateActualName(index, timeStamp);
Map<String, MappingMetadata> elasticMapping = elasticUtils.getMapping(index);
......@@ -253,6 +258,32 @@ public class RecordSteps extends TestsBase {
return numOfIndexedDocuments;
}
private long getRecordsInIndex(String index, int expectedCount) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
// index.refresh_interval is set to default 30s, wait for 40s initially
Thread.sleep(40000);
for (iterator = 0; iterator < 20; iterator++) {
numOfIndexedDocuments = elasticUtils.fetchRecords(index);
if (expectedCount == numOfIndexedDocuments) {
log.info(String.format("index: %s | attempts: %s | documents acknowledged by elastic: %s", index, iterator, numOfIndexedDocuments));
break;
} else {
log.info(String.format("index: %s | documents acknowledged by elastic: %s", index, numOfIndexedDocuments));
Thread.sleep(5000);
}
if ((iterator + 1) % 5 == 0) elasticUtils.refreshIndex(index);
}
if (iterator >= 20) {
fail(String.format("index not created after waiting for %s seconds", ((40000 + iterator * 5000) / 1000)));
}
return numOfIndexedDocuments;
}
private Boolean areJsonEqual(String firstJson, String secondJson) {
Gson gson = new Gson();
Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
......
......@@ -3,13 +3,15 @@ Feature: Indexing of the documents
Background:
Given the schema is created with the following kind
| kind | index | schemaFile |
| tenant1:indexer:test-data--Integration:1.0.1 | tenant1-indexer-test-data--integration-1.0.1 | index_records_1 |
| tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
| tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
| tenant1:indexer:test-mapping--Sync:2.0.0 | tenant1-indexer-test-mapping--sync-2.0.0 | index_record_sync_mapping |
| kind | index | schemaFile |
| tenant1:indexer:test-data--Integration:1.0.1 | tenant1-indexer-test-data--integration-1.0.1 | index_records_1 |
| tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
| tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
| tenant1:indexer:test-mapping--Sync:2.0.0 | tenant1-indexer-test-mapping--sync-2.0.0 | index_record_sync_mapping |
| tenant1:indexer:test-update-data--Integration:1.0.1 | tenant1-indexer-test-update-data--integration-1.0.1 | index_update_records_kind_v1 |
| tenant1:indexer:test-update-data--Integration:2.0.1 | tenant1-indexer-test-update-data--integration-2.0.1 | index_update_records_kind_v2 |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
......@@ -64,4 +66,15 @@ Feature: Indexing of the documents
Examples:
| kind | index | recordFile | mappingFile | acl |
| "tenant1:indexer:test-mapping--Sync:2.0.0" | "tenant1-indexer-test-mapping--sync-2.0.0" | "index_record_sync_mapping" | "index_record_sync_mapping" | "data.default.viewers@tenant1" |
\ No newline at end of file
| "tenant1:indexer:test-mapping--Sync:2.0.0" | "tenant1-indexer-test-mapping--sync-2.0.0" | "index_record_sync_mapping" | "index_record_sync_mapping" | "data.default.viewers@tenant1" |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind_v1>
Then I should get the 1 documents for the <index_v1> in the Elastic Search
Then I ingest records with the <recordFile> with <acl> for a given <kind_v2>
Then I should get the 1 documents for the <index_v2> in the Elastic Search
Then I should not get any documents for the <index_v1> in the Elastic Search
Examples:
| kind_v1 | index_v1 | recordFile | acl | kind_v2 | index_v2 |
| "tenant1:indexer:test-update-data--Integration:1.0.1" | "tenant1-indexer-test-update-data--integration-1.0.1" | "index_update_records_kind_v1" | "data.default.viewers@tenant1" | "tenant1:indexer:test-update-data--Integration:2.0.1" | "tenant1-indexer-test-update-data--integration-2.0.1" |
\ No newline at end of file
[
{
"id": "tenant1:test:testSchemaUpdate<timestamp>",
"data": {
"message": "test record1"
}
}
]
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "tenant1",
"source": "indexer",
"entityType": "test-update-data--Integration",
"schemaVersionMajor": "1",
"schemaVersionMinor": "0",
"schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
"schema": {
"properties": {
"data": {
"allOf": [
{
"type": "object",
"properties": {
"message": {
"type": "string"
}
}
}
]
}
}
}
}
\ No newline at end of file
[
{
"id": "tenant1:test:testSchemaUpdate<timestamp>",
"data": {
"message": "test record2"
}
}
]
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "tenant1",
"source": "indexer",
"entityType": "test-update-data--Integration",
"schemaVersionMajor": "2",
"schemaVersionMinor": "0",
"schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
"schema": {
"properties": {
"data": {
"allOf": [
{
"type": "object",
"properties": {
"message": {
"type": "string"
}
}
}
]
}
}
}
}
\ No newline at end of file
......@@ -46,6 +46,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
@Then("^I should not get any documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
super.i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(index);
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment