Skip to content
Snippets Groups Projects
Commit d5694878 authored by Neelesh Thakur's avatar Neelesh Thakur
Browse files

IT

parent e1584572
No related branches found
No related tags found
1 merge request!170enable indexing of audit-trail attributes
Pipeline #52511 failed
Stage: .pre
Stage: review
Stage: build
Pipeline: Indexer

#52512

    Hide whitespace changes
    Inline Side-by-side
    Showing
    with 58 additions and 14 deletions
    indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
    + 2
    − 2
    View file @ d5694878
    ......@@ -49,8 +49,8 @@ public class TypeMapper {
    metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
    metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
    metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
    metaAttributeIndexerType.put(RecordMetaAttribute.AUTHORITY.getValue(), ElasticType.KEYWORD.getValue());
    metaAttributeIndexerType.put(RecordMetaAttribute.SOURCE.getValue(), ElasticType.KEYWORD.getValue());
    metaAttributeIndexerType.put(RecordMetaAttribute.AUTHORITY.getValue(), ElasticType.CONSTANT_KEYWORD.getValue());
    metaAttributeIndexerType.put(RecordMetaAttribute.SOURCE.getValue(), ElasticType.CONSTANT_KEYWORD.getValue());
    metaAttributeIndexerType.put(RecordMetaAttribute.CREATE_USER.getValue(), ElasticType.KEYWORD.getValue());
    metaAttributeIndexerType.put(RecordMetaAttribute.MODIFY_USER.getValue(), ElasticType.KEYWORD.getValue());
    metaAttributeIndexerType.put(RecordMetaAttribute.CREATE_TIME.getValue(), ElasticType.DATE.getValue());
    ......
    indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
    + 2
    − 2
    View file @ d5694878
    ......@@ -31,8 +31,8 @@ public class TypeMapperTest {
    assertEquals(ElasticType.LONG.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.VERSION));
    assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.X_ACL));
    assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.AUTHORITY));
    assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.SOURCE));
    assertEquals(ElasticType.CONSTANT_KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.AUTHORITY));
    assertEquals(ElasticType.CONSTANT_KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.SOURCE));
    assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_USER));
    assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_USER));
    assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_TIME));
    ......
    testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
    + 7
    − 2
    View file @ d5694878
    ......@@ -54,8 +54,13 @@ public class Steps extends SchemaServiceRecordSteps {
    }
    @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
    public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
    super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
    public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
    super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
    }
    @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
    public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
    super.i_can_validate_indexed_attributes(index, kind);
    }
    @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
    ......
    testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
    + 23
    − 4
    View file @ d5694878
    ......@@ -48,7 +48,7 @@ public class RecordSteps extends TestsBase {
    public RecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
    super(httpClient, elasticUtils);
    }
    /******************One time cleanup for whole feature**************/
    public void tearDown() {
    for (String kind : inputIndexMap.keySet()) {
    ......@@ -101,7 +101,7 @@ public class RecordSteps extends TestsBase {
    for (Map<String, Object> testRecord : records) {
    testRecord.put("kind", actualKind);
    testRecord.put("id", generateRecordId(testRecord));
    testRecord.put("id", generateRecordId(testRecord));
    testRecord.put("legal", generateLegalTag());
    String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
    Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
    ......@@ -126,17 +126,36 @@ public class RecordSteps extends TestsBase {
    assertEquals(expectedCount, numOfIndexedDocuments);
    }
    public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
    public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
    index = generateActualName(index, timeStamp);
    Map<String, MappingMetadata> elasticMapping = elasticUtils.getMapping(index);
    assertNotNull(elasticMapping);
    String[] kindParts = kind.split(":");
    String authority = tenantMap.get(kindParts[0]);
    String source = kindParts[1];
    expectedMapping = expectedMapping.replaceAll("<authority-id>", authority).replaceAll("<source-id>", source);
    MappingMetadata typeMapping = elasticMapping.get(index);
    Map<String, Object> mapping = typeMapping.sourceAsMap();
    assertNotNull(mapping);
    assertTrue(areJsonEqual(expectedMapping, mapping.toString()));
    }
    public void i_can_validate_indexed_attributes(String index, String kind) throws Throwable {
    String authority = tenantMap.get(kind.substring(0, kind.indexOf(":")));
    index = generateActualName(index, timeStamp);
    List<Map<String, Object>> hits = elasticUtils.fetchRecordsByAttribute(index, "authority", authority);
    assertTrue(hits.size() > 0);
    for (Map<String, Object> result : hits) {
    assertTrue(result.containsKey("authority"));
    assertEquals(authority, result.get("authority"));
    assertTrue(result.containsKey("source"));
    assertTrue(result.containsKey("createUser"));
    assertTrue(result.containsKey("createTime"));
    }
    }
    public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
    index = generateActualName(index, timeStamp);
    long numOfIndexedDocuments = createIndex(index);
    ......@@ -228,7 +247,7 @@ public class RecordSteps extends TestsBase {
    log.info(String.format("difference: %s", result.entriesDiffering()));
    return false;
    }
    @Override
    protected String getApi() {
    return null;
    ......
    testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
    + 19
    − 0
    View file @ d5694878
    ......@@ -52,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexRequest;
    import org.elasticsearch.client.indices.GetMappingsRequest;
    import org.elasticsearch.client.indices.GetMappingsResponse;
    import org.elasticsearch.cluster.metadata.MappingMetadata;
    import org.elasticsearch.search.SearchHit;
    import org.elasticsearch.search.SearchHits;
    import org.locationtech.jts.geom.Coordinate;
    import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
    ......@@ -241,6 +242,24 @@ public class ElasticUtils {
    }
    }
    public List<Map<String, Object>> fetchRecordsByAttribute(String index, String attributeKey, String attributeValue) throws IOException {
    List<Map<String, Object>> out = new ArrayList<>();
    try {
    try (RestHighLevelClient client = this.createClient(username, password, host)) {
    SearchRequest request = new SearchRequest(index);
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
    sourceBuilder.query(boolQuery().must(termsQuery(attributeKey, attributeValue)));
    request.source(sourceBuilder);
    SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
    for(SearchHit searchHit : searchResponse.getHits()) out.add(searchHit.getSourceAsMap());
    return out;
    }
    } catch (ElasticsearchStatusException e) {
    log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
    return out;
    }
    }
    public long fetchRecordsByExistQuery(String index, String attributeName) throws Exception {
    try {
    TimeUnit.SECONDS.sleep(40);
    ......
    testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
    + 5
    − 4
    View file @ d5694878
    ......@@ -13,12 +13,13 @@ Feature: Indexing of the documents
    Scenario Outline: Ingest the record and Index in the Elastic Search
    When I ingest records with the <recordFile> with <acl> for a given <kind>
    Then I should get the <number> documents for the <index> in the Elastic Search
    Then I should get the elastic <mapping> for the <type> and <index> in the Elastic Search
    Then I should get the elastic <mapping> for the <kind> and <index> in the Elastic Search
    Then I can validate indexed meta attributes for the <index> and given <kind>
    Examples:
    | kind | recordFile | number | index | type | acl | mapping |
    | "tenant1:indexer:test-data--Integration:1.0.0" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.0" | "test-data--Integration" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
    | "tenant1:indexer:test-data--Integration:3.0.0" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.0" | "test-data--Integration" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
    | kind | recordFile | number | index | acl | mapping |
    | "tenant1:indexer:test-data--Integration:1.0.0" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.0" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
    | "tenant1:indexer:test-data--Integration:3.0.0" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.0" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
    Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
    When I ingest records with the <recordFile> with <acl> for a given <kind>
    ......
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Please register or to comment