Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • osdu/platform/system/indexer-service
  • schundu/indexer-service
2 results
Show changes
Showing
with 81 additions and 66 deletions
......@@ -14,22 +14,20 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
<version>0.0.5-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-azure</artifactId>
<version>0.0.3-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
......@@ -47,7 +45,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
<version>0.0.3-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<!-- Azure dependencies -->
......@@ -55,13 +53,6 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>0.0.33</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -176,6 +167,18 @@
<scope>test</scope>
</dependency>
<!--Added dependency for utilizing azure service principle class-->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>0.0.33</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
......
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
<version>0.0.5-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
<version>0.0.3-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
......
......@@ -87,8 +87,8 @@ public class CleanupIndiciesSteps extends TestsBase {
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> testRecord : records) {
testRecord.put("id", generateActualName(testRecord.get("id").toString(), timeStamp));
testRecord.put("kind", actualKind);
testRecord.put("id", generateActualId(testRecord.get("id").toString(), timeStamp, actualKind));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
......
......@@ -96,8 +96,8 @@ public class RecordSteps extends TestsBase {
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> testRecord : records) {
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("kind", actualKind);
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
......@@ -113,7 +113,7 @@ public class RecordSteps extends TestsBase {
}
protected String generateRecordId(Map<String, Object> testRecord) {
return generateActualName(testRecord.get("id").toString(), timeStamp);
return generateActualId(testRecord.get("id").toString(), timeStamp, testRecord.get("kind").toString());
}
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
......
......@@ -39,7 +39,7 @@ public class SchemaServiceRecordSteps extends RecordSteps {
@Override
protected String generateRecordId(Map<String, Object> testRecord) {
return generateActualNameWithoutTs(testRecord.get("id").toString());
return generateActualIdWithoutTs(testRecord.get("id").toString(), testRecord.get("kind").toString());
}
@Override
......
......@@ -130,6 +130,17 @@ public abstract class TestsBase {
return rawName.replaceAll("<timestamp>", timeStamp);
}
protected String generateActualId(String rawName, String timeStamp, String kind) {
rawName = generateActualName(rawName, timeStamp);
String kindSubType = kind.split(":")[2];
return rawName.replaceAll("<kindSubType>", kindSubType);
}
public String generateActualNameWithoutTs(String rawName) {
for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
rawName = rawName.replaceAll(tenant.getKey(), tenant.getValue());
......@@ -137,6 +148,14 @@ public abstract class TestsBase {
return rawName.replaceAll("<timestamp>", "");
}
public String generateActualIdWithoutTs(String rawName, String kind) {
rawName = generateActualNameWithoutTs(rawName);
String kindSubType = kind.split(":")[2];
return rawName.replaceAll("<kindSubType>", kindSubType);
}
protected Legal generateLegalTag() {
Legal legal = new Legal();
Set<String> legalTags = new HashSet<>();
......
......@@ -14,9 +14,9 @@ Feature: Indexing of the documents
Then I should get the elastic <mapping> for the <type> and <index> in the Elastic Search
Examples:
| kind | recordFile | number | index | type | acl | mapping |
| "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| kind | recordFile | number | index | type | acl | mapping |
| "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
......
[
{
"id": "tenant1:ihs:testIngest2<timestamp>",
"id": "tenant1:<kindSubType>:testIngest2<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -23,7 +23,7 @@
}
},
{
"id": "tenant1:ihs:testIngest3<timestamp>",
"id": "tenant1:<kindSubType>:testIngest3<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -46,7 +46,7 @@
}
},
{
"id": "tenant1:ihs:testIngest4<timestamp>",
"id": "tenant1:<kindSubType>:testIngest4<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -68,7 +68,7 @@
}
},
{
"id": "tenant1:ihs:testIngest5<timestamp>",
"id": "tenant1:<kindSubType>:testIngest5<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -90,7 +90,7 @@
}
},
{
"id": "tenant1:ihs:testIngest6<timestamp>",
"id": "tenant1:<kindSubType>:testIngest6<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......
[
{
"id": "tenant1:ihs:testIngest7<timestamp>",
"id": "tenant1:<kindSubType>:testIngest7<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -22,7 +22,7 @@
}
},
{
"id": "tenant1:ihs:testIngest8<timestamp>",
"id": "tenant1:<kindSubType>:testIngest8<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -44,7 +44,7 @@
}
},
{
"id": "tenant1:ihs:testIngest9<timestamp>",
"id": "tenant1:<kindSubType>:testIngest9<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -66,7 +66,7 @@
}
},
{
"id": "tenant1:ihs:testIngest10<timestamp>",
"id": "tenant1:<kindSubType>:testIngest10<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -88,7 +88,7 @@
}
},
{
"id": "tenant1:ihs:testIngest11<timestamp>",
"id": "tenant1:<kindSubType>:testIngest11<timestamp>",
"data": {
"Field": 1234,
"Location": {
......
[
{
"id": "tenant1:ihs:testIngest51<timestamp>",
"id": "tenant1:<kindSubType>:testIngest51<timestamp>",
"data": {
"WellName": "Data Platform Services - 51",
"GeoShape": {
......@@ -13,7 +13,7 @@
}
},
{
"id": "tenant1:ihs:testIngest52<timestamp>",
"id": "tenant1:<kindSubType>:testIngest52<timestamp>",
"data": {
"WellName": "Data Platform Services - 52",
"GeoShape": {
......@@ -32,7 +32,7 @@
}
},
{
"id": "tenant1:ihs:testIngest53<timestamp>",
"id": "tenant1:<kindSubType>:testIngest53<timestamp>",
"data": {
"WellName": "Data Platform Services - 53",
"GeoShape": {
......@@ -147,7 +147,7 @@
}
},
{
"id": "tenant1:ihs:testIngest54<timestamp>",
"id": "tenant1:<kindSubType>:testIngest54<timestamp>",
"data": {
"WellName": "Data Platform Services - 54",
"GeoShape": {
......@@ -238,7 +238,7 @@
}
},
{
"id": "tenant1:ihs:testIngest55<timestamp>",
"id": "tenant1:<kindSubType>:testIngest55<timestamp>",
"data": {
"WellName": "Data Platform Services - 55",
"GeoShape": {
......@@ -271,7 +271,7 @@
}
},
{
"id": "tenant1:ihs:testIngest56<timestamp>",
"id": "tenant1:<kindSubType>:testIngest56<timestamp>",
"data": {
"WellName": "Data Platform Services - 56",
"GeoShape": {
......@@ -330,7 +330,7 @@
}
},
{
"id": "tenant1:ihs:testIngest57<timestamp>",
"id": "tenant1:<kindSubType>:testIngest57<timestamp>",
"data": {
"WellName": "Data Platform Services - 57",
"GeoShape": {
......
[
{
"id": "test:well:1.0.0:1",
"id": "tenant1:well:1.0.0:1",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -38,7 +38,7 @@
}
},
{
"id": "test:well:1.0.0:2",
"id": "tenant1:well:1.0.0:2",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -77,7 +77,7 @@
}
},
{
"id": "test:well:1.0.0:3",
"id": "tenant1:well:1.0.0:3",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......
[
{
"id": "test:well:2.0.0:1",
"id": "tenant1:well:2.0.0:1",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......@@ -37,7 +37,7 @@
}
},
{
"id": "test:well:2.0.0:2",
"id": "tenant1:well:2.0.0:2",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......@@ -74,7 +74,7 @@
}
},
{
"id": "test:well:2.0.0:3",
"id": "tenant1:well:2.0.0:3",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......
[
{
"id": "test:well:1.0.0:1",
"id": "tenant1:well:1.0.0:1",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -39,7 +39,7 @@
}
},
{
"id": "test:well:1.0.0:2",
"id": "tenant1:well:1.0.0:2",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -79,7 +79,7 @@
}
},
{
"id": "test:well:1.0.0:3",
"id": "tenant1:well:1.0.0:3",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
<version>0.0.5-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-gcp</artifactId>
<version>0.0.3-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
......@@ -39,7 +37,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
<version>0.0.3-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<!-- Cucumber -->
......
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
<version>0.0.5-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-ibm</artifactId>
<version>0.0.3</version>
<version>0.0.2</version>
<packaging>jar</packaging>
<properties>
......@@ -40,7 +38,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
<version>0.0.3-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
......
......@@ -14,12 +14,11 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
<version>0.0.5-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<description>Indexer Service Integration Test Root Project</description>
<properties>
<spring.version>5.1.19.RELEASE</spring.version>
......