Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/indexer-service

provider/indexer-ibm/src/main/resources/application.properties

@@ -26,6 +26,8 @@ KINDS_REDIS_DATABASE=1
 CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
 CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
 
+SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
+
 storage_service_url=http://localhost:8082
 #storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud
 STORAGE_SCHEMA_HOST=${storage_service_url}/api/storage/v2/schemas
@@ -64,3 +66,5 @@ ELASTIC_DATASTORE_ID=indexer-service
 ELASTIC_HOST=elasticsearch.com
 ELASTIC_PORT=443
 ELASTIC_USER_PASSWORD=REPLACE_ME:REPLACE_ME
+
+

testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java

@@ -31,7 +31,6 @@ import org.opengroup.osdu.util.LegalTagUtilsAws;
 import java.util.HashSet;
 import java.util.Set;
 
-import static org.opengroup.osdu.util.Config.getLegalTag;
 import static org.opengroup.osdu.util.Config.getOtherRelevantDataCountries;
 
 @Log

testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java

@@ -20,7 +20,7 @@ import org.junit.runner.RunWith;
 
 @RunWith(Cucumber.class)
 @CucumberOptions(
-        features = "classpath:features/indexrecord/IndexRecord.feature",
+        features = "classpath:features/indexrecord/indexRecord-schema-service.feature",
         glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
         plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
 public class RunTest {

testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java

@@ -14,20 +14,21 @@
 
 package org.opengroup.osdu.step_definitions.index.record;
 
-import lombok.extern.java.Log;
-
 import cucumber.api.Scenario;
 import cucumber.api.java.Before;
+import lombok.extern.java.Log;
+
 import cucumber.api.DataTable;
 import cucumber.api.java.en.Given;
 import cucumber.api.java.en.Then;
 import cucumber.api.java.en.When;
-import org.opengroup.osdu.common.RecordSteps;
+import org.opengroup.osdu.common.SchemaServiceRecordSteps;
 import org.opengroup.osdu.util.AzureHTTPClient;
 import org.opengroup.osdu.util.ElasticUtils;
 
+
 @Log
-public class Steps extends RecordSteps {
+public class Steps extends SchemaServiceRecordSteps {
 
     public Steps() {
         super(new AzureHTTPClient(), new ElasticUtils());
@@ -36,7 +37,6 @@ public class Steps extends RecordSteps {
     @Before
     public void before(Scenario scenario) {
         this.scenario = scenario;
-        this.httpClient = new AzureHTTPClient();
     }
 
     @Given("^the schema is created with the following kind$")

testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureHTTPClient.java

@@ -17,7 +17,6 @@ package org.opengroup.osdu.util;
 import lombok.ToString;
 import lombok.extern.java.Log;
 
-import java.io.IOException;
 
 @Log
 @ToString

testing/indexer-test-core/pom.xml

@@ -129,5 +129,10 @@
             <artifactId>guava</artifactId>
             <version>27.1-jre</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.11.3</version>
+        </dependency>
     </dependencies>
 </project>

testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java

@@ -15,6 +15,7 @@ import org.opengroup.osdu.models.TestIndex;
 import org.opengroup.osdu.util.ElasticUtils;
 import org.opengroup.osdu.util.FileHandler;
 import org.opengroup.osdu.util.HTTPClient;
+import org.springframework.util.CollectionUtils;
 
 import javax.ws.rs.HttpMethod;
 import java.io.IOException;
@@ -22,6 +23,7 @@ import java.lang.reflect.Type;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.logging.Level;
 
 import static org.junit.Assert.*;
 import static org.opengroup.osdu.util.Config.getEntitlementsDomain;
@@ -51,12 +53,17 @@ public class RecordSteps extends TestsBase {
             testIndex.cleanupIndex();
             testIndex.deleteSchema(kind);
         }
-        if (records != null && records.size() > 0) {
-            for (Map<String, Object> testRecord : records) {
-                String id = testRecord.get("id").toString();
-                httpClient.send(HttpMethod.DELETE, getStorageBaseURL() + "records/" + id, null, headers, httpClient.getAccessToken());
-                log.info("Deleted the records");
-            }
+
+        if (!CollectionUtils.isEmpty(records)) {
+            cleanupRecords();
+        }
+    }
+
+    protected void cleanupRecords() {
+        for (Map<String, Object> testRecord : records) {
+            String id = testRecord.get("id").toString();
+            httpClient.send(HttpMethod.DELETE, getStorageBaseURL() + "records/" + id, null, headers, httpClient.getAccessToken());
+            log.info("Deleted the records");
         }
     }
 
@@ -74,17 +81,12 @@ public class RecordSteps extends TestsBase {
 
         /******************One time setup for whole feature**************/
         if (!shutDownHookAdded) {
-            Runtime.getRuntime().addShutdownHook(new Thread() {
-                public void run() {
-                    tearDown();
-                }
-            });
-            shutDownHookAdded = true;
             for (String kind : inputIndexMap.keySet()) {
                 TestIndex testIndex = inputIndexMap.get(kind);
                 testIndex.setupSchema();
             }
         }
+        addShutDownHook();
     }
 
     public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
@@ -95,7 +97,7 @@ public class RecordSteps extends TestsBase {
             records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
 
             for (Map<String, Object> testRecord : records) {
-                testRecord.put("id", generateActualName(testRecord.get("id").toString(), timeStamp));
+                testRecord.put("id", generateRecordId(testRecord));
                 testRecord.put("kind", actualKind);
                 testRecord.put("legal", generateLegalTag());
                 String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
@@ -103,6 +105,7 @@ public class RecordSteps extends TestsBase {
                 testRecord.put("acl", acl);
             }
             String payLoad = new Gson().toJson(records);
+            log.log(Level.INFO, "Start ingesting records={0}", payLoad);
             ClientResponse clientResponse = httpClient.send(HttpMethod.PUT, getStorageBaseURL() + "records", payLoad, headers, httpClient.getAccessToken());
             assertEquals(201, clientResponse.getStatus());
         } catch (Exception ex) {
@@ -110,6 +113,10 @@ public class RecordSteps extends TestsBase {
         }
     }
 
+    protected String generateRecordId(Map<String, Object> testRecord) {
+        return generateActualName(testRecord.get("id").toString(), timeStamp);
+    }
+
     public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
         index = generateActualName(index, timeStamp);
         long numOfIndexedDocuments = createIndex(index);
@@ -182,4 +189,18 @@ public class RecordSteps extends TestsBase {
         return null;
     }
 
+    public Map<String, TestIndex> getInputIndexMap() {
+        return inputIndexMap;
+    }
+
+    public String getTimeStamp() {
+        return timeStamp;
+    }
+
+    protected void addShutDownHook() {
+        if (!shutDownHookAdded) {
+            Runtime.getRuntime().addShutdownHook(new Thread(this::tearDown));
+            shutDownHookAdded = true;
+        }
+    }
 }
\ No newline at end of file

testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java

+package org.opengroup.osdu.common;
+
+import cucumber.api.DataTable;
+import org.opengroup.osdu.models.Setup;
+import org.opengroup.osdu.models.schema.PersistentSchemaTestIndex;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.HTTPClient;
+
+import java.util.List;
+import java.util.Map;
+
+public class SchemaServiceRecordSteps extends RecordSteps {
+
+    public SchemaServiceRecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
+        super(httpClient, elasticUtils);
+    }
+
+    public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+        List<Setup> inputList = dataTable.asList(Setup.class);
+        inputList.forEach(this::createSchema);
+        inputList.forEach(s -> deleteIndex(generateActualNameWithoutTs(s.getIndex())));
+        super.addShutDownHook();
+    }
+
+    private void createSchema(Setup input) {
+        PersistentSchemaTestIndex testIndex = new PersistentSchemaTestIndex(super.elasticUtils, super.httpClient, this);
+        testIndex.setIndex(generateActualName(input.getIndex(), super.getTimeStamp()));
+        testIndex.setSchemaFile(input.getSchemaFile());
+        testIndex.setHttpClient(super.httpClient);
+        testIndex.setupSchema();
+        testIndex.setKind(testIndex.getSchemaModel().getSchemaInfo().getSchemaIdentity().getId());
+
+        super.getInputIndexMap().put(testIndex.getKind(), testIndex);
+    }
+
+    private void deleteIndex(String index) {
+        this.elasticUtils.deleteIndex(index);
+    }
+
+    @Override
+    protected String generateRecordId(Map<String, Object> testRecord) {
+        return generateActualNameWithoutTs(testRecord.get("id").toString());
+    }
+
+    @Override
+    public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+        super.i_ingest_records_with_the_for_a_given(record.replaceFirst("_schema", ""), dataGroup, kind);
+    }
+
+}

testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java

@@ -123,18 +123,18 @@ public abstract class TestsBase {
         log.info(String.format("Scenario Name: %s, Correlation-Id: %s", scenario.getId(), headers.get("correlation-id")));
     }
 
-    protected String getTenantMapping(String tenant) {
-        if (tenantMap.containsKey(tenant)) {
-            return tenantMap.get(tenant);
+    protected String generateActualName(String rawName, String timeStamp) {
+        for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
+            rawName = rawName.replaceAll(tenant.getKey(), tenant.getValue());
         }
-        return null;
+        return rawName.replaceAll("<timestamp>", timeStamp);
     }
 
-    protected String generateActualName(String rawName, String timeStamp) {
-        for (String tenant : tenantMap.keySet()) {
-            rawName = rawName.replaceAll(tenant, getTenantMapping(tenant));
+    public String generateActualNameWithoutTs(String rawName) {
+        for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
+            rawName = rawName.replaceAll(tenant.getKey(), tenant.getValue());
         }
-        return rawName.replaceAll("<timestamp>", timeStamp);
+        return rawName.replaceAll("<timestamp>", "");
     }
 
     protected Legal generateLegalTag() {

testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java

@@ -84,7 +84,7 @@ public class TestIndex {
         return String.format("%s.mapping", this.mappingFile);
     }
 
-    private String getSchemaFile() {
+    protected String getSchemaFile() {
         return String.format("%s.schema", this.schemaFile);
     }
 

testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/schema/PersistentSchemaTestIndex.java

+package org.opengroup.osdu.models.schema;
+
+import org.opengroup.osdu.common.SchemaServiceRecordSteps;
+import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.FileHandler;
+import org.opengroup.osdu.util.HTTPClient;
+import org.opengroup.osdu.util.SchemaServiceClient;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+
+public class PersistentSchemaTestIndex extends TestIndex {
+
+    private static final Logger LOGGER = Logger.getLogger(PersistentSchemaTestIndex.class.getName());
+    private final SchemaServiceClient schemaServiceClient;
+    private final SchemaServiceRecordSteps recordSteps;
+    private SchemaModel schemaModel;
+
+    public PersistentSchemaTestIndex(ElasticUtils elasticUtils, HTTPClient client, SchemaServiceRecordSteps recordSteps) {
+        super(elasticUtils);
+        this.schemaServiceClient = new SchemaServiceClient(client);
+        this.recordSteps = recordSteps;
+    }
+
+    @Override
+    public void setupSchema() {
+        this.schemaModel = readSchemaFromJson();
+        SchemaIdentity schemaIdentity = schemaModel.getSchemaInfo().getSchemaIdentity();
+        LOGGER.log(Level.INFO, "Read the schema={0}", schemaIdentity);
+        schemaIdentity.setAuthority(recordSteps.generateActualNameWithoutTs(schemaIdentity.getAuthority()));
+        LOGGER.log(Level.INFO, "Updated the schema={0}", schemaIdentity);
+        schemaServiceClient.createIfNotExist(schemaModel);
+        LOGGER.log(Level.INFO, "Finished setting up the schema={0}", schemaIdentity);
+    }
+
+    @Override
+    public void deleteSchema(String kind) {
+        // The DELETE API is not supported in the Schema service.
+        // In order not to overwhelm a DB with a lots of test schemas
+        // the integration tests create/update a schema per schema file if the schema does not exists
+        // If a developer updates the schema manually, the developer is supposed to update its version as well
+    }
+
+    private SchemaModel readSchemaFromJson(){
+        try {
+            return FileHandler.readFile(getSchemaFile(), SchemaModel.class);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public SchemaModel getSchemaModel() {
+        return schemaModel;
+    }
+
+    @Override
+    protected String getSchemaFile() {
+        return super.getSchemaFile() + ".json";
+    }
+}