Skip to content
Snippets Groups Projects
Commit ba811128 authored by MIchael Nguyen's avatar MIchael Nguyen
Browse files

updating tests.

parent 229b4a61
No related branches found
No related tags found
1 merge request!6Trusted ibm
Showing
with 505 additions and 22 deletions
......@@ -113,7 +113,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-search-core-lib</artifactId>
<version>1.0.8</version>
<version>1.0.11</version>
<scope>compile</scope>
</dependency>
<dependency>
......@@ -138,4 +138,23 @@
<version>6.6.2</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>org.opengroup.osdu.indexer.IndexerApplication</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
......@@ -2,26 +2,17 @@ package org.opengroup.osdu.indexer.aws.util;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.model.*;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.apache.http.HttpStatus;
import org.elasticsearch.index.Index;
import org.opengroup.osdu.core.aws.sqs.SQSBuilder;
import org.opengroup.osdu.indexer.aws.model.CallableResult;
import org.opengroup.osdu.indexer.aws.model.SQSMessage;
import org.opengroup.osdu.indexer.aws.util.sqs.IndexProcessor;
import org.opengroup.osdu.indexer.service.IndexerService;
import org.opengroup.osdu.indexer.util.JobStatus;
import org.opengroup.osdu.indexer.util.RecordInfo;
import org.opengroup.osdu.is.core.model.RecordChangedMessages;
import org.opengroup.osdu.is.core.provider.interfaces.util.IRequestInfo;
import org.opengroup.osdu.is.core.util.AppException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
......
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-aws</artifactId>
<version>0.0.1</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
<version>0.0.1</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-request-signing-apache-interceptor</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>aws-osdu-util</artifactId>
<version>0.0.1</version>
</dependency>
<!-- Cucumber -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>info.cukes</groupId>
<artifactId>cucumber-java</artifactId>
<version>${cucumber.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>info.cukes</groupId>
<artifactId>cucumber-junit</artifactId>
<version>${cucumber.version}</version>
<scope>test</scope>
</dependency>
<!-- Gson: Java to Json conversion -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.5</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>2.9.9</version>
</dependency>
<dependency>
<groupId>org.glassfish</groupId>
<artifactId>javax.json</artifactId>
<version>1.1.4</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>1.19.4</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<!--Elasticsearch-->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.6.2</version>
</dependency>
<!--Logging-->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>2.11.2</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>1.8.0-beta4</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>27.1-jre</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package org.opengroup.osdu.step_definitions.index.deleteschema;
import cucumber.api.CucumberOptions;
import cucumber.api.junit.Cucumber;
import org.junit.runner.RunWith;
@RunWith(Cucumber.class)
@CucumberOptions(
features = "classpath:features/delete/Delete.feature",
glue={"classpath:org.opengroup.osdu.step_definitions/index/deleteschema"},
format = {"pretty", "junit:target/cucumber-reports/TEST-deleteschema.xml"})
public class RunTest {
}
\ No newline at end of file
package org.opengroup.osdu.step_definitions.index.deleteschema;
import cucumber.api.Scenario;
import cucumber.api.java.Before;
import org.opengroup.osdu.common.DeleteSchemaSteps;
import org.opengroup.osdu.util.AWSHTTPClient;
import cucumber.api.DataTable;
import cucumber.api.java.en.Given;
import cucumber.api.java.en.Then;
import cucumber.api.java.en.When;
import java.util.List;
public class Steps extends DeleteSchemaSteps {
public Steps() {
super(new AWSHTTPClient());
}
@Before
public void before(Scenario scenario) {
this.scenario = scenario;
this.httpClient = new AWSHTTPClient();
}
@Given("^the elastic search is initialized with the following data$")
public void the_elastic_search_is_initialized_with_the_following_data(DataTable dataTable) throws Throwable {
super.the_elastic_search_is_initialized_with_the_following_data(dataTable);
}
@When("^I send a delete request with \"([^\"]*)\"$")
public void i_send_a_delete_request_with(String kind) throws Throwable {
super.i_send_a_delete_request_with(kind);
}
@Then("^the index should get delete and I should get (\\d+) response$")
public void the_index_should_get_delete_and_I_should_get_response(int code) throws Throwable {
super.the_index_should_get_delete_and_I_should_get_response(code);
}
@Then("^I should get ([^\"]*) response with reason: \"(.*?)\", message: \"(.*?)\" and errors: \"(.*?)\"$")
public void i_should_get_response_with_reason_message_and_errors(List<Integer> codes, String type, String msg,
String error) throws Throwable {
super.i_should_get_response_with_reason_message_and_errors(codes, type, msg, error);
}
}
package org.opengroup.osdu.step_definitions.index.getschema;
import cucumber.api.CucumberOptions;
import cucumber.api.junit.Cucumber;
import org.junit.runner.RunWith;
@RunWith(Cucumber.class)
@CucumberOptions(
features = "classpath:features/kindschema/KindSchema.feature",
glue = {"classpath:org.opengroup.osdu.step_definitions/index/getschema"},
format = {"pretty", "junit:target/cucumber-reports/TEST-getschema.xml"})
public class RunTest {
}
\ No newline at end of file
package org.opengroup.osdu.step_definitions.index.getschema;
import org.opengroup.osdu.common.GetSchemaSteps;
import org.opengroup.osdu.util.AWSHTTPClient;
import cucumber.api.Scenario;
import cucumber.api.java.Before;
import cucumber.api.DataTable;
import cucumber.api.java.en.Given;
import cucumber.api.java.en.Then;
import cucumber.api.java.en.When;
public class Steps extends GetSchemaSteps {
public Steps() {
super(new AWSHTTPClient());
}
@Before
public void before(Scenario scenario) {
this.scenario = scenario;
this.httpClient = new AWSHTTPClient();
}
@Given("^the elastic search is initialized with the following data$")
public void the_elastic_search_is_initialized_with_the_following_data(DataTable dataTable) throws Throwable {
super.the_elastic_search_is_initialized_with_the_following_data(dataTable);
}
@When("^I send get schema request with \"([^\"]*)\"$")
public void i_send_get_schema_request_with(String kind) throws Throwable {
super.i_send_get_schema_request_with(kind);
}
@When("^I send request to tenant \"(.*?)\"$")
public void i_send_request_to_tenant(String tenant) throws Throwable {
super.i_send_request_to_tenant(tenant);
}
@Then("^I should get ([^\"]*) response with reason: \"(.*?)\", message: \"(.*?)\" and errors: \"(.*?)\"$")
public void i_should_get_response_with_reason_message_and_errors(int responseCode, String type, String msg,
String error) throws Throwable {
super.i_should_get_response_with_reason_message_and_errors(responseCode, type, msg, error);
}
@Then("^I should get (\\d+) status with response \"(.*?)\"$")
public void i_should_get_status_with_response(int statusCode, String response) throws Throwable {
super.i_should_get_status_with_response(statusCode, response);
}
}
\ No newline at end of file
package org.opengroup.osdu.step_definitions.index.record;
import cucumber.api.CucumberOptions;
import cucumber.api.junit.Cucumber;
import org.junit.runner.RunWith;
@RunWith(Cucumber.class)
@CucumberOptions(
features = "classpath:features/indexrecord/IndexRecord.feature",
glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
public class RunTest {
}
\ No newline at end of file
package org.opengroup.osdu.step_definitions.index.record;
import lombok.extern.java.Log;
import org.opengroup.osdu.common.RecordSteps;
import org.opengroup.osdu.util.AWSHTTPClient;
import cucumber.api.Scenario;
import cucumber.api.java.Before;
import cucumber.api.DataTable;
import cucumber.api.java.en.Given;
import cucumber.api.java.en.Then;
import cucumber.api.java.en.When;
@Log
public class Steps extends RecordSteps {
public Steps() {
super(new AWSHTTPClient());
}
@Before
public void before(Scenario scenario) {
this.scenario = scenario;
this.httpClient = new AWSHTTPClient();
}
@Given("^the schema is created with the following kind$")
public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
}
}
\ No newline at end of file
package org.opengroup.osdu.util;
import lombok.ToString;
import lombok.extern.java.Log;
import java.io.IOException;
@Log
@ToString
public class AWSHTTPClient extends HTTPClient {
private static String token = null;
@Override
public synchronized String getAccessToken() {
if(token == null) {
try {
token = "Bearer " + JwtTokenUtil.getAccessToken();
} catch (IOException e) {
e.printStackTrace();
}
}
return token;
}
}
\ No newline at end of file
package org.opengroup.osdu.util;
import com.amazonaws.auth.AWS4Signer;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.http.AWSRequestSigningApacheInterceptor;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.opengroup.osdu.core.aws.iam.IAMConfig;
public class ElasticUtilsAws extends ElasticUtils {
@Override
protected RestHighLevelClient createClient(String username, String password, String host) {
AWSCredentialsProvider credentials = new IAMConfig().amazonAWSCredentials();
AWS4Signer signer = new AWS4Signer();
signer.setServiceName(username);
signer.setRegionName(password);
// TODO: Legal please review the use of this code coming from https://github.com/awslabs/aws-request-signing-apache-interceptor
HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(username, signer, credentials);
return new RestHighLevelClient(RestClient.builder(HttpHost.create(host)).setHttpClientConfigCallback(configCallBack -> configCallBack.addInterceptorLast(interceptor)));
}
}
package org.opengroup.osdu.util;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.json.webtoken.JsonWebSignature;
import com.google.api.client.json.webtoken.JsonWebToken;
import com.google.api.client.util.Clock;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import lombok.Data;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
class JwtTokenUtil {
private static String accessToken;
static String getAccessToken() throws IOException {
if (Strings.isNullOrEmpty(accessToken)) {
accessToken = getServiceAccountAccessToken(getJwtForIntegrationTesterAccount());
}
// return accessToken;
return null;
}
private static String getServiceAccountAccessToken(String key) throws IOException {
try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) {
List<NameValuePair> parameters = new ArrayList<>();
parameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"));
parameters.add(new BasicNameValuePair("assertion", key));
HttpPost postRequest = new HttpPost("https://www.googleapis.com/oauth2/v4/token");
postRequest.addHeader("Content-Type", "application/x-www-form-urlencoded");
postRequest.setEntity(new UrlEncodedFormEntity(parameters));
HttpResponse response = httpClient.execute(postRequest);
String responseContent = IOUtils.toString(response.getEntity().getContent(), Charsets.toCharset("UTF-8"));
JwtTokenUtil.ResponseToken responseToken = new Gson().fromJson(responseContent, JwtTokenUtil.ResponseToken.class);
return responseToken.getId_token();
}
}
private static String getJwtForIntegrationTesterAccount() throws IOException {
String serviceAccountFile = Config.getKeyValue();
return getJwt(serviceAccountFile);
}
private static String getJwt(String serviceAccountFile) throws IOException {
String targetAudience = Config.getTargetAudience();
long currentTime = Clock.SYSTEM.currentTimeMillis();
InputStream stream = new ByteArrayInputStream(Base64.getDecoder().decode(serviceAccountFile));
GoogleCredential credential = GoogleCredential.fromStream(stream);
JsonWebSignature.Header header = new JsonWebSignature.Header();
header.setAlgorithm("RS256");
header.setType("JWT");
header.setKeyId(credential.getServiceAccountPrivateKeyId());
JsonWebSignature.Payload payload = new JsonWebToken.Payload();
payload.setIssuedAtTimeSeconds(currentTime / 1000);
payload.setExpirationTimeSeconds(currentTime / 1000 + 3600);
payload.setAudience("https://www.googleapis.com/oauth2/v4/token");
payload.setIssuer(credential.getServiceAccountId());
payload.set("target_audience", targetAudience);
JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
String signedJwt = null;
try {
signedJwt = JsonWebSignature.signUsingRsaSha256(credential.getServiceAccountPrivateKey(), jsonFactory, header, payload);
} catch (GeneralSecurityException e) {
e.printStackTrace();
}
return signedJwt;
}
@Data
class ResponseToken {
public String id_token;
}
}
......@@ -2,9 +2,9 @@ package org.opengroup.osdu.util;
public class Config {
private static final String DEFAULT_ELASTIC_HOST = "";
private static final String DEFAULT_ELASTIC_USER_NAME = "";
private static final String DEFAULT_ELASTIC_PASSWORD = "";
private static final String DEFAULT_ELASTIC_HOST = "https://search-dev-osdu-indexer-i5bpf2gv4iv6ha2xi7rook2rga.us-east-1.es.amazonaws.com";
private static final String DEFAULT_ELASTIC_USER_NAME = "es";
private static final String DEFAULT_ELASTIC_PASSWORD = "us-east-1";
static final int PORT = 9243;
private static final String DEFAULT_INDEXER_HOST = "";
......
......@@ -64,7 +64,7 @@ public class ElasticUtils {
public void createIndex(String index, String mapping) {
try {
try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 1).build();
......@@ -115,7 +115,7 @@ public class ElasticUtils {
BulkResponse bulkResponse = null;
try {
try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("Done creating records inside index with name: " + index);
}
......@@ -129,7 +129,7 @@ public class ElasticUtils {
}
try {
try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
RefreshRequest request = new RefreshRequest(index);
RefreshResponse refreshResponse = client.indices().refresh(request, RequestOptions.DEFAULT);
log.info(String.format("refreshed index, acknowledged shards: %s | failed shards: %s | total shards: %s ", refreshResponse.getSuccessfulShards(), refreshResponse.getFailedShards(), refreshResponse.getTotalShards()));
......@@ -142,7 +142,7 @@ public class ElasticUtils {
}
public void deleteIndex(String index) {
try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
//retry if the elastic cluster is snapshotting and we cant delete it
for (int retries = 0; ; retries++) {
try {
......@@ -178,7 +178,7 @@ public class ElasticUtils {
public long fetchRecords(String index) throws IOException {
try {
try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
SearchRequest request = new SearchRequest(index);
SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
return searchResponse.getHits().totalHits;
......@@ -191,7 +191,7 @@ public class ElasticUtils {
public long fetchRecordsByExistQuery(String index, String attributeName) throws IOException {
try {
try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.existsQuery(attributeName));
......@@ -207,7 +207,7 @@ public class ElasticUtils {
}
public ImmutableOpenMap<String, MappingMetaData> getMapping(String index) throws IOException {
try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
GetMappingsRequest request = new GetMappingsRequest();
request.indices(index);
GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT);
......@@ -217,7 +217,7 @@ public class ElasticUtils {
}
public void refreshIndex(String index) throws IOException {
try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
try {
RefreshRequest request = new RefreshRequest(index);
client.indices().refresh(request, RequestOptions.DEFAULT);
......@@ -256,7 +256,8 @@ public class ElasticUtils {
return dataList;
}
private static RestHighLevelClient createClient(String username, String password, String host) {
protected RestHighLevelClient createClient(String username, String password, String host) {
RestHighLevelClient restHighLevelClient;
int port = Config.PORT;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment