GONRG-540 Integration tests for File

Added storage cleaning after tests

testing/file-test-core/pom.xml

 <!--
   ~ Copyright 2020 Google LLC
+  ~ Copyright 2020 EPAM Systems, Inc
   ~
   ~ Licensed under the Apache License, Version 2.0 (the "License");
   ~ you may not use this file except in compliance with the License.
@@ -37,6 +38,7 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <maven.compiler.source>1.8</maven.compiler.source>
     <spring-boot.version>2.2.2.RELEASE</spring-boot.version>
+    <os-core-common.version>0.3.5</os-core-common.version>
   </properties>
 
   <dependencyManagement>
@@ -53,10 +55,25 @@
 
   <dependencies>
     <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
+      <groupId>org.opengroup.osdu</groupId>
+      <artifactId>os-core-common</artifactId>
+      <version>${os-core-common.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-client</artifactId>
+      <version>1.19.4</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.12</version>
+      <scope>test</scope>
     </dependency>
-
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter</artifactId>

testing/file-test-core/src/main/java/org/opengroup/osdu/file/HttpClient.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.file;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import java.io.IOException;
+import java.net.URL;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.util.Map;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import javax.ws.rs.core.MediaType;
+import org.opengroup.osdu.file.apitest.Config;
+
+public abstract class HttpClient {
+
+  protected static String accessToken;
+  protected static String noDataAccessToken;
+
+  public abstract String getAccessToken() throws IOException;
+
+  public abstract String getNoDataAccessToken() throws IOException;
+
+  public ClientResponse send(String path, String httpMethod, Map<String, String> headers,
+      String requestBody)
+      throws Exception {
+
+    Client client = this.getClient();
+
+    String mergedURL = new URL(Config.getFileServiceHost() + path).toString();
+    System.out.println(String.format("calling %s API:%s", httpMethod, mergedURL));
+    System.out.println(String.format("request body:%s", requestBody));
+
+    if (requestBody != null) {
+      headers.put("Content-Length", Long.toString(requestBody.length()));
+    } else {
+      headers.put("Content-Length", "0");
+    }
+
+    WebResource webResource = client.resource(mergedURL);
+
+    WebResource.Builder builder = webResource.accept(MediaType.APPLICATION_JSON)
+        .type(MediaType.APPLICATION_JSON);
+    headers.forEach(builder::header);
+
+    return builder.method(httpMethod, ClientResponse.class, requestBody);
+  }
+
+
+  private Client getClient() throws Exception {
+    TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
+      @Override
+      public X509Certificate[] getAcceptedIssuers() {
+        return null;
+      }
+
+      @Override
+      public void checkClientTrusted(X509Certificate[] certs, String authType) {
+      }
+
+      @Override
+      public void checkServerTrusted(X509Certificate[] certs, String authType) {
+      }
+    }};
+
+    try {
+      SSLContext sc = SSLContext.getInstance("TLS");
+      sc.init(null, trustAllCerts, new SecureRandom());
+      HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+    } catch (Exception e) {
+      throw new Exception();
+    }
+
+    return Client.create();
+  }
+
+}

testing/file-test-core/src/main/java/org/opengroup/osdu/file/ReplaceCamelCase.java

@@ -21,8 +21,8 @@ import org.apache.commons.lang3.StringUtils;
 import org.junit.jupiter.api.DisplayNameGenerator;
 
 /**
- * JUnit display name generator. It replaces a "camelCase" name
- * by "Capitalized Whitespace Separated" name.
+ * JUnit display name generator. It replaces a "camelCase" name by "Capitalized Whitespace
+ * Separated" name.
  */
 public class ReplaceCamelCase extends DisplayNameGenerator.Standard {
 

testing/file-test-core/src/main/java/org/opengroup/osdu/file/TestBase.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.file;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.opengroup.osdu.file.apitest.Config;
+import org.opengroup.osdu.file.util.CloudStorageUtil;
+
+public abstract class TestBase {
+
+  protected static HttpClient client;
+
+  protected static CloudStorageUtil cloudStorageUtil;
+
+  public static Map<String, String> getCommonHeader() throws IOException {
+    return getHeaders(Config.getDataPartitionId(), client.getAccessToken());
+  }
+
+  public static Map<String, String> getHeaders(String dataPartition, String token) {
+    Map<String, String> headers = new HashMap<>();
+    if (dataPartition != null && !dataPartition.isEmpty()) {
+      headers.put("Data-Partition-Id", dataPartition);
+    }
+    if (token != null && !token.isEmpty()) {
+      headers.put("Authorization", token);
+    }
+    return headers;
+  }
+
+}

testing/file-test-core/src/main/java/org/opengroup/osdu/file/apitest/Config.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.file.apitest;
+
+import java.util.TimeZone;
+
+public class Config {
+
+  private static final String FILE_SERVICE_HOST = "";
+  private static final String INTEGRATION_TESTER = "";
+  private static final String NO_DATA_ACCESS_TESTER = "";
+  private static final String TARGET_AUDIENCE = "";
+  private static final String DATA_PARTITION_ID = "";
+  private static final String USER_ID = "";
+  //  Storage time zone id, example = UTC+0
+  private static final String TIME_ZONE = TimeZone.getDefault().getID();
+
+  public static String getFileServiceHost() {
+    return getEnvironmentVariableOrDefaultValue("FILE_SERVICE_HOST", FILE_SERVICE_HOST);
+  }
+
+  public static String getIntegrationTester() {
+    return getEnvironmentVariableOrDefaultValue("INTEGRATION_TESTER", INTEGRATION_TESTER);
+  }
+
+  public static String getNoAccessTester() {
+    return getEnvironmentVariableOrDefaultValue("NO_DATA_ACCESS_TESTER", NO_DATA_ACCESS_TESTER);
+  }
+
+  public static String getTargetAudience() {
+    return getEnvironmentVariableOrDefaultValue("TARGET_AUDIENCE", TARGET_AUDIENCE);
+  }
+
+  public static String getDataPartitionId() {
+    return getEnvironmentVariableOrDefaultValue("DATA_PARTITION_ID", DATA_PARTITION_ID);
+  }
+
+  public static String getUserId() {
+    return getEnvironmentVariableOrDefaultValue("USER_ID", USER_ID);
+  }
+
+  public static String getTimeZone() {
+    return getEnvironmentVariableOrDefaultValue("TIME_ZONE", TIME_ZONE);
+  }
+
+  private static String getEnvironmentVariableOrDefaultValue(String key, String defaultValue) {
+    String environmentVariable = getEnvironmentVariable(key);
+    if (environmentVariable == null) {
+      environmentVariable = defaultValue;
+    }
+    return environmentVariable;
+  }
+
+  private static String getEnvironmentVariable(String propertyKey) {
+    return System.getProperty(propertyKey, System.getenv(propertyKey));
+  }
+
+}

testing/file-test-core/src/main/java/org/opengroup/osdu/file/apitest/File.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.file.apitest;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.sun.jersey.api.client.ClientResponse;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.http.HttpStatus;
+import org.junit.jupiter.api.Test;
+import org.opengroup.osdu.core.common.model.file.FileListResponse;
+import org.opengroup.osdu.core.common.model.file.FileLocation;
+import org.opengroup.osdu.core.common.model.file.FileLocationResponse;
+import org.opengroup.osdu.core.common.model.file.LocationResponse;
+import org.opengroup.osdu.file.TestBase;
+import org.opengroup.osdu.file.util.FileUtils;
+
+
+public abstract class File extends TestBase {
+
+  protected static final String getLocation = "/getLocation";
+  protected static final String getFileLocation = "/getFileLocation";
+  protected static final String getFileList = "/getFileList";
+
+  protected static List<LocationResponse> locationResponses = new ArrayList<>();
+  protected static ObjectMapper mapper = new ObjectMapper();
+
+  @Test
+  public void shouldReturnNotEmptyFileIdAndSignedURL_whenGetLocationWithoutBody()
+      throws Exception {
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getCommonHeader(),
+        "{}");
+
+    LocationResponse locationResponse = mapper
+        .readValue(getLocationResponse.getEntity(String.class), LocationResponse.class);
+
+    assertFalse(locationResponse.getFileID().isEmpty() && locationResponse.getLocation().isEmpty());
+    locationResponses.add(locationResponse);
+  }
+
+  @Test
+  public void shouldReturnSignedUrlAndSameFileId_whenGetLocationWithBody() throws Exception {
+    String fileID = FileUtils.generateUniqueFileID();
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getCommonHeader(),
+        FileUtils.generateFileRequestBody(fileID)
+    );
+
+    LocationResponse locationResponse = mapper
+        .readValue(getLocationResponse.getEntity(String.class), LocationResponse.class);
+
+    assertFalse(locationResponse.getLocation().isEmpty());
+    assertEquals(fileID, locationResponse.getFileID());
+    locationResponses.add(locationResponse);
+  }
+
+  @Test
+  public void first_getLocation_then_GetFileLocationShouldReturnUnsignedUrl() throws Exception {
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getCommonHeader(),
+        FileUtils.generateFileRequestBody(FileUtils.generateUniqueFileID())
+    );
+
+    LocationResponse locationResponse = mapper
+        .readValue(getLocationResponse.getEntity(String.class), LocationResponse.class);
+
+    ClientResponse getFileLocationResponse = client.send(
+        getFileLocation,
+        "POST",
+        getCommonHeader(),
+        FileUtils.generateFileRequestBody(locationResponse.getFileID()));
+
+    FileLocationResponse fileLocationResponse = mapper
+        .readValue(getFileLocationResponse.getEntity(String.class), FileLocationResponse.class);
+
+    assertNotEquals(HttpStatus.SC_BAD_REQUEST, getFileLocationResponse.getStatus());
+    assertFalse(fileLocationResponse.getLocation().isEmpty());
+    locationResponses.add(locationResponse);
+  }
+
+  @Test
+  public void shouldReturnBadRequest_whenGetNonExistingFileLocation() throws Exception {
+    ClientResponse getFileLocationResponse = client.send(
+        getFileLocation,
+        "POST",
+        getCommonHeader(),
+        FileUtils.generateFileRequestBody(FileUtils.generateUniqueFileID()));
+
+    assertEquals(HttpStatus.SC_BAD_REQUEST, getFileLocationResponse.getStatus());
+  }
+
+  @Test
+  public void shouldReturnBadRequest_whenGivenNotValidFileId() throws Exception {
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getCommonHeader(),
+        FileUtils.generateFileRequestBody("/" + FileUtils.generateUniqueFileID() + "/")
+    );
+    assertEquals(HttpStatus.SC_BAD_REQUEST, getLocationResponse.getStatus());
+  }
+
+  @Test
+  public void first_getLocation_then_shouldReturnFileList_sameFileId() throws Exception {
+    LocalDateTime from = LocalDateTime.now(ZoneId.of(Config.getTimeZone()));
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getCommonHeader(),
+        "{}");
+
+    LocationResponse locationResponse = mapper
+        .readValue(getLocationResponse.getEntity(String.class), LocationResponse.class);
+
+    LocalDateTime to = LocalDateTime.now(ZoneId.of(Config.getTimeZone()));
+
+    String fileListRequestBody = FileUtils.generateFileListRequestBody(from, to, 0, (short) 1);
+
+    ClientResponse fileListResponse = client.send(
+        getFileList,
+        "POST",
+        getCommonHeader(),
+        fileListRequestBody
+    );
+    try {
+      FileListResponse fileList = mapper
+          .readValue(fileListResponse.getEntity(String.class), FileListResponse.class);
+
+      assertFalse(fileList.getContent().isEmpty());
+      FileLocation fileLocation = fileList.getContent().get(0);
+      assertEquals(fileLocation.getFileID(), locationResponse.getFileID());
+    } catch (Throwable e) {
+      locationResponses.add(locationResponse);
+      throw new Exception(String
+          .format("%s request failed, with body %s , make sure that TIME_ZONE = %s is correct",
+              getFileList, fileListRequestBody, Config.getTimeZone()), e);
+    }
+    locationResponses.add(locationResponse);
+  }
+
+  @Test
+  public void shouldReturnBadRequest_whenGetFileListInvalidRequest() throws Exception {
+    LocalDateTime now = LocalDateTime.now(ZoneId.of(Config.getTimeZone()));
+    ClientResponse fileListResponse = client.send(
+        getFileList,
+        "POST",
+        getCommonHeader(),
+        FileUtils.generateFileListRequestBody(now.plusHours(1), now, 0, (short) -1)
+    );
+    assertEquals(HttpStatus.SC_BAD_REQUEST, fileListResponse.getStatus());
+  }
+
+  @Test
+  public void shouldReturnUnauthorized_whenGivenInvalidPartitionId() throws Exception {
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getHeaders("invalid_partition", client.getAccessToken()),
+        "{}");
+    assertEquals(HttpStatus.SC_UNAUTHORIZED, getLocationResponse.getStatus());
+  }
+
+  @Test
+  public void shouldReturnUnauthorized_whenPartitionIdNotGiven() throws Exception {
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getHeaders(null, client.getAccessToken()),
+        "{}");
+    assertEquals(HttpStatus.SC_UNAUTHORIZED, getLocationResponse.getStatus());
+  }
+
+  @Test
+  public void shouldReturnUnauthorized_whenGivenAnonimus() throws Exception {
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getHeaders(Config.getDataPartitionId(), null),
+        "{}");
+    assertEquals(HttpStatus.SC_UNAUTHORIZED, getLocationResponse.getStatus());
+  }
+
+  @Test
+  public void getLocationShouldReturnForbidden_whenGivenNoDataAccess() throws Exception {
+    ClientResponse getLocationResponse = client.send(
+        getLocation,
+        "POST",
+        getHeaders(Config.getDataPartitionId(), client.getNoDataAccessToken()),
+        "{}");
+    assertEquals(HttpStatus.SC_FORBIDDEN, getLocationResponse.getStatus());
+  }
+
+}

testing/file-test-core/src/main/java/org/opengroup/osdu/file/util/CloudStorageUtil.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.file.util;
+
+public abstract class CloudStorageUtil {
+
+  public void deleteCloudFile(String bucketName, String fileName) {
+  }
+
+}

testing/file-test-core/src/main/java/org/opengroup/osdu/file/util/FileUtils.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.file.util;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import java.time.LocalDateTime;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.opengroup.osdu.core.common.model.file.FileListRequest;
+import org.opengroup.osdu.core.common.model.file.LocationRequest;
+import org.opengroup.osdu.file.apitest.Config;
+
+public class FileUtils {
+
+  private static final String FILE_ID = "file-integration-test-";
+
+  private static final ObjectMapper mapper = new ObjectMapper();
+
+  static {
+    mapper.registerModule(new JavaTimeModule());
+    mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+  }
+
+  public static String generateUniqueFileID() {
+    return FILE_ID + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
+  }
+
+  public static String generateFileRequestBody(String fileId) throws JsonProcessingException {
+    LocationRequest locationRequest = LocationRequest.builder()
+        .fileID(fileId)
+        .build();
+    return mapper.writerFor(LocationRequest.class)
+        .writeValueAsString(locationRequest);
+  }
+