Commit 432d9a95 authored by Igor Filippov (EPAM)'s avatar Igor Filippov (EPAM)
Browse files

GONRG-424 add minio implementation for Anthos PoC

parent a2d84f02
......@@ -70,7 +70,8 @@
<module>provider/delivery-ibm</module>
<module>provider/delivery-gcp</module>
<module>provider/delivery-azure</module>
</modules>
<module>provider/delivery-reference</module>
</modules>
<repositories>
<repository>
......
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>os-delivery</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.0.1</version>
<relativePath>../..</relativePath>
</parent>
<artifactId>delivery-reference</artifactId>
<description>Delivery service reference</description>
<version>0.0.1</version>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>delivery-core</artifactId>
<version>0.0.1</version>
</dependency>
<!-- Testing packages -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<version>2.0.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>7.1.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.1.9.RELEASE</version>
<configuration>
<mainClass>org.opengroup.osdu.delivery.DeliveryApplication</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>SNAPSHOT</classifier>
<mainClass>org.opengroup.osdu.delivery.DeliveryApplication</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package org.opengroup.osdu.delivery.provider.reference;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@SpringBootApplication
@ComponentScan({"org.opengroup.osdu"})
public class DeliveryReferenceApplication {
public static void main(String[] args) {
SpringApplication.run(DeliveryReferenceApplication.class, args);
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.provider.reference.cache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.model.entitlements.Groups;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class GroupCache extends RedisCache<String, Groups> {
public GroupCache(
@Value("${gcp.redis.host}") final String redisHost,
@Value("${gcp.redis.port}") final Integer redisPort,
@Value("${gcp.redis.exp.time}") final Integer expTimeSec) {
super(redisHost, redisPort, expTimeSec, String.class, Groups.class);
}
}
package org.opengroup.osdu.delivery.provider.reference.factory;
import io.minio.MinioClient;
import io.minio.errors.InvalidEndpointException;
import io.minio.errors.InvalidPortException;
import javax.annotation.PostConstruct;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@Component
@Lazy
public class CloudObjectStorageFactory {
private static final Logger logger = LoggerFactory.getLogger(CloudObjectStorageFactory.class);
@Value("${minio.endpoint_url}")
private String endpointURL;
@Value("${minio.access_key}")
private String accessKey;
@Value("${minio.secret_key}")
private String secretKey;
@Value("${minio.region:us-east-1}")
private String region;
@Value("${minio.prefix:local-dev}")
private String bucketNamePrefix;
private MinioClient minioClient;
private String bucketName;
public CloudObjectStorageFactory() { }
@PostConstruct
public void init() throws InvalidEndpointException, InvalidPortException {
this.minioClient = new MinioClient(this.endpointURL, this.accessKey, this.secretKey, this.region);
logger.info("Minio client initialized");
}
public MinioClient getClient() {
return this.minioClient;
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.provider.reference.security;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter {
@Override
protected void configure(HttpSecurity http) throws Exception {
http
.httpBasic().disable()
.csrf().disable();
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.provider.reference.security;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
@Controller
public class WhoamiController {
@RequestMapping(value = "/whoami")
@ResponseBody
public String whoami() {
final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
String userName = auth.getName();
String roles = String.valueOf(auth.getAuthorities());
String details = String.valueOf(auth.getPrincipal());
return "user: " + userName + "<BR>" +
"roles: " + roles + "<BR>" +
"details: " + details + "<BR>";
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.provider.reference.service;
import java.time.Instant;
import org.springframework.stereotype.Component;
@Component
public class InstantHelper {
public Instant getCurrentInstant() {
return Instant.now();
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.provider.reference.service;
import io.minio.MinioClient;
import io.minio.errors.ErrorResponseException;
import io.minio.errors.InsufficientDataException;
import io.minio.errors.InternalException;
import io.minio.errors.InvalidBucketNameException;
import io.minio.errors.InvalidExpiresRangeException;
import io.minio.errors.InvalidResponseException;
import io.minio.errors.ServerException;
import io.minio.errors.XmlParserException;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Date;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.delivery.model.SignedUrl;
import org.opengroup.osdu.delivery.provider.reference.factory.CloudObjectStorageFactory;
import org.opengroup.osdu.delivery.provider.interfaces.IStorageService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
@Slf4j
public class StorageServiceImpl implements IStorageService {
@Autowired
private CloudObjectStorageFactory factory;
@Value("${minio.signed-url.expiration-days:1}")
private int signedUrlExpirationTimeInDays;
private InstantHelper instantHelper;
private MinioClient minioClient;
private final static String INVALID_MI_PATH_REASON = "Unsigned url invalid, needs to be full MI path";
private final static String URI_EXCEPTION_REASON = "Exception creating signed url";
private final static String SDK_EXCEPTION_MSG = "There was an error communicating with the SDK request for URL signing.";
@PostConstruct
public void init() {
minioClient = factory.getClient();
instantHelper = new InstantHelper();
}
@Override
public SignedUrl createSignedUrl(String unsignedUrl, String authorizationToken) {
String[] miPathParts = unsignedUrl.split("s3://");
if (miPathParts.length < 2) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), "Malformed URL", INVALID_MI_PATH_REASON);
}
String[] miObjectKeyParts = miPathParts[1].split("/");
if (miObjectKeyParts.length < 1) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), "Malformed URL", INVALID_MI_PATH_REASON);
}
String bucketName = miObjectKeyParts[0];
String key = String.join("/", Arrays.copyOfRange(miObjectKeyParts, 1, miObjectKeyParts.length));
SignedUrl url = new SignedUrl();
try {
URL signedUrl = generateSignedUrl(bucketName, key, "GET");
url.setUri(new URI(signedUrl.toString()));
url.setUrl(signedUrl);
url.setCreatedAt(instantHelper.getCurrentInstant());
} catch (Exception e) {
log.error("There was an error generating the URI.", e);
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Malformed URL", URI_EXCEPTION_REASON, e);
}
return url;
}
private URL generateSignedUrl(String bucketName, String ObjectKey, String httpMethod) {
Date expiration = getExpirationDate();
log.debug("Requesting a signed URL with an expiration of: " + expiration.toString() + " ("
+ signedUrlExpirationTimeInDays + " minutes from now)");
try {
log.debug("creating signed url from minio ");
int expiryTime = 24 * 60 * 60 * signedUrlExpirationTimeInDays;
String url = minioClient.presignedGetObject(bucketName, ObjectKey, expiryTime);
log.debug("url from minio " + url);
return new URL(url);
} catch (InvalidKeyException | ErrorResponseException | IllegalArgumentException | InsufficientDataException
| InternalException | InvalidBucketNameException | InvalidExpiresRangeException | InvalidResponseException
| NoSuchAlgorithmException | XmlParserException | IOException | ServerException e) {
log.error("error creating signed url from minio ", e);
throw new AppException(HttpStatus.SERVICE_UNAVAILABLE.value(), "Remote Service Unavailable",
SDK_EXCEPTION_MSG, e);
}
}
private Date getExpirationDate(){
Date expiration = new Date();
long expTimeMillis = expiration.getTime();
expTimeMillis += 1000 * 60 * 60 * 24 * signedUrlExpirationTimeInDays;
expiration.setTime(expTimeMillis);
return expiration;
}
}
LOG_PREFIX=delivery
logging.level.org.springframework.web=DEBUG
server.servlet.contextPath=/api/delivery/v2/
server.port=8080
gcp.redis.port=6379
gcp.redis.host=localhost
gcp.redis.exp.time=300
minio.signed-url.expiration-days=1
minio.endpoint_url=http://127.0.0.1:9000
minio.access_key=adminadmin
minio.secret_key=adminadmin
minio.region=admin
minio.prefix=local-dev
gcp.search.query.url=localhost/api/search/v2/query
gcp.search.query.limit=1000
gcp.search.query.size=100
gcp.entitlements.url=localhost/entitlements/v1
# TODO: This is a bad practice. We need to clean-up this. ENV style properties should not be used in Spring.
SEARCH_QUERY_RECORD_HOST=${gcp.search.query.url}
SEARCH_QUERY_LIMIT=${gcp.search.query.limit}
SEARCH_BATCH_SIZE=${gcp.search.query.size}
AUTHORIZE_API=${gcp.entitlements.url}
\ No newline at end of file
package org.opengroup.osdu.delivery.provider.reference.service;
import io.minio.MinioClient;
import io.minio.errors.ErrorResponseException;
import io.minio.errors.InsufficientDataException;
import io.minio.errors.InternalException;
import io.minio.errors.InvalidBucketNameException;
import io.minio.errors.InvalidExpiresRangeException;
import io.minio.errors.InvalidResponseException;
import io.minio.errors.ServerException;
import io.minio.errors.XmlParserException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import org.apache.http.HttpStatus;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.delivery.DeliveryApplication;
import org.opengroup.osdu.delivery.model.SignedUrl;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.time.Instant;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes = {DeliveryApplication.class})
public class StorageServiceImplTest {
@Mock
private InstantHelper instantHelper;
@Mock
private MinioClient minioClient;
@InjectMocks
private StorageServiceImpl storageService;
private String bucketName = "osdu-sample-osdu-file";
private String key = "object.csv";
private String unsignedUrl = "s3://" + bucketName + "/" + key;
private String authorizationToken = "123";
@Test
public void createSignedUrl() throws IOException, URISyntaxException, InvalidKeyException,
InvalidResponseException, InsufficientDataException, InvalidExpiresRangeException, ServerException,
InternalException, NoSuchAlgorithmException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
String url = "http://testsignedurl.com";
Mockito.when(minioClient.presignedGetObject(Mockito.any(String.class), Mockito.any(String.class),
Mockito.any(Integer.class))).thenReturn(url);
Instant instant = Instant.now();
Mockito.when(instantHelper.getCurrentInstant()).thenReturn(instant);
SignedUrl expected = new SignedUrl();
expected.setUri(new URI(url));
expected.setUrl(new URL(url));
expected.setCreatedAt(instant);
SignedUrl actual = storageService.createSignedUrl(unsignedUrl, authorizationToken);
Assert.assertEquals(expected, actual);
}
@Test
public void createSignedUrl_malformedUnsignedUrl_throwsAppException() {
try {
String unsignedUrl = "malformedUrlString";
String authorizationToken = "testAuthorizationToken";
storageService.createSignedUrl(unsignedUrl, authorizationToken);
fail("Should not succeed!");
} catch (AppException e) {
assertEquals(HttpStatus.SC_BAD_REQUEST, e.getError().getCode());
assertEquals("Malformed URL", e.getError().getReason());