Skip to content
GitLab
Commit 1c0e1be1 authored by Rostislav Dublin (EPAM)'s avatar Rostislav Dublin (EPAM)
Browse files

Merge branch 'feature/GCP(GONRG-1053)'

parents 3b21bfd5 a6d2d8be
Pipeline #14547 passed with stages
in 27 minutes and 21 seconds
Hide whitespace changes
Inline Side-by-side
NOTICE
View file @ 1c0e1be1
......@@ -286,9 +286,9 @@ The following software have components provided under the terms of this license:
- MapStruct Core (from )
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Netty/Buffer (from http://netty.io/)
- Netty/Codec (from )
- Netty/Codec/HTTP (from )
......@@ -361,8 +361,8 @@ The following software have components provided under the terms of this license:
- jackson-databind (from http://github.com/FasterXML/jackson)
- java-cloudant (from https://cloudant.com)
- java-cloudant (from https://cloudant.com)
- javax.inject (from http://code.google.com/p/atinject/)
- javax.inject (from )
- javax.inject (from http://code.google.com/p/atinject/)
- jersey-repackaged-guava (from )
- jose4j (from https://bitbucket.org/b_c/jose4j/)
- lang-mustache (from https://github.com/elastic/elasticsearch)
......@@ -374,9 +374,13 @@ The following software have components provided under the terms of this license:
- powermock-api-support (from )
- powermock-api-support (from )
- powermock-core (from http://www.powermock.org)
- powermock-core (from http://www.powermock.org)
- powermock-module-junit4 (from http://www.powermock.org)
- powermock-module-junit4 (from http://www.powermock.org)
- powermock-module-junit4-common (from )
- powermock-module-junit4-common (from )
- powermock-reflect (from )
- powermock-reflect (from )
- proton-j (from )
- rank-eval (from https://github.com/elastic/elasticsearch)
- rest (from https://github.com/elastic/elasticsearch)
......@@ -599,8 +603,8 @@ The following software have components provided under the terms of this license:
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Netty/Codec/HTTP (from )
- Netty/Common (from )
- Project Lombok (from https://projectlombok.org)
......
delivery-core/src/main/java/org/opengroup/osdu/delivery/model/SignedUrl.java
View file @ 1c0e1be1
......@@ -15,10 +15,7 @@
package org.opengroup.osdu.delivery.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.*;
import java.net.URI;
import java.net.URL;
......@@ -27,6 +24,7 @@ import java.time.Instant;
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode
@Builder
public class SignedUrl {
......
delivery-core/src/main/java/org/opengroup/osdu/delivery/service/LocationServiceImpl.java
View file @ 1c0e1be1
// Copyright © Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* Copyright 2020 Amazon Web Services
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.service;
......@@ -33,41 +37,54 @@ import java.util.Map;
@RequiredArgsConstructor
public class LocationServiceImpl implements ILocationService {
@Inject
private DpsHeaders headers;
final IStorageService storageService;
final ISearchService searchService;
@Inject
private DpsHeaders headers;
final IStorageService storageService;
final ISearchService searchService;
@Override
public UrlSigningResponse getSignedUrlsBySrn(List<String> srns) {
@Override
public UrlSigningResponse getSignedUrlsBySrn(List<String> srns) {
UrlSigningResponse unsignedUrls = searchService.GetUnsignedUrlsBySrn(srns);
UrlSigningResponse unsignedUrls = searchService.GetUnsignedUrlsBySrn(srns);
return getSignedUrls(unsignedUrls);
}
return getSignedUrls(unsignedUrls);
}
private UrlSigningResponse getSignedUrls(UrlSigningResponse unsignedUrls) {
private UrlSigningResponse getSignedUrls(UrlSigningResponse unsignedUrls) {
List<String> unprocessed = unsignedUrls.getUnprocessed();
Map<String, SrnFileData> processed = new HashMap<>();
List<String> unprocessed = unsignedUrls.getUnprocessed();
Map<String, SrnFileData> processed = new HashMap<>();
for (Map.Entry<String, SrnFileData> entry : unsignedUrls.getProcessed().entrySet()) {
for (Map.Entry<String, SrnFileData> entry : unsignedUrls.getProcessed().entrySet()) {
SrnFileData value = entry.getValue();
SrnFileData value = entry.getValue();
SignedUrl signedUrl = storageService.createSignedUrl(entry.getKey(), value.getUnsignedUrl(), headers.getAuthorization());
log.debug("before getSignedUrl for key {} and unsignedUrl {}", entry.getKey(), value.getUnsignedUrl());
SignedUrl signedUrl = storageService.createSignedUrl(entry.getKey(), value.getUnsignedUrl(), headers.getAuthorization());
log.debug("process getSignedUrl. got signedUrl {}", signedUrl);
if(signedUrl != null && signedUrl.getUrl() != null){
if(!(entry.getKey().toLowerCase().contains("ovds"))){
value.setSignedUrl(signedUrl.getUrl().toString());
}
value.setConnectionString(signedUrl.getConnectionString());
processed.put(entry.getKey(), value);
} else {
unprocessed.add(entry.getKey());
}
}
boolean isOvds = entry.getKey().toLowerCase().contains("ovds");
if (signedUrl != null && !isOvds && signedUrl.getUrl() != null) {
log.debug("process signedUrl. processed as \"!ovds && url\"");
value.setSignedUrl(signedUrl.getUrl().toString());
value.setConnectionString(signedUrl.getConnectionString());
processed.put(entry.getKey(), value);
} else if (signedUrl != null && isOvds && signedUrl.getConnectionString() != null) {
log.debug("process signedUrl. processed as \"ovds && connectionString\"");
value.setConnectionString(signedUrl.getConnectionString());
processed.put(entry.getKey(), value);
} else {
log.debug("process signedUrl. unprocessed");
unprocessed.add(entry.getKey());
}
}
return UrlSigningResponse.builder().
return UrlSigningResponse.builder().processed(processed).unprocessed(unprocessed).build();
}
processed(processed).
unprocessed(unprocessed).
build();
}
}
delivery-core/src/test/java/org/opengroup/osdu/delivery/service/LocationServiceImplTest.java
View file @ 1c0e1be1
......@@ -59,21 +59,27 @@ public class LocationServiceImplTest {
String srn1 = "srn:file/csv:7344999246049527:";
String srn2 = "srn:file/csv:69207556434748899880399:";
String srn3 = "srn:file/csv:59158134479121976019:";
String srnOvds = "srn:type:file/ovds:12345::";
String unsignedUrl1 = "http://unsignedurl1.com";
String unsignedUrl2 = "http://unsignedurl2.com";
String unsignedUrl3 = "http://unsignedurl3.com";
String unsignedUrlOvds = "http://unsignedurlOvds.com";
String kind = "opendes:osdu:file:0.0.4";
String kindOvds = "opendes:osdu:file:0.2.0";
String connectionString = "connectionString";
String connectionStringOvds = "connectionStringOvds";
List<String> srns = new ArrayList<>();
srns.add(srn1);
srns.add(srn2);
srns.add(srn3);
srns.add(srnOvds);
Map<String, SrnFileData> processed = new HashMap<>();
processed.put(srn1, new SrnFileData(null, unsignedUrl1, kind, connectionString));
processed.put(srn2, new SrnFileData(null, unsignedUrl2, kind, connectionString));
processed.put(srn3, new SrnFileData(null, unsignedUrl3, kind, connectionString));
processed.put(srnOvds, new SrnFileData(null, unsignedUrlOvds, kindOvds, connectionStringOvds));
List<String> unprocessed = new ArrayList<>();
UrlSigningResponse unsignedUrlsResponse = UrlSigningResponse.builder().processed(processed).unprocessed(unprocessed).build();
......@@ -89,17 +95,22 @@ public class LocationServiceImplTest {
signedUrl2.setUri(new URI(unsignedUrl2));
signedUrl2.setUrl(new URL(unsignedUrl2));
signedUrl2.setCreatedAt(Instant.now());
signedUrl1.setConnectionString(connectionString);
signedUrl2.setConnectionString(connectionString);
SignedUrl signedUrl3 = new SignedUrl();
signedUrl3.setUri(new URI(unsignedUrl3));
signedUrl3.setUrl(new URL(unsignedUrl3));
signedUrl3.setCreatedAt(Instant.now());
signedUrl1.setConnectionString(connectionString);
signedUrl3.setConnectionString(connectionString);
SignedUrl signedUrlOvds = new SignedUrl();
signedUrlOvds.setCreatedAt(Instant.now());
signedUrlOvds.setConnectionString(connectionStringOvds);
Mockito.when(storageService.createSignedUrl(Mockito.eq(srn1), Mockito.eq(unsignedUrl1), Mockito.any())).thenReturn(signedUrl1);
Mockito.when(storageService.createSignedUrl(Mockito.eq(srn2), Mockito.eq(unsignedUrl2), Mockito.any())).thenReturn(signedUrl2);
Mockito.when(storageService.createSignedUrl(Mockito.eq(srn3), Mockito.eq(unsignedUrl3), Mockito.any())).thenReturn(signedUrl3);
Mockito.when(storageService.createSignedUrl(Mockito.eq(srnOvds), Mockito.eq(unsignedUrlOvds), Mockito.any())).thenReturn(signedUrlOvds);
ReflectionTestUtils.setField(CUT, "headers", new DpsHeaders());
......
provider/delivery-gcp/pom.xml
View file @ 1c0e1be1
......@@ -24,7 +24,7 @@
<artifactId>os-delivery</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.0.2-SNAPSHOT</version>
<relativePath>../..</relativePath>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>delivery-gcp</artifactId>
......@@ -63,10 +63,17 @@
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito</artifactId>
<version>1.7.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<version>2.0.2</version>
<version>1.7.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
......
provider/delivery-gcp/src/main/java/org/opengroup/osdu/delivery/provider/gcp/config/StorageConfiguration.java
View file @ 1c0e1be1
......@@ -19,14 +19,17 @@ package org.opengroup.osdu.delivery.provider.gcp.config;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import org.opengroup.osdu.delivery.provider.gcp.config.properties.GcpConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties(GcpConfigurationProperties.class)
public class StorageConfiguration {
@Bean
public Storage storageDev() throws Exception {
public Storage storageDev() {
return StorageOptions.getDefaultInstance().getService();
}
......
provider/delivery-gcp/src/main/java/org/opengroup/osdu/delivery/provider/gcp/config/properties/GcpConfigurationProperties.java 0 → 100644
View file @ 1c0e1be1
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.provider.gcp.config.properties;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties(prefix = "gcp")
@Getter
@Setter
public class GcpConfigurationProperties {
private SignedUrl signedUrl = new SignedUrl();
@Getter
@Setter
public static class SignedUrl {
private int expirationDays = 1;
}
}
provider/delivery-gcp/src/main/java/org/opengroup/osdu/delivery/provider/gcp/service/StorageServiceImpl.java
View file @ 1c0e1be1
......@@ -17,25 +17,27 @@
package org.opengroup.osdu.delivery.provider.gcp.service;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.HttpMethod;
import com.google.cloud.storage.Storage;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.storage.*;
import com.google.cloud.storage.Storage.SignUrlOption;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.delivery.model.SignedUrl;
import org.opengroup.osdu.delivery.provider.gcp.config.properties.GcpConfigurationProperties;
import org.opengroup.osdu.delivery.provider.gcp.service.downscoped.*;
import org.opengroup.osdu.delivery.provider.interfaces.IStorageService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
@Service
......@@ -43,59 +45,84 @@ import java.util.concurrent.TimeUnit;
@RequiredArgsConstructor
public class StorageServiceImpl implements IStorageService {
private final static String URI_EXCEPTION_REASON = "Exception creating signed url";
private final static String INVALID_GS_PATH_REASON = "Unsigned url invalid, needs to be full GS path";
private final static HttpMethod signedUrlMethod = HttpMethod.GET;
private final Storage storage;
private final InstantHelper instantHelper;
@Value("${gcp.signed-url.expiration-days}")
private int gcSignedUrlExpirationTimeInDays;
@Override
public SignedUrl createSignedUrl(String unsignedUrl, String authorizationToken) {
String[] gsPathParts = unsignedUrl.split("gs://");
if (gsPathParts.length < 2) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), "Malformed URL", INVALID_GS_PATH_REASON);
}
String[] gsObjectKeyParts = gsPathParts[1].split("/");
if (gsObjectKeyParts.length < 1) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), "Malformed URL", INVALID_GS_PATH_REASON);
}
String bucketName = gsObjectKeyParts[0];
String filePath = String.join("/", Arrays.copyOfRange(gsObjectKeyParts, 1, gsObjectKeyParts.length));
URL gcSignedUrl = generateSignedGcURL(bucketName, filePath);
try {
return SignedUrl.builder()
.uri(new URI(gcSignedUrl.toString()))
.url(gcSignedUrl)
.createdAt(instantHelper.getCurrentInstant())
.build();
} catch (URISyntaxException e) {
log.error("There was an error generating the URI.", e);
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Malformed URL", URI_EXCEPTION_REASON, e);
}
}
private URL generateSignedGcURL(String bucketName, String filePath) {
BlobId blobId = BlobId.of(bucketName, filePath);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId)
.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE)
.build();
URL url = storage.signUrl(blobInfo,
gcSignedUrlExpirationTimeInDays, TimeUnit.DAYS, SignUrlOption.httpMethod(signedUrlMethod),
SignUrlOption.withV4Signature());
return url;
}
public static final String AVAILABILITY_CONDITION_EXPRESSION =
"resource.name.startsWith('projects/_/buckets/openvds-test-data/objects/<<folder>>/') " +
"|| api.getAttribute('storage.googleapis.com/objectListPrefix', '').startsWith('<<folder>>')";
public static final String MALFORMED_URL = "Malformed URL";
private static final String URI_EXCEPTION_REASON = "Exception creating signed url";
private static final String INVALID_GS_PATH_REASON = "Unsigned url invalid, needs to be full GS path";
private static final HttpMethod signedUrlMethod = HttpMethod.GET;
private final Storage storage;
private final InstantHelper instantHelper;
private final DownScopedCredentialsService downscopedCredentialsService;
private final GcpConfigurationProperties gcpConfigurationProperties;
@Override
public SignedUrl createSignedUrl(String unsignedUrl, String authorizationToken) {
String[] gsPathParts = unsignedUrl.split("gs://");
if (gsPathParts.length < 2) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), MALFORMED_URL, INVALID_GS_PATH_REASON);
}
String[] gsObjectKeyParts = gsPathParts[1].split("/");
if (gsObjectKeyParts.length < 1) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), MALFORMED_URL, INVALID_GS_PATH_REASON);
}
String bucketName = gsObjectKeyParts[0];
String filePath = String.join("/", Arrays.copyOfRange(gsObjectKeyParts, 1, gsObjectKeyParts.length));
BlobId blobId = BlobId.of(bucketName, filePath);
SignedUrl.SignedUrlBuilder builder = SignedUrl.builder().createdAt(instantHelper.getCurrentInstant());
Blob blob = storage.get(blobId);
if (!Objects.isNull(blob)) {
log.debug("resource is a blob. get SignedUrl");
URL gcSignedUrl = generateSignedGcURL(blobId);
try {
builder.uri(new URI(gcSignedUrl.toString())).url(gcSignedUrl);
} catch (URISyntaxException e) {
log.error("There was an error generating the URI.", e);
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, MALFORMED_URL, URI_EXCEPTION_REASON, e);
}
} else {
log.debug("resource is not a blob. assume it is a folder. get DownScoped token");
ServiceAccountCredentials sourceCredentials = (ServiceAccountCredentials) storage.getOptions().getCredentials();
String availabilityConditionExpression = AVAILABILITY_CONDITION_EXPRESSION.replace("<<folder>>", filePath);
AvailabilityCondition ap = new AvailabilityCondition("obj", availabilityConditionExpression);
AccessBoundaryRule abr = new AccessBoundaryRule(
"//storage.googleapis.com/projects/_/buckets/" + bucketName,
Collections.singletonList("inRole:roles/storage.objectViewer"),
ap);
DownScopedOptions downScopedOptions = new DownScopedOptions(Collections.singletonList(abr));
DownScopedCredentials downScopedCredentials =
downscopedCredentialsService.getDownScopedCredentials(sourceCredentials, downScopedOptions);
try {
builder.connectionString(downScopedCredentials.refreshAccessToken().getTokenValue());
} catch (IOException e) {
log.error("There was an error getting the DownScoped token.", e);
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, MALFORMED_URL, URI_EXCEPTION_REASON, e);
}
}
return builder.build();
}
private URL generateSignedGcURL(BlobId blobId) {
BlobInfo blobInfo = BlobInfo.newBuilder(blobId)
.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE)
.build();
return storage.signUrl(blobInfo,
gcpConfigurationProperties.getSignedUrl().getExpirationDays(), TimeUnit.DAYS, SignUrlOption.httpMethod(signedUrlMethod),
SignUrlOption.withV4Signature());
}
}
provider/delivery-gcp/src/main/java/org/opengroup/osdu/delivery/provider/gcp/service/downscoped/AccessBoundary.java 0 → 100644
View file @ 1c0e1be1
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.provider.gcp.service.downscoped;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
import java.util.List;
@RequiredArgsConstructor
@ToString
@Getter
public class AccessBoundary implements Serializable {
private static final long serialVersionUID = 1147589172954290279L;
private final List<AccessBoundaryRule> accessBoundaryRules;
}
provider/delivery-gcp/src/main/java/org/opengroup/osdu/delivery/provider/gcp/service/downscoped/AccessBoundaryRule.java 0 → 100644
View file @ 1c0e1be1
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.delivery.provider.gcp.service.downscoped;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
import java.util.List;
@RequiredArgsConstructor
@ToString
@Getter
public class AccessBoundaryRule implements Serializable {