Commit a405897d authored by Jørgen Lind's avatar Jørgen Lind
Browse files

Merge branch feature/GONRG-61_signed_urls_support with refs/heads/master into...

Merge branch feature/GONRG-61_signed_urls_support with refs/heads/master into refs/merge-requests/239/train
parents dab9b4c4 86f8b68b
Pipeline #9294 passed with stages
in 6 minutes and 8 seconds
......@@ -63,8 +63,24 @@ function(addGoogleCloudToTarget target)
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/bucket_metadata.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/iam_bindings.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/lifecycle_rule.cc
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/client.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/client_options.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/hash_validator.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/object_streambuf.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/logging_client.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/policy_document_request.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/retry_client.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/signed_url_requests.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/object_stream.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/hash_validator_impl.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/iam_policy.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/empty_response.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/service_account.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/logging_resumable_upload_session.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/idempotency_policy.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/internal/backoff_policy.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/retry_object_read_source.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/retry_resumable_upload_session.cc
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
......
......@@ -108,6 +108,31 @@ PyGlobal::initModule(py::module& m)
GoogleCredentialsJson_.def(py::init<const std::string & >(), py::arg("json").none(false), OPENVDS_DOCSTRING(GoogleCredentialsJson_GoogleCredentialsJson));
GoogleCredentialsJson_.def(py::init<std::string && >(), py::arg("json").none(false), OPENVDS_DOCSTRING(GoogleCredentialsJson_GoogleCredentialsJson_2));
// GoogleCredentialsSignedUrl
py::class_<GoogleCredentialsSignedUrl, std::unique_ptr<GoogleCredentialsSignedUrl>>
GoogleCredentialsSignedUrl_(m,"GoogleCredentialsSignedUrl", OPENVDS_DOCSTRING(GoogleCredentialsSignedUrl));
GoogleCredentialsSignedUrl_.def(py::init<const std::string & >(), py::arg("region").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrl_GoogleCredentialsSignedUrl));
GoogleCredentialsSignedUrl_.def(py::init<std::string && >(), py::arg("region").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrl_GoogleCredentialsSignedUrl_2));
// GoogleCredentialsSignedUrlPath
py::class_<GoogleCredentialsSignedUrlPath, std::unique_ptr<GoogleCredentialsSignedUrlPath>>
GoogleCredentialsSignedUrlPath_(m,"GoogleCredentialsSignedUrlPath", OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlPath));
GoogleCredentialsSignedUrlPath_.def(py::init<const std::string &, const std::string &>(), py::arg("region").none(false), py::arg("path").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlPath_GoogleCredentialsSignedUrlPath));
GoogleCredentialsSignedUrlPath_.def(py::init<std::string &&, const std::string &>(), py::arg("region").none(false), py::arg("path").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlPath_GoogleCredentialsSignedUrlPath_2));
GoogleCredentialsSignedUrlPath_.def(py::init<const std::string &, std::string &&>(), py::arg("region").none(false), py::arg("path").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlPath_GoogleCredentialsSignedUrlPath_3));
GoogleCredentialsSignedUrlPath_.def(py::init<std::string &&, std::string &&>(), py::arg("region").none(false), py::arg("path").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlPath_GoogleCredentialsSignedUrlPath_4));
// GoogleCredentialsSignedUrlJson
py::class_<GoogleCredentialsSignedUrlJson, std::unique_ptr<GoogleCredentialsSignedUrlJson>>
GoogleCredentialsSignedUrlJson_(m,"GoogleCredentialsSignedUrlJson", OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlJson));
GoogleCredentialsSignedUrlJson_.def(py::init<const std::string &, const std::string &>(), py::arg("region").none(false), py::arg("json").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlJson_GoogleCredentialsSignedUrlJson));
GoogleCredentialsSignedUrlJson_.def(py::init<std::string &&, const std::string &>(), py::arg("region").none(false), py::arg("json").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlJson_GoogleCredentialsSignedUrlJson_2));
GoogleCredentialsSignedUrlJson_.def(py::init<const std::string &, std::string &&>(), py::arg("region").none(false), py::arg("json").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlJson_GoogleCredentialsSignedUrlJson_3));
GoogleCredentialsSignedUrlJson_.def(py::init<std::string &&, std::string &&>(), py::arg("region").none(false), py::arg("json").none(false), OPENVDS_DOCSTRING(GoogleCredentialsSignedUrlJson_GoogleCredentialsSignedUrlJson_4));
// GoogleOpenOptions
py::class_<GoogleOpenOptions, OpenOptions, std::unique_ptr<GoogleOpenOptions>>
GoogleOpenOptions_(m,"GoogleOpenOptions", OPENVDS_DOCSTRING(GoogleOpenOptions));
......@@ -117,11 +142,15 @@ PyGlobal::initModule(py::module& m)
GoogleOpenOptions_.def(py::init<const std::string &, const std::string &, const native::GoogleCredentialsToken &>(), py::arg("bucket").none(false), py::arg("pathPrefix").none(false), py::arg("credentials").none(false), OPENVDS_DOCSTRING(GoogleOpenOptions_GoogleOpenOptions_3));
GoogleOpenOptions_.def(py::init<const std::string &, const std::string &, const native::GoogleCredentialsPath &>(), py::arg("bucket").none(false), py::arg("pathPrefix").none(false), py::arg("credentials").none(false), OPENVDS_DOCSTRING(GoogleOpenOptions_GoogleOpenOptions_4));
GoogleOpenOptions_.def(py::init<const std::string &, const std::string &, const native::GoogleCredentialsJson &>(), py::arg("bucket").none(false), py::arg("pathPrefix").none(false), py::arg("credentials").none(false), OPENVDS_DOCSTRING(GoogleOpenOptions_GoogleOpenOptions_5));
GoogleOpenOptions_.def(py::init<const std::string &, const std::string &, const native::GoogleCredentialsSignedUrl &>(), py::arg("bucket").none(false), py::arg("pathPrefix").none(false), py::arg("credentials").none(false), OPENVDS_DOCSTRING(GoogleOpenOptions_GoogleOpenOptions_6));
GoogleOpenOptions_.def(py::init<const std::string &, const std::string &, const native::GoogleCredentialsSignedUrlPath &>(), py::arg("bucket").none(false), py::arg("pathPrefix").none(false), py::arg("credentials").none(false), OPENVDS_DOCSTRING(GoogleOpenOptions_GoogleOpenOptions_7));
GoogleOpenOptions_.def(py::init<const std::string &, const std::string &, const native::GoogleCredentialsSignedUrlJson &>(), py::arg("bucket").none(false), py::arg("pathPrefix").none(false), py::arg("credentials").none(false), OPENVDS_DOCSTRING(GoogleOpenOptions_GoogleOpenOptions_8));
GoogleOpenOptions_.def_readwrite("credentialsType" , &GoogleOpenOptions::credentialsType, OPENVDS_DOCSTRING(GoogleOpenOptions_credentialsType));
GoogleOpenOptions_.def_readwrite("bucket" , &GoogleOpenOptions::bucket , OPENVDS_DOCSTRING(GoogleOpenOptions_bucket));
GoogleOpenOptions_.def_readwrite("pathPrefix" , &GoogleOpenOptions::pathPrefix , OPENVDS_DOCSTRING(GoogleOpenOptions_pathPrefix));
GoogleOpenOptions_.def_readwrite("credentials" , &GoogleOpenOptions::credentials, OPENVDS_DOCSTRING(GoogleOpenOptions_credentials));
GoogleOpenOptions_.def_readwrite("storageClass" , &GoogleOpenOptions::storageClass, OPENVDS_DOCSTRING(GoogleOpenOptions_storageClass));
GoogleOpenOptions_.def_readwrite("region" , &GoogleOpenOptions::region , OPENVDS_DOCSTRING(GoogleOpenOptions_region));
py::enum_<GoogleOpenOptions::CredentialsType>
GoogleOpenOptions_CredentialsType_(GoogleOpenOptions_,"CredentialsType", OPENVDS_DOCSTRING(GoogleOpenOptions_CredentialsType));
......@@ -130,6 +159,9 @@ PyGlobal::initModule(py::module& m)
GoogleOpenOptions_CredentialsType_.value("AccessToken" , GoogleOpenOptions::CredentialsType::AccessToken, OPENVDS_DOCSTRING(GoogleOpenOptions_CredentialsType_AccessToken));
GoogleOpenOptions_CredentialsType_.value("JsonPath" , GoogleOpenOptions::CredentialsType::JsonPath, OPENVDS_DOCSTRING(GoogleOpenOptions_CredentialsType_JsonPath));
GoogleOpenOptions_CredentialsType_.value("Json" , GoogleOpenOptions::CredentialsType::Json, OPENVDS_DOCSTRING(GoogleOpenOptions_CredentialsType_Json));
GoogleOpenOptions_CredentialsType_.value("SignedUrl" , GoogleOpenOptions::CredentialsType::SignedUrl, OPENVDS_DOCSTRING(GoogleOpenOptions_CredentialsType_SignedUrl));
GoogleOpenOptions_CredentialsType_.value("SignedUrlJsonPath" , GoogleOpenOptions::CredentialsType::SignedUrlJsonPath, OPENVDS_DOCSTRING(GoogleOpenOptions_CredentialsType_SignedUrlJsonPath));
GoogleOpenOptions_CredentialsType_.value("SignedUrlJson" , GoogleOpenOptions::CredentialsType::SignedUrlJson, OPENVDS_DOCSTRING(GoogleOpenOptions_CredentialsType_SignedUrlJson));
// HttpOpenOptions
py::class_<HttpOpenOptions, OpenOptions, std::unique_ptr<HttpOpenOptions>>
......
......@@ -611,6 +611,75 @@ static const char *__doc_OpenVDS_GoogleCredentialsPath_GoogleCredentialsPath_2 =
static const char *__doc_OpenVDS_GoogleCredentialsPath_path = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrl =
R"doc(Credentials for opening a VDS in Google Cloud Storage by using the
default credentials Using signed URL mechanism)doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlJson =
R"doc(Credentials for opening a VDS in Google Cloud Storage by the string
containing json with credentials Using signed URL mechanism)doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlJson_GoogleCredentialsSignedUrlJson =
R"doc(GoogleCredentialsSignedUrlJson constructor
Parameters:
-----------
region :
The string containing the region required for signature generation
json :
The string containing json with credentials)doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlJson_GoogleCredentialsSignedUrlJson_2 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlJson_GoogleCredentialsSignedUrlJson_3 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlJson_GoogleCredentialsSignedUrlJson_4 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlJson_json = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlJson_region = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlPath =
R"doc(Credentials for opening a VDS in Google Cloud Storage by path to the
service account json file Using signed URL mechanism)doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlPath_GoogleCredentialsSignedUrlPath =
R"doc(GoogleCredentialsSignedUrlPath constructor
Parameters:
-----------
region :
The string containing the region required for signature generation
path :
The path to the service account json file)doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlPath_GoogleCredentialsSignedUrlPath_2 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlPath_GoogleCredentialsSignedUrlPath_3 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlPath_GoogleCredentialsSignedUrlPath_4 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlPath_path = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrlPath_region = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrl_GoogleCredentialsSignedUrl =
R"doc(GoogleCredentialsSignedUrl constructor
Parameters:
-----------
region :
The string containing the region required for signature generation)doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrl_GoogleCredentialsSignedUrl_2 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsSignedUrl_region = R"doc()doc";
static const char *__doc_OpenVDS_GoogleCredentialsToken =
R"doc(Credentials for opening a VDS in Google Cloud Storage by using the
string containing an access token Using OAuth)doc";
......@@ -640,6 +709,12 @@ static const char *__doc_OpenVDS_GoogleOpenOptions_CredentialsType_Json = R"doc(
static const char *__doc_OpenVDS_GoogleOpenOptions_CredentialsType_JsonPath = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_CredentialsType_SignedUrl = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_CredentialsType_SignedUrlJson = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_CredentialsType_SignedUrlJsonPath = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_GoogleOpenOptions = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_GoogleOpenOptions_2 =
......@@ -663,6 +738,12 @@ static const char *__doc_OpenVDS_GoogleOpenOptions_GoogleOpenOptions_4 = R"doc()
static const char *__doc_OpenVDS_GoogleOpenOptions_GoogleOpenOptions_5 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_GoogleOpenOptions_6 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_GoogleOpenOptions_7 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_GoogleOpenOptions_8 = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_bucket = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_credentials = R"doc()doc";
......@@ -671,6 +752,8 @@ static const char *__doc_OpenVDS_GoogleOpenOptions_credentialsType = R"doc()doc"
static const char *__doc_OpenVDS_GoogleOpenOptions_pathPrefix = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_region = R"doc()doc";
static const char *__doc_OpenVDS_GoogleOpenOptions_storageClass = R"doc()doc";
static const char *__doc_OpenVDS_HttpOpenOptions =
......
......@@ -25,21 +25,140 @@
namespace OpenVDS
{
static const std::string GOOGLEAPIS = "https://storage.googleapis.com";
static const std::string HEADER = "HEADER";
static const std::string GET = "GET";
static const std::string PUT = "PUT";
class OAuthTokenCredentials : public google::cloud::storage::v1::oauth2::Credentials
class CredentialManagerGoogle
{
public:
typedef std::pair<const std::string, const std::string> Option;
virtual bool Authorize(
std::string& url,
std::vector<std::string>& headers,
const std::string& verb,
const std::string& bucket_name,
const std::string& path_prefix,
const std::string& object_name,
const Option& option = Option()) = 0;
virtual ~CredentialManagerGoogle() = default;
};
class CredentialManagerGoogleOAuthToken : public CredentialManagerGoogle
{
std::string token;
public:
explicit OAuthTokenCredentials(const std::string& token) : token(token) {}
explicit OAuthTokenCredentials(std::string&& token) : token(std::move(token)) {}
explicit CredentialManagerGoogleOAuthToken(const std::string& token) : token(token) {}
google::cloud::v1::StatusOr<std::string> AuthorizationHeader() override;
bool Authorize(
std::string& url,
std::vector<std::string>& headers,
const std::string& verb,
const std::string& bucket_name,
const std::string& path_prefix,
const std::string& object_name,
const Option& option = Option()) override;
};
google::cloud::v1::StatusOr<std::string> OAuthTokenCredentials::AuthorizationHeader()
bool CredentialManagerGoogleOAuthToken::Authorize(
std::string& url,
std::vector<std::string>& headers,
const std::string& verb,
const std::string& bucket_name,
const std::string& path_prefix,
const std::string& object_name,
const Option& option)
{
return token;
headers.push_back(token);
return true;
}
class CredentialManagerGoogleOAuth2 : public CredentialManagerGoogle
{
typedef std::shared_ptr<google::cloud::storage::v1::oauth2::Credentials> CredentialsPtr;
CredentialsPtr m_credentials;
public:
explicit CredentialManagerGoogleOAuth2(CredentialsPtr&& credentials) : m_credentials(std::move(credentials)) {}
bool Authorize(
std::string& url,
std::vector<std::string>& headers,
const std::string& verb,
const std::string& bucket_name,
const std::string& path_prefix,
const std::string& object_name,
const Option& option = Option()) override;
};
bool CredentialManagerGoogleOAuth2::Authorize(
std::string& url,
std::vector<std::string>& headers,
const std::string& verb,
const std::string& bucket_name,
const std::string& path_prefix,
const std::string& object_name,
const Option& option)
{
auto authorization_header = m_credentials->AuthorizationHeader();
if (!authorization_header) {
return false;
}
headers.push_back(*authorization_header);
return true;
}
class CredentialManagerGoogleSignedUrl : public CredentialManagerGoogle
{
typedef std::shared_ptr<google::cloud::storage::v1::oauth2::Credentials> CredentialsPtr;
const std::chrono::seconds duration = std::chrono::minutes(15);
google::cloud::storage::v1::Client m_client;
public:
explicit CredentialManagerGoogleSignedUrl(CredentialsPtr&& credentials) : m_client(std::move(credentials)) {}
bool Authorize(
std::string& url,
std::vector<std::string>& headers,
const std::string& verb,
const std::string& bucket_name,
const std::string& path_prefix,
const std::string& object_name,
const Option& option = Option()) override;
};
bool CredentialManagerGoogleSignedUrl::Authorize(
std::string& url,
std::vector<std::string>& headers,
const std::string& verb,
const std::string& bucket_name,
const std::string& path_prefix,
const std::string& object_name,
const Option& option)
{
namespace gcs = google::cloud::storage;
std::string object_prefix_name = (path_prefix.empty())
? object_name
: fmt::format("{}/{}", path_prefix, object_name);
auto signed_url = option.first.empty()
? m_client.CreateV4SignedUrl(verb, bucket_name, object_prefix_name, gcs::SignedUrlDuration(duration))
: m_client.CreateV4SignedUrl(verb, bucket_name, object_prefix_name, gcs::SignedUrlDuration(duration),
gcs::AddExtensionHeader(option.first, option.second));
if (!signed_url) {
return false;
}
url = *signed_url;
return true;
}
IOManagerGoogle::IOManagerGoogle(const GoogleOpenOptions& openOptions, Error &error)
......@@ -48,6 +167,7 @@ namespace OpenVDS
, m_bucket(openOptions.bucket)
, m_pathPrefix(openOptions.pathPrefix)
, m_storageClass(openOptions.storageClass)
, m_region(openOptions.region)
{
if (m_bucket.empty())
{
......@@ -65,7 +185,7 @@ namespace OpenVDS
error.string = "Google Cloud Storage Config error. Unable to get Google Default Credentials.";
return;
}
m_credentials = *credentials;
m_credentialsManager.reset(new CredentialManagerGoogleOAuth2(std::move(*credentials)));
}
break;
case GoogleOpenOptions::CredentialsType::AccessToken:
......@@ -74,7 +194,7 @@ namespace OpenVDS
error.string = "Google Cloud Storage Config error. Authorization Token is empty";
return;
}
m_credentials = std::make_shared<OAuthTokenCredentials>(openOptions.credentials);
m_credentialsManager.reset(new CredentialManagerGoogleOAuthToken(openOptions.credentials));
break;
case GoogleOpenOptions::CredentialsType::JsonPath:
{
......@@ -84,7 +204,7 @@ namespace OpenVDS
error.string = "Google Cloud Storage Config error. Unable to create service account credentials fromJson file path.";
return;
}
m_credentials = *credentials;
m_credentialsManager.reset(new CredentialManagerGoogleOAuth2(std::move(*credentials)));
}
break;
case GoogleOpenOptions::CredentialsType::Json:
......@@ -95,7 +215,40 @@ namespace OpenVDS
error.string = "Google Cloud Storage Config error. Unable to create service account credentials from json contents";
return;
}
m_credentials = *credentials;
m_credentialsManager.reset(new CredentialManagerGoogleOAuth2(std::move(*credentials)));
}
break;
case GoogleOpenOptions::CredentialsType::SignedUrl:
{
auto credentials = google::cloud::storage::v1::oauth2::GoogleDefaultCredentials();
if (!credentials) {
error.code = -2;
error.string = "Google Cloud Storage Config error. Unable to get Google Default Credentials.";
return;
}
m_credentialsManager.reset(new CredentialManagerGoogleSignedUrl(std::move(*credentials)));
}
break;
case GoogleOpenOptions::CredentialsType::SignedUrlJsonPath:
{
auto credentials = google::cloud::storage::v1::oauth2::CreateServiceAccountCredentialsFromJsonFilePath(openOptions.credentials);
if (!credentials) {
error.code = -2;
error.string = "Google Cloud Storage Config error. Unable to create service account credentials fromJson file path.";
return;
}
m_credentialsManager.reset(new CredentialManagerGoogleSignedUrl(std::move(*credentials)));
}
break;
case GoogleOpenOptions::CredentialsType::SignedUrlJson:
{
auto credentials = google::cloud::storage::v1::oauth2::CreateServiceAccountCredentialsFromJsonContents(openOptions.credentials);
if (!credentials) {
error.code = -2;
error.string = "Google Cloud Storage Config error. Unable to create service account credentials from json contents";
return;
}
m_credentialsManager.reset(new CredentialManagerGoogleSignedUrl(std::move(*credentials)));
}
break;
}
......@@ -115,15 +268,13 @@ namespace OpenVDS
std::shared_ptr<DownloadRequestCurl> request = std::make_shared<DownloadRequestCurl>(objectName, handler);
std::vector<std::string> headers;
auto authorization_header = m_credentials->AuthorizationHeader();
if (!authorization_header) {
if (!m_credentialsManager->Authorize(url, headers, HEADER, m_bucket, m_pathPrefix, objectName)) {
request->m_done = true;
request->m_cancelled = true;
request->m_error.code = -3;
request->m_error.string = "Google Cloud Storage Config error. Unable to generate Authorization Header.";
return request;
}
headers.push_back(*authorization_header);
m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER);
return request;
}
......@@ -133,20 +284,23 @@ namespace OpenVDS
std::string url = downloadUrl(GOOGLEAPIS, m_bucket, m_pathPrefix, objectName);
std::shared_ptr<DownloadRequestCurl> request = std::make_shared<DownloadRequestCurl>(objectName, handler);
std::vector<std::string> headers;
auto authorization_header = m_credentials->AuthorizationHeader();
if (!authorization_header) {
request->m_done = true;
request->m_cancelled = true;
request->m_error.code = -3;
request->m_error.string = "Google Cloud Storage Config error. Unable to generate Authorization Header.";
return request;
}
headers.push_back(*authorization_header);
std::string option_name;
std::string option_value;
if (range.start != range.end)
{
option_name = "range";
option_value = fmt::format("bytes={}-{}", range.start, range.end);
headers.emplace_back();
auto& header = headers.back();
header = fmt::format("range: bytes={}-{}", range.start, range.end);
header = fmt::format("{}: {}", option_name, option_value);
}
if (!m_credentialsManager->Authorize(url, headers, GET, m_bucket, m_pathPrefix, objectName, std::make_pair(option_name, option_value))) {
request->m_done = true;
request->m_cancelled = true;
request->m_error.code = -3;
request->m_error.string = "Google Cloud Storage Config error. Unable to generate Authorization Header.";
return request;
}
m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::GET);
return request;
......@@ -158,15 +312,6 @@ namespace OpenVDS
std::shared_ptr<UploadRequestCurl> request = std::make_shared<UploadRequestCurl>(objectName, completedCallback);
static const std::string delimiterStr = "foo_openvds_delimiter_baz";
std::vector<std::string> headers;
auto authorization_header = m_credentials->AuthorizationHeader();
if (!authorization_header) {
request->m_done = true;
request->m_cancelled = true;
request->m_error.code = -3;
request->m_error.string = "Google Cloud Storage Config error. Unable to generate Authorization Header.";
return request;
}
headers.push_back(*authorization_header);
headers.push_back(fmt::format("Content-Type: multipart/related; boundary={}", delimiterStr));
Json::Value jsonHeader;
......@@ -202,7 +347,15 @@ namespace OpenVDS
if (!m_storageClass.empty())
{
headers.push_back(fmt::format("x-goog-storage-class: {}", m_storageClass));
headers.push_back(fmt::format("x-goog-storage-class: {}", m_storageClass));
}
if (!m_credentialsManager->Authorize(url, headers, PUT, m_bucket, m_pathPrefix, objectName)) {
request->m_done = true;
request->m_cancelled = true;
request->m_error.code = -3;
request->m_error.string = "Google Cloud Storage Config error. Unable to generate Authorization Header.";
return request;
}
m_curlHandler.addUploadRequest(request, url, headers, true, std::move(upload_buffers), uploadSize);
......
......@@ -26,6 +26,8 @@
namespace OpenVDS
{
class CredentialManagerGoogle;
class IOManagerGoogle : public IOManager
{
public:
......@@ -38,9 +40,11 @@ private:
CurlHandler m_curlHandler;
std::string m_bucket;
std::string m_pathPrefix;
std::shared_ptr<google::cloud::storage::v1::oauth2::Credentials> m_credentials;
std::string m_storageClass;
std::string m_region;
Json::StreamWriterBuilder m_jsonWriterBuilder;
std::unique_ptr<CredentialManagerGoogle> m_credentialsManager;
};
}
......
......@@ -230,6 +230,84 @@ public:
explicit GoogleCredentialsJson(std::string&& json) noexcept : json(std::move(json)) {}
};
/// <summary>
/// Credentials for opening a VDS in Google Cloud Storage
/// by using the default credentials
/// Using signed URL mechanism
/// </summary>
class GoogleCredentialsSignedUrl
{
friend struct GoogleOpenOptions;
std::string region;
public:
/// <summary>
/// GoogleCredentialsSignedUrl constructor
/// </summary>
/// <param name="region">
/// The string containing the region required for signature generation
/// </param>
explicit GoogleCredentialsSignedUrl(std::string const& region) : region(region) {}
explicit GoogleCredentialsSignedUrl(std::string&& region) noexcept : region(std::move(region)) {}
};
/// <summary>
/// Credentials for opening a VDS in Google Cloud Storage
/// by path to the service account json file
/// Using signed URL mechanism
/// </summary>
class GoogleCredentialsSignedUrlPath
{
friend struct GoogleOpenOptions;
std::string region;
std::string path;
public:
/// <summary>
/// GoogleCredentialsSignedUrlPath constructor
/// </summary>
/// <param name="region">
/// The string containing the region required for signature generation
/// </param>
/// <param name="path">
/// The path to the service account json file
/// </param>
explicit GoogleCredentialsSignedUrlPath(std::string const& region, std::string const& path) : region(region), path(path) {}
explicit GoogleCredentialsSignedUrlPath(std::string&& region, std::string const& path) : region(std::move(region)), path(path) {}
explicit GoogleCredentialsSignedUrlPath(std::string const& region, std::string&& path) : region(region), path(std::move(path)) {}
explicit GoogleCredentialsSignedUrlPath(std::string&& region, std::string&& path) noexcept : region(std::move(region)), path(std::move(path)) {}
};
/// <summary>
/// Credentials for opening a VDS in Google Cloud Storage
/// by the string containing json with credentials
/// Using signed URL mechanism