Skip to content
GitLab
Commit 99108e1d authored by Sid Stenersen's avatar Sid Stenersen
Browse files

Cleanup

parent fb4f5715
Pipeline #38057 failed with stage
in 48 seconds
Hide whitespace changes
Inline Side-by-side
src/src/lib/auth/auth_provider.cc 0 → 100644
View file @ 99108e1d
// ============================================================================
// Copyright 2017-2021, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ============================================================================
#ifdef _MSC_VER
#define NO_DLL_PUBLIC
#endif
#include "auth/auth_provider.h"
//#include "shared/utils.h"
namespace seismicdrive {
AuthProvider::AuthProvider(const std::string &sdurl, const std::string &sdapikey, const std::string &token)
: sdurl(sdurl),
sdapikey(sdapikey),
token(token),
expire(0)
{
}
std::string AuthProvider::getGcsAuthToken(const std::string &sd_suproject_resource,
bool readonly,
GcsAuthProvider &gcs_auth) {
auto acb = [this]() -> std::string {
auto t = getAuthToken();
return t.rfind("Bearer ", 0) == t.npos ? "Bearer " + t : t;
};
gcs_auth.setLogStatus(log.isEnabled());
return gcs_auth.getToken(sdurl, sdapikey, acb, sd_suproject_resource, readonly);
}
} // namespace
src/src/lib/auth/auth_provider.h
View file @ 99108e1d
......@@ -44,6 +44,13 @@ class AuthProvider
bool readonly,
GcsAuthProvider &gcs_auth);
std::string getGcsAuthTokenGap(const std::string &sd_suproject_resource = "",
bool readonly = false)
{
GcsAuthProvider gap;
return getGcsAuthToken(sd_suproject_resource, readonly, gap);
}
virtual void refreshAuthToken(const std::string &/*token*/)
{
throw SDExceptionAuthProviderError("refreshAuthToken() has not been implemented for this AuthProvider.");
......
src/src/lib/auth/callback_auth_provider.cc
View file @ 99108e1d
......@@ -51,24 +51,4 @@ std::string CallbackAuthProvider::getAuthToken(HTTPRequest */*request*/)
return token;
}
// TODO: create auth_provider.cc
AuthProvider::AuthProvider(const std::string &sdurl, const std::string &sdapikey, const std::string &token)
: sdurl(sdurl),
sdapikey(sdapikey),
token(token),
expire(0)
{
}
std::string AuthProvider::getGcsAuthToken(const std::string &sd_suproject_resource,
bool readonly,
GcsAuthProvider &gcs_auth) {
auto acb = [this]() -> std::string {
auto t = getAuthToken();
return t.rfind("Bearer ", 0) == t.npos ? "Bearer " + t : t;
};
gcs_auth.setLogStatus(log.isEnabled());
return gcs_auth.getToken(sdurl, sdapikey, acb, sd_suproject_resource, readonly);
}
} // namespace
src/src/lib/cloud/providers/Storage.cc
View file @ 99108e1d
......@@ -23,7 +23,6 @@
#include "shared/utils.h"
#include <iostream>
#include <regex>
#ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER
#include "cloud/providers/azure/cpprest/AzureStorage.h"
......@@ -336,24 +335,4 @@ void parseAzsPath(const std::string &cs_path, std::string &container, std::strin
}
}
void parseSasUri(const std::string &sasUri,
std::string &accountName,
std::string &containerName,
std::string &sasToken)
{
accountName.clear();
containerName.clear();
sasToken.clear();
const std::regex re("https://(.*).blob.core.windows.net/(.*)\\?(.*)");
std::smatch match;
if(!std::regex_match(sasUri, match, re)) {
throw seismicdrive::SDException("Invalid sas uri");
}
accountName = match[1].str();
containerName = match[2].str();
sasToken = match[3].str();
}
} // namespace seismicdrive
\ No newline at end of file
src/src/lib/cloud/providers/Storage.h
View file @ 99108e1d
......@@ -350,11 +350,6 @@ class Storage
void parseAzsPath(const std::string &cs_path, std::string &container, std::string &block_blob);
void parseSasUri(const std::string &sasUri,
std::string &accountName,
std::string &containerName,
std::string &sasToken);
template<class Catch, class Throw>
void exponentialRetry(const ExponentialRetryBackoffPolicy &retryPolicy,
const std::string operation,
......
src/src/lib/cloud/providers/azure/cpprest/AzureStorage.cc
View file @ 99108e1d
......@@ -335,7 +335,7 @@ CloudBlobClientIF *AzureStorage::getBlobClient() {
std::string accountName,
containerName,
new_sas_token;
parseSasUri(sas_uri, accountName, containerName, new_sas_token);
sdutils::parseSasUri(sas_uri, accountName, containerName, new_sas_token);
if (sasToken.empty() || sasToken != new_sas_token) {
// Re-init the blob client
......
src/src/lib/cloud/providers/azure/curl/AzureStorage.cc
View file @ 99108e1d
......@@ -417,7 +417,7 @@ CloudBlobClientIF *AzureStorage::getBlobClient()
std::string accountName,
containerName,
new_sas_token;
parseSasUri(sas_uri, accountName, containerName, new_sas_token);
sdutils::parseSasUri(sas_uri, accountName, containerName, new_sas_token);
if (sasToken.empty() || sasToken != new_sas_token)
{
......
src/src/lib/cloud/providers/gcp/GcsStorage.cc
View file @ 99108e1d
......@@ -310,7 +310,7 @@ GCSClientIF *GCSStorage::getGCSClient()
std::string accountName;
std::string containerName;
std::string new_sas_token;
parseSasUri(sas_uri, accountName, containerName, new_sas_token);
sdutils::parseSasUri(sas_uri, accountName, containerName, new_sas_token);
// Re-init the blob client if needed
if (sasToken.empty() || sasToken != new_sas_token) {
......
src/src/lib/shared/utils.cc
View file @ 99108e1d
......@@ -23,6 +23,7 @@
#include <algorithm>
#include <chrono>
#include <memory>
#include <regex>
#include <random>
#include <sstream>
#include <unordered_set>
......@@ -223,5 +224,24 @@ std::string toUpper(std::string s) {
#pragma warning (pop)
#endif
void parseSasUri(const std::string &sasUri,
std::string &accountName,
std::string &containerName,
std::string &sasToken)
{
accountName.clear();
containerName.clear();
sasToken.clear();
const std::regex re("https://(.*).blob.core.windows.net/(.*)\\?(.*)");
std::smatch match;
if(!std::regex_match(sasUri, match, re)) {
throw seismicdrive::SDException("Invalid sas uri");
}
accountName = match[1].str();
containerName = match[2].str();
sasToken = match[3].str();
}
}} // namespace
\ No newline at end of file
src/src/lib/shared/utils.h
View file @ 99108e1d
......@@ -87,4 +87,9 @@ inline auto to_string_t(const std::string &s)
return utility::conversions::to_string_t(s);
}
void parseSasUri(const std::string &sasUri,
std::string &accountName,
std::string &containerName,
std::string &sasToken);
}} // namespace
src/test/seismic-store-lib/CMakeLists.txt
View file @ 99108e1d
......@@ -15,12 +15,12 @@
# ============================================================================
# HTTP REQUEST TEST
add_executable(http_request ../../src/lib/http/http_request.cc ../../src/lib/shared/utils.cc ../shared/utils.cc http_request.cc)
add_executable(http_request ../../src/lib/http/http_request.cc ../../src/lib/shared/utils.cc ../../src/core/SDException.cc ../shared/utils.cc http_request.cc)
set_target_properties(http_request PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib")
target_link_libraries(http_request ${CURL_LIBRARY} ${CMAKE_DL_LIBS})
# GCS ACCESSOR TEST
add_executable(gcs_accessor ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp gcs_accessor.cc)
add_executable(gcs_accessor ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/auth_provider.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp gcs_accessor.cc)
add_dependencies(gcs_accessor crc32c)
set_target_properties(gcs_accessor PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib")
target_include_directories(gcs_accessor PRIVATE ${CRC32CLIB_DIR}/include)
......@@ -28,7 +28,7 @@ target_link_libraries(gcs_accessor ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CRC32C_L
# AZURE STORAGE TEST
if(AZURE_PROVIDER_ENABLED)
add_executable(azure_storage ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/lib/cloud/providers/azure/cpprest/AzureStorage.cc ../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/direct_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp azure_storage.cc)
add_executable(azure_storage ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/lib/cloud/providers/Storage.cc ../../src/lib/cloud/providers/azure/cpprest/AzureStorage.cc ../../src/core/SDException.cc ../../src/lib/auth/auth_provider.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/direct_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp azure_storage.cc)
add_dependencies(azure_storage crc32c)
set_target_properties(azure_storage PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib")
target_include_directories(azure_storage PRIVATE ${CRC32CLIB_DIR}/include)
......@@ -75,7 +75,9 @@ if(AZURE_CURL_PROVIDER_ENABLED)
../../src/lib/shared/base64.cc
../../src/lib/shared/utils.cc
../shared/utils.cc
../../src/lib/cloud/providers/Storage.cc
../../src/lib/cloud/providers/azure/curl/AzureStorage.cc
../../src/core/SDException.cc ../../src/lib/auth/auth_provider.cc
../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc
../../src/lib/auth/direct_auth_provider.cc
../../src/lib/auth/service_account.cc
......@@ -116,6 +118,7 @@ if ( AWS_PROVIDER_ENABLED )
../../src/lib/shared/utils.cc
../../src/lib/cloud/providers/aws/AwsStorage.cc
../../src/core/SDException.cc
../../src/lib/auth/auth_provider.cc
../../src/lib/auth/gcs_auth_provider.cc
../../src/lib/auth/direct_auth_provider.cc
../../src/lib/auth/service_account.cc
......@@ -146,6 +149,7 @@ if (GCP_PROVIDER_ENABLED)
../../src/lib/shared/utils.cc
../../src/lib/cloud/providers/gcp/GcsStorage.cc
../../src/core/SDException.cc
../../src/lib/auth/auth_provider.cc
../../src/lib/auth/gcs_auth_provider.cc
../../src/lib/auth/direct_auth_provider.cc
../../src/lib/auth/service_account.cc
......@@ -163,14 +167,14 @@ if (GCP_PROVIDER_ENABLED)
endif(GCP_PROVIDER_ENABLED)
# GCS XML 2 JSON
add_executable(gcs_xml_vs_json ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp gcs_xml_vs_json.cc)
add_executable(gcs_xml_vs_json ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/auth_provider.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp gcs_xml_vs_json.cc)
add_dependencies(gcs_xml_vs_json crc32c)
set_target_properties(gcs_xml_vs_json PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib")
target_include_directories(gcs_xml_vs_json PRIVATE ${CRC32CLIB_DIR}/include)
target_link_libraries(gcs_xml_vs_json ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CRC32C_LIBRARY} ${CMAKE_DL_LIBS})
# GCS BATCH MULTITHREAD
add_executable(gcs_batch_multithread ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp gcs_batch_multithread.cc)
add_executable(gcs_batch_multithread ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/auth_provider.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp gcs_batch_multithread.cc)
add_dependencies(gcs_batch_multithread crc32c)
set_target_properties(gcs_batch_multithread PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib")
if(NOT WIN32)
......@@ -180,6 +184,6 @@ else()
endif()
# AUTH PROVIDER TEST
add_executable(auth_provider ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../../src/lib/shared/utils_ext.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/imp_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp auth_provider.cc)
add_executable(auth_provider ../../src/lib/auth/auth_provider.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../../src/lib/shared/utils_ext.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/imp_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp auth_provider.cc)
set_target_properties(auth_provider PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib")
target_link_libraries(auth_provider ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CMAKE_DL_LIBS})
\ No newline at end of file
src/test/seismic-store-lib/auth_provider.cc
View file @ 99108e1d
......@@ -33,7 +33,7 @@ void test_auth_provider(AuthProvider *auth, const std::string &resource, bool re
std::cout << "Auth Token: ";
std::cout << token << '\n';
std::cout << "Gcs Token: ";
token = auth->getGcsAuthToken(resource, readonly);
token = auth->getGcsAuthTokenGap(resource, readonly);
std::cout << token << '\n';
}
......
src/test/seismic-store-lib/aws_storage.cc
View file @ 99108e1d
......@@ -187,7 +187,7 @@ int main(int /*argc*/, char */*argv*/[])
DirectAuthProvider auth(SDURL, SDAPIKEY, TOKEN);
std::string sdresource = "sd://" + TENANT + "/" + SUBPROJECT + "/" + DATASET;
AwsStorage accessor((void *)&auth, sdresource);
const std::string sas_uri = auth.getGcsAuthToken(sdresource, false);
const std::string sas_uri = auth.getGcsAuthTokenGap(sdresource, false);
std::cout << "TEST Getting the token\n";
......
src/test/seismic-store-lib/azure_storage.cc
View file @ 99108e1d
......@@ -165,7 +165,7 @@ int main() {
seismicdrive::DirectAuthProvider auth(SDURL, SDAPIKEY, TOKEN);
std::string sdresource = "sd://" + TENANT + '/' + SUBPROJECT + '/' + DATASET;
AzureStorage accessor(&auth, sdresource);
const std::string sas_uri = auth.getGcsAuthToken(sdresource, false);
const std::string sas_uri = auth.getGcsAuthTokenGap(sdresource, false);
const std::regex re("https://(.*).blob.core.windows.net/(.*)\\?(.*)");
std::smatch match;
std::regex_match(sas_uri, match, re);
......
src/test/seismic-store-lib/azure_storage_curl.cc
View file @ 99108e1d
......@@ -163,7 +163,7 @@ int main() {
seismicdrive::DirectAuthProvider auth(SDURL, SDAPIKEY, TOKEN);
std::string sdresource = "sd://" + TENANT + '/' + SUBPROJECT + '/' + DATASET;
AzureStorage accessor(&auth, sdresource);
const std::string sas_uri = auth.getGcsAuthToken(sdresource, false);
const std::string sas_uri = auth.getGcsAuthTokenGap(sdresource, false);
const std::regex re("https://(.*).blob.core.windows.net/(.*)\\?(.*)");
std::smatch match;
std::regex_match(sas_uri, match, re);
......
src/test/seismic-store-lib/gcp_storage.cc
View file @ 99108e1d
......@@ -44,7 +44,7 @@ int main( [[maybe_unused]] int argc, char** argv ){
std::string sdresource = "sd://" + TENANT + "/" + SUBPROJECT + "/" + DATASET;
GCSStorage accessor((void*)&auth, sdresource);
// ?
const std::string sas_uri = auth.getGcsAuthToken(sdresource, false);
const std::string sas_uri = auth.getGcsAuthTokenGap(sdresource, false);
bool exists = accessor.objectExists( sas_uri );
if( exists ){
......
src/test/seismic-store-lib/gcs_xml_vs_json.cc
View file @ 99108e1d
......@@ -71,7 +71,7 @@ int64_t download_json(const std::string &bucket, const std::string &object, APIS
static GoogleAuthProvider KAuthProvider;
http.set_auth_callback([]() -> std::string {
return KAuthProvider.getGcsAuthToken();
return KAuthProvider.getGcsAuthTokenGap();
});
http.set_range(offset, offset + len - 1);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment