diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc1f2df4a63d5a4f91d0cabb2bc2ea5f435a0b7d..d5af131b2fa20cd1e56d39467ff15305b27c9bd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,9 @@ include: # aws - local: "/devops/osdu/compile/aws.yml" + # anthos + - local: "/devops/osdu/compile/anthos.yml" + # ibm - local: "/devops/osdu/compile/ibm.yml" @@ -63,6 +66,9 @@ include: # aws - local: "/devops/osdu/scanners/abi-check-sdapi-aws.yml" + # anthos + - local: "/devops/osdu/scanners/abi-check-sdapi-anthos.yml" + # gcs - local: "/devops/osdu/scanners/abi-check-sdapi-gcs.yml" @@ -82,6 +88,9 @@ include: # aws - local: "/devops/osdu/deploy/aws.yml" + # anthos + - local: "/devops/osdu/deploy/anthos.yml" + # ibm - local: "/devops/osdu/deploy/ibm.yml" diff --git a/README.md b/README.md index 04d443d5c997d8fd2500053551fe5e98dea012d5..ecc7b7cae833035905ac4772412ba202bd6f3f92 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,9 @@ options: --ibm-provider Also build the optional IBM cloud provider code. (non-functional, based on AWS C++ SDK) + + --anthos-provider + Also build the optional Anthos provider code. (non-functional, based on AWS C++ SDK) ``` ```bash diff --git a/devops/osdu/compile/anthos.yml b/devops/osdu/compile/anthos.yml new file mode 100644 index 0000000000000000000000000000000000000000..f1693621d1634cc36184674b9899032a94753a50 --- /dev/null +++ b/devops/osdu/compile/anthos.yml @@ -0,0 +1,4 @@ +compile-and-unit-test-anthos: + variables: + cloud_provider: anthos-provider + extends: .compile-and-unit-test diff --git a/devops/osdu/compile/polycloud.yml b/devops/osdu/compile/polycloud.yml index 4f4aaeb1b06d65d4c99f5d9541e2c19ec3481380..319ec188a74e55d783ea38ef7cf4775b94a1c766 100644 --- a/devops/osdu/compile/polycloud.yml +++ b/devops/osdu/compile/polycloud.yml @@ -1,5 +1,5 @@ compile-and-unit-test-polycloud: extends: .compile-and-unit-test script: - - ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --build-utest + - ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --anthos-provider --build-utest - ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest \ No newline at end of file diff --git a/devops/osdu/deploy/anthos.yml b/devops/osdu/deploy/anthos.yml new file mode 100644 index 0000000000000000000000000000000000000000..d6d7e06b596e967675efbadd7d336b50edc045a0 --- /dev/null +++ b/devops/osdu/deploy/anthos.yml @@ -0,0 +1,7 @@ +osdu-package-aws: + variables: + cloud_provider: anthos + extends: .osdu-package + dependencies: + - compile-and-unit-test-anthos + - abi-check-for-sdapi-anthos diff --git a/devops/osdu/scanners/abi-check-sdapi-anthos.yml b/devops/osdu/scanners/abi-check-sdapi-anthos.yml new file mode 100644 index 0000000000000000000000000000000000000000..d04eff6d9a7ff88f9910a6c07c59f0bc09e58414 --- /dev/null +++ b/devops/osdu/scanners/abi-check-sdapi-anthos.yml @@ -0,0 +1,8 @@ +abi-check-for-sdapi-anthos: + variables: + cloud_provider: anthos + extends: .abi-check-for-sdapi + needs: ['compile-and-unit-test-anthos'] + # TODO: Delete this after anthos-cpp-lib package apper in Registry + rules: + - allow_failure: true diff --git a/devops/osdu/scanners/sanitizer-global.yml b/devops/osdu/scanners/sanitizer-global.yml index cfcc5f99f072ea7082188d9f0865173189514d31..02e9bfffe8f764573706a8f174b45a4ba10d8448 100644 --- a/devops/osdu/scanners/sanitizer-global.yml +++ b/devops/osdu/scanners/sanitizer-global.yml @@ -4,7 +4,7 @@ image: $CI_REGISTRY_IMAGE/sdapi-osdu-centos7-static-deps before_script: - git submodule update --init --recursive - - ./devops/scripts/build-linux64.sh --build-mode=$build_mode --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --build-utest + - ./devops/scripts/build-linux64.sh --build-mode=$build_mode --azure-provider --azure-curl-provider --gcs-provider --aws-provider --anthos-provider --ibm-provider --build-utest script: - ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest diff --git a/devops/scripts/build-linux64.sh b/devops/scripts/build-linux64.sh index 8a44d1ea0e768267d4c347065eaebae2d6a7a4f9..d9e6fb668c28c9735a8b82ebf214add9a5adb573 100755 --- a/devops/scripts/build-linux64.sh +++ b/devops/scripts/build-linux64.sh @@ -84,6 +84,14 @@ for i in "$@"; do cloud_providers="${cloud_providers};aws" fi ;; + --anthos-provider) + shift + if [ -z "${cloud_providers}" ]; then + cloud_providers="anthos" + else + cloud_providers="${cloud_providers};anthos" + fi + ;; --ibm-provider) shift if [ -z "${cloud_providers}" ]; then @@ -148,6 +156,9 @@ options: --aws-provider Also build the AWS optional cloud provider code. ( non-functional, based on AWS C++ SDK ) + --anthos-provider + Also build the ANTHOS optional cloud provider code. ( non-functional, based on AWS C++ SDK ) + --ibm-provider Also build the IBM optional cloud provider code. ( non-functional, based on AWS C++ SDK ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac74254b659e9906f6e9adc2d146ca18bdb17883..4de5d2dff7fba70bb3b49114bb795b24e64e2a29 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -132,6 +132,7 @@ endif() option( AZURE_PROVIDER_ENABLED OFF ) option( AZURE_CURL_PROVIDER_ENABLED OFF) option( AWS_PROVIDER_ENABLED OFF ) +option( ANTHOS_PROVIDER_ENABLED OFF ) option( IBM_PROVIDER_ENABLED OFF ) option( GCS_PROVIDER_ENABLED OFF ) @@ -167,6 +168,13 @@ else () message( STATUS "NOT BUILDING AWS PROVIDER" ) endif() +if( "anthos" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED ) + set( ANTHOS_PROVIDER_ENABLED ON) + add_definitions( -DHAS_ANTHOS_STORAGE_PROVIDER ) +else () + message( STATUS "NOT BUILDING ANTHOS PROVIDER" ) +endif() + if( "ibm" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED ) set( IBM_PROVIDER_ENABLED ON) add_definitions( -DHAS_IBM_STORAGE_PROVIDER ) @@ -178,6 +186,7 @@ message( STATUS "AZURE_PROVIDER_ENABLED=${AZURE_PROVIDER_ENABLED}" ) message( STATUS "AZURE_CURL_PROVIDER_ENABLED=${AZURE_CURL_PROVIDER_ENABLED}" ) message( STATUS "AWS_PROVIDER_ENABLED=${AWS_PROVIDER_ENABLED}" ) message( STATUS "IBM_PROVIDER_ENABLED=${IBM_PROVIDER_ENABLED}" ) +message( STATUS "ANTHOS_PROVIDER_ENABLED=${ANTHOS_PROVIDER_ENABLED}" ) message( STATUS "GCS_PROVIDER_ENABLED=${GCS_PROVIDER_ENABLED}" ) # set polycloud definition @@ -239,6 +248,13 @@ if (IBM_PROVIDER_ENABLED) list(APPEND SRC_LIB_PROVIDERS_IBM "${SRC_LIB_PROVIDERS_IBM_EXTRA}") endif() endif() +if (ANTHOS_PROVIDER_ENABLED) + file(GLOB SRC_LIB_PROVIDERS_ANTHOS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/anthos/*.cc) + if (NOT AWS_PROVIDER_ENABLED) + file(GLOB SRC_LIB_PROVIDERS_ANTHOS_EXTRA ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/aws/*.cc) + list(APPEND SRC_LIB_PROVIDERS_ANTHOS "${SRC_LIB_PROVIDERS_ANTHOS_EXTRA}") + endif() +endif() if (GCS_PROVIDER_ENABLED) file(GLOB SRC_LIB_PROVIDERS_GCS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/gcs/*.cc) endif() @@ -375,6 +391,12 @@ if ( IBM_PROVIDER_ENABLED ) endif() endif() +if ( ANTHOS_PROVIDER_ENABLED ) + find_package( AWSSDK REQUIRED COMPONENTS s3 transfer) + if ( AWSSDK_FOUND ) + set(SOURCES ${SOURCES} ${SRC_LIB_PROVIDERS_ANTHOS} ) + endif() +endif() if (GCS_PROVIDER_ENABLED) # Google cloud storage c++ client library @@ -463,6 +485,11 @@ if(NOT WIN32) target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) endif() + if( ANTHOS_PROVIDER_ENABLED ) + target_include_directories( sdapi PRIVATE ${AWSSDK_INCLUDE_DIR} ) + target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) + endif() + if( GCS_PROVIDER_ENABLED ) target_link_libraries( sdapi PRIVATE storage_client ) endif() @@ -497,6 +524,12 @@ else() # WIN32 static lib message( STATUS "[IBM Test] - WIN32 flow:target_include_directories:target_link_libraries") endif() + if( ANTHOS_PROVIDER_ENABLED ) + target_include_directories( sdapi PRIVATE ${AWSSDK_INCLUDE_DIR} ) + target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) + message( STATUS "[ANTHOS Test] - WIN32 flow:target_include_directories:target_link_libraries") + endif() + endif() if(NOT WIN32) @@ -602,6 +635,11 @@ if(NOT WIN32) target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) endif() + if( ANTHOS_PROVIDER_ENABLED ) + target_include_directories( sdapi_static PRIVATE ${AWSSDK_INCLUDE_DIR} ) + target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) + endif() + if (GCS_PROVIDER_ENABLED) target_link_libraries(sdapi_static PRIVATE storage_client) endif() @@ -635,6 +673,12 @@ else() message( STATUS "[IBM Test] - WIN32 flow:target_include_directories:target_link_libraries") endif() + if( ANTHOS_PROVIDER_ENABLED ) + target_include_directories( sdapi_static PRIVATE ${AWSSDK_INCLUDE_DIR} ) + target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) + message( STATUS "[Anthos Test] - WIN32 flow:target_include_directories:target_link_libraries") + endif() + SET(S "sdapi") if (DEFINED LIB_VERSION_ON_NAME) SET(V "_${LIB_VERSION_MAJOR}") diff --git a/src/src/lib/cloud/Storage.cc b/src/src/lib/cloud/Storage.cc index 05d60d624d5c60cb87508e9aeff136795da1dbec..5814e8b56ba62a5f79fbeca966c255a882a07133 100644 --- a/src/src/lib/cloud/Storage.cc +++ b/src/src/lib/cloud/Storage.cc @@ -36,6 +36,10 @@ #include "cloud/providers/aws/AwsStorage.h" #endif +#ifdef HAS_ANTHOS_STORAGE_PROVIDER + #include "cloud/providers/anthos/AnthosStorage.h" +#endif + #ifdef HAS_IBM_STORAGE_PROVIDER #include "cloud/providers/ibm/IbmStorage.h" #endif @@ -47,6 +51,7 @@ static const std::string kAzureCloudProvider{"azure"}; static const std::string kAwsCloudProvider{"aws"}; static const std::string kIbmCloudProvider{"ibm"}; +static const std::string kAnthosCloudProvider{"anthos"}; static const bool UseAzureRest = seismicdrive::sdutils::getSwitchEnv("SDAPI_USE_OUTDATED_AZURE_REST"), UseGcsSdk = seismicdrive::sdutils::getSwitchEnv("SDAPI_USE_GCS_SDK"); @@ -137,11 +142,18 @@ namespace seismicdrive return new cpprest::AzureStorage(authProvider, sdResource, sdReadOnly); #endif } + else if (cloudProvider == kAnthosCloudProvider) + { + #ifdef HAS_ANTHOS_STORAGE_PROVIDER + return new AnthosStorage(authProvider, sdResource, sdReadOnly); + #endif + } else if (cloudProvider == kAwsCloudProvider) { #ifdef HAS_AWS_S3_STORAGE_PROVIDER return new AwsStorage(authProvider, sdResource, sdReadOnly); #endif + } else if (cloudProvider == kIbmCloudProvider) { diff --git a/src/src/lib/cloud/Storage.h b/src/src/lib/cloud/Storage.h index d70e7985d3d20124fa85f9920d19b879e5a61cbf..13ae28bb36ed548107ccb17173d4e5da180b7fd0 100644 --- a/src/src/lib/cloud/Storage.h +++ b/src/src/lib/cloud/Storage.h @@ -46,6 +46,9 @@ namespace seismicdrive #ifdef HAS_IBM_STORAGE_PROVIDER class IbmStorage; #endif +#ifdef HAS_ANTHOS_STORAGE_PROVIDER + class AnthosStorage; +#endif #ifdef HAS_GCS_STORAGE_PROVIDER class GcsStorage; #endif diff --git a/src/src/lib/cloud/providers/anthos/AnthosStorage.cc b/src/src/lib/cloud/providers/anthos/AnthosStorage.cc new file mode 100644 index 0000000000000000000000000000000000000000..fa55b7963b7b3a8a579695504194d3cc790b67f7 --- /dev/null +++ b/src/src/lib/cloud/providers/anthos/AnthosStorage.cc @@ -0,0 +1,13 @@ +#include "shared/utils.h" + +#include "AnthosStorage.h" + +namespace seismicdrive +{ + AnthosStorage::AnthosStorage(AuthProvider *auth_provider, const std::string &sdresource, bool readonly) + : AwsStorage(auth_provider, sdresource, readonly) + { + endpoint_override = sdutils::getStringEnv("S3_ENDPOINT_OVERRIDE"); + } + +} diff --git a/src/src/lib/cloud/providers/anthos/AnthosStorage.h b/src/src/lib/cloud/providers/anthos/AnthosStorage.h new file mode 100644 index 0000000000000000000000000000000000000000..9ed81dd7c30ef6bec9337a12b73b051c3304fc9a --- /dev/null +++ b/src/src/lib/cloud/providers/anthos/AnthosStorage.h @@ -0,0 +1,11 @@ +#include "../aws/AwsStorage.h" + +namespace seismicdrive +{ + class AnthosStorage : public AwsStorage + { + public: + AnthosStorage(AuthProvider *auth_provider, const std::string &sdresource = "", bool readonly = false); + + }; +} \ No newline at end of file diff --git a/src/test/seismic-store-lib/CMakeLists.txt b/src/test/seismic-store-lib/CMakeLists.txt index cb72a4684d4cab62e5973fc06a9b7881123e134a..dd79a29467c795431aaa7bd59e5230f0d3c41e66 100644 --- a/src/test/seismic-store-lib/CMakeLists.txt +++ b/src/test/seismic-store-lib/CMakeLists.txt @@ -48,6 +48,29 @@ if (IBM_PROVIDER_ENABLED) endif(IBM_PROVIDER_ENABLED) +if (ANTHOS_PROVIDER_ENABLED) + + find_package(Crc32c) + set(SOURCES anthos_storage.cc + ../shared/utils.cc) + + add_executable(anthos_storage ${SOURCES}) + + target_include_directories(anthos_storage PRIVATE ${AWSSDK_INCLUDE_DIR} Crc32c::crc32c) + + # explictly remove all other providers from this test... + # this is a hack, I should be retrieving and removing all the -D from the target compile definitions + target_compile_options(anthos_storage PRIVATE -UHAS_GCS_STORAGE_PROVIDER -UHAS_AZURE_BLOB_STORAGE_PROVIDER_CURL -UHAS_AZURE_BLOB_STORAGE_PROVIDER -UHAS_AWS_S3_STORAGE_PROVIDER) + target_link_libraries(anthos_storage ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} + ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CMAKE_DL_LIBS} + Crc32c::crc32c) + + add_executable(anthos_storage_static anthos_storage.cc) + target_link_libraries(anthos_storage_static ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) + +endif(ANTHOS_PROVIDER_ENABLED) + + # AZURE STORAGE TEST if (AZURE_PROVIDER_ENABLED OR AZURE_CURL_PROVIDER_ENABLED) set(SOURCES azure_storage.cc ../shared/utils.cc)