From 671a31e2920f6c4a4081aba2317324529b767e60 Mon Sep 17 00:00:00 2001 From: "Yan Sushchynski (EPAM)" Date: Fri, 12 Aug 2022 09:57:55 +0000 Subject: [PATCH] feat: Cherry-pick-dec1c2bd-Anthos --- .gitlab-ci.yml | 9 ++++ README.md | 3 ++ devops/osdu/compile/anthos.yml | 4 ++ devops/osdu/compile/polycloud.yml | 2 +- devops/osdu/deploy/anthos.yml | 7 +++ .../osdu/scanners/abi-check-sdapi-anthos.yml | 8 ++++ devops/osdu/scanners/sanitizer-global.yml | 2 +- devops/scripts/build-linux64.sh | 11 +++++ src/CMakeLists.txt | 44 +++++++++++++++++++ src/src/lib/cloud/Storage.cc | 12 +++++ src/src/lib/cloud/Storage.h | 3 ++ .../cloud/providers/anthos/AnthosStorage.cc | 13 ++++++ .../cloud/providers/anthos/AnthosStorage.h | 11 +++++ src/test/seismic-store-lib/CMakeLists.txt | 23 ++++++++++ 14 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 devops/osdu/compile/anthos.yml create mode 100644 devops/osdu/deploy/anthos.yml create mode 100644 devops/osdu/scanners/abi-check-sdapi-anthos.yml create mode 100644 src/src/lib/cloud/providers/anthos/AnthosStorage.cc create mode 100644 src/src/lib/cloud/providers/anthos/AnthosStorage.h diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc1f2df..d5af131 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,9 @@ include: # aws - local: "/devops/osdu/compile/aws.yml" + # anthos + - local: "/devops/osdu/compile/anthos.yml" + # ibm - local: "/devops/osdu/compile/ibm.yml" @@ -63,6 +66,9 @@ include: # aws - local: "/devops/osdu/scanners/abi-check-sdapi-aws.yml" + # anthos + - local: "/devops/osdu/scanners/abi-check-sdapi-anthos.yml" + # gcs - local: "/devops/osdu/scanners/abi-check-sdapi-gcs.yml" @@ -82,6 +88,9 @@ include: # aws - local: "/devops/osdu/deploy/aws.yml" + # anthos + - local: "/devops/osdu/deploy/anthos.yml" + # ibm - local: "/devops/osdu/deploy/ibm.yml" diff --git a/README.md b/README.md index 04d443d..ecc7b7c 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,9 @@ options: --ibm-provider Also build the optional IBM cloud provider code. (non-functional, based on AWS C++ SDK) + + --anthos-provider + Also build the optional Anthos provider code. (non-functional, based on AWS C++ SDK) ``` ```bash diff --git a/devops/osdu/compile/anthos.yml b/devops/osdu/compile/anthos.yml new file mode 100644 index 0000000..f169362 --- /dev/null +++ b/devops/osdu/compile/anthos.yml @@ -0,0 +1,4 @@ +compile-and-unit-test-anthos: + variables: + cloud_provider: anthos-provider + extends: .compile-and-unit-test diff --git a/devops/osdu/compile/polycloud.yml b/devops/osdu/compile/polycloud.yml index 4f4aaeb..319ec18 100644 --- a/devops/osdu/compile/polycloud.yml +++ b/devops/osdu/compile/polycloud.yml @@ -1,5 +1,5 @@ compile-and-unit-test-polycloud: extends: .compile-and-unit-test script: - - ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --build-utest + - ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --anthos-provider --build-utest - ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest \ No newline at end of file diff --git a/devops/osdu/deploy/anthos.yml b/devops/osdu/deploy/anthos.yml new file mode 100644 index 0000000..d6d7e06 --- /dev/null +++ b/devops/osdu/deploy/anthos.yml @@ -0,0 +1,7 @@ +osdu-package-aws: + variables: + cloud_provider: anthos + extends: .osdu-package + dependencies: + - compile-and-unit-test-anthos + - abi-check-for-sdapi-anthos diff --git a/devops/osdu/scanners/abi-check-sdapi-anthos.yml b/devops/osdu/scanners/abi-check-sdapi-anthos.yml new file mode 100644 index 0000000..d04eff6 --- /dev/null +++ b/devops/osdu/scanners/abi-check-sdapi-anthos.yml @@ -0,0 +1,8 @@ +abi-check-for-sdapi-anthos: + variables: + cloud_provider: anthos + extends: .abi-check-for-sdapi + needs: ['compile-and-unit-test-anthos'] + # TODO: Delete this after anthos-cpp-lib package apper in Registry + rules: + - allow_failure: true diff --git a/devops/osdu/scanners/sanitizer-global.yml b/devops/osdu/scanners/sanitizer-global.yml index cfcc5f9..02e9bff 100644 --- a/devops/osdu/scanners/sanitizer-global.yml +++ b/devops/osdu/scanners/sanitizer-global.yml @@ -4,7 +4,7 @@ image: $CI_REGISTRY_IMAGE/sdapi-osdu-centos7-static-deps before_script: - git submodule update --init --recursive - - ./devops/scripts/build-linux64.sh --build-mode=$build_mode --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --build-utest + - ./devops/scripts/build-linux64.sh --build-mode=$build_mode --azure-provider --azure-curl-provider --gcs-provider --aws-provider --anthos-provider --ibm-provider --build-utest script: - ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest diff --git a/devops/scripts/build-linux64.sh b/devops/scripts/build-linux64.sh index 8a44d1e..d9e6fb6 100755 --- a/devops/scripts/build-linux64.sh +++ b/devops/scripts/build-linux64.sh @@ -84,6 +84,14 @@ for i in "$@"; do cloud_providers="${cloud_providers};aws" fi ;; + --anthos-provider) + shift + if [ -z "${cloud_providers}" ]; then + cloud_providers="anthos" + else + cloud_providers="${cloud_providers};anthos" + fi + ;; --ibm-provider) shift if [ -z "${cloud_providers}" ]; then @@ -148,6 +156,9 @@ options: --aws-provider Also build the AWS optional cloud provider code. ( non-functional, based on AWS C++ SDK ) + --anthos-provider + Also build the ANTHOS optional cloud provider code. ( non-functional, based on AWS C++ SDK ) + --ibm-provider Also build the IBM optional cloud provider code. ( non-functional, based on AWS C++ SDK ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac74254..4de5d2d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -132,6 +132,7 @@ endif() option( AZURE_PROVIDER_ENABLED OFF ) option( AZURE_CURL_PROVIDER_ENABLED OFF) option( AWS_PROVIDER_ENABLED OFF ) +option( ANTHOS_PROVIDER_ENABLED OFF ) option( IBM_PROVIDER_ENABLED OFF ) option( GCS_PROVIDER_ENABLED OFF ) @@ -167,6 +168,13 @@ else () message( STATUS "NOT BUILDING AWS PROVIDER" ) endif() +if( "anthos" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED ) + set( ANTHOS_PROVIDER_ENABLED ON) + add_definitions( -DHAS_ANTHOS_STORAGE_PROVIDER ) +else () + message( STATUS "NOT BUILDING ANTHOS PROVIDER" ) +endif() + if( "ibm" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED ) set( IBM_PROVIDER_ENABLED ON) add_definitions( -DHAS_IBM_STORAGE_PROVIDER ) @@ -178,6 +186,7 @@ message( STATUS "AZURE_PROVIDER_ENABLED=${AZURE_PROVIDER_ENABLED}" ) message( STATUS "AZURE_CURL_PROVIDER_ENABLED=${AZURE_CURL_PROVIDER_ENABLED}" ) message( STATUS "AWS_PROVIDER_ENABLED=${AWS_PROVIDER_ENABLED}" ) message( STATUS "IBM_PROVIDER_ENABLED=${IBM_PROVIDER_ENABLED}" ) +message( STATUS "ANTHOS_PROVIDER_ENABLED=${ANTHOS_PROVIDER_ENABLED}" ) message( STATUS "GCS_PROVIDER_ENABLED=${GCS_PROVIDER_ENABLED}" ) # set polycloud definition @@ -239,6 +248,13 @@ if (IBM_PROVIDER_ENABLED) list(APPEND SRC_LIB_PROVIDERS_IBM "${SRC_LIB_PROVIDERS_IBM_EXTRA}") endif() endif() +if (ANTHOS_PROVIDER_ENABLED) + file(GLOB SRC_LIB_PROVIDERS_ANTHOS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/anthos/*.cc) + if (NOT AWS_PROVIDER_ENABLED) + file(GLOB SRC_LIB_PROVIDERS_ANTHOS_EXTRA ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/aws/*.cc) + list(APPEND SRC_LIB_PROVIDERS_ANTHOS "${SRC_LIB_PROVIDERS_ANTHOS_EXTRA}") + endif() +endif() if (GCS_PROVIDER_ENABLED) file(GLOB SRC_LIB_PROVIDERS_GCS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/gcs/*.cc) endif() @@ -375,6 +391,12 @@ if ( IBM_PROVIDER_ENABLED ) endif() endif() +if ( ANTHOS_PROVIDER_ENABLED ) + find_package( AWSSDK REQUIRED COMPONENTS s3 transfer) + if ( AWSSDK_FOUND ) + set(SOURCES ${SOURCES} ${SRC_LIB_PROVIDERS_ANTHOS} ) + endif() +endif() if (GCS_PROVIDER_ENABLED) # Google cloud storage c++ client library @@ -463,6 +485,11 @@ if(NOT WIN32) target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) endif() + if( ANTHOS_PROVIDER_ENABLED ) + target_include_directories( sdapi PRIVATE ${AWSSDK_INCLUDE_DIR} ) + target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) + endif() + if( GCS_PROVIDER_ENABLED ) target_link_libraries( sdapi PRIVATE storage_client ) endif() @@ -497,6 +524,12 @@ else() # WIN32 static lib message( STATUS "[IBM Test] - WIN32 flow:target_include_directories:target_link_libraries") endif() + if( ANTHOS_PROVIDER_ENABLED ) + target_include_directories( sdapi PRIVATE ${AWSSDK_INCLUDE_DIR} ) + target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) + message( STATUS "[ANTHOS Test] - WIN32 flow:target_include_directories:target_link_libraries") + endif() + endif() if(NOT WIN32) @@ -602,6 +635,11 @@ if(NOT WIN32) target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) endif() + if( ANTHOS_PROVIDER_ENABLED ) + target_include_directories( sdapi_static PRIVATE ${AWSSDK_INCLUDE_DIR} ) + target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) + endif() + if (GCS_PROVIDER_ENABLED) target_link_libraries(sdapi_static PRIVATE storage_client) endif() @@ -635,6 +673,12 @@ else() message( STATUS "[IBM Test] - WIN32 flow:target_include_directories:target_link_libraries") endif() + if( ANTHOS_PROVIDER_ENABLED ) + target_include_directories( sdapi_static PRIVATE ${AWSSDK_INCLUDE_DIR} ) + target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} ) + message( STATUS "[Anthos Test] - WIN32 flow:target_include_directories:target_link_libraries") + endif() + SET(S "sdapi") if (DEFINED LIB_VERSION_ON_NAME) SET(V "_${LIB_VERSION_MAJOR}") diff --git a/src/src/lib/cloud/Storage.cc b/src/src/lib/cloud/Storage.cc index 05d60d6..5814e8b 100644 --- a/src/src/lib/cloud/Storage.cc +++ b/src/src/lib/cloud/Storage.cc @@ -36,6 +36,10 @@ #include "cloud/providers/aws/AwsStorage.h" #endif +#ifdef HAS_ANTHOS_STORAGE_PROVIDER + #include "cloud/providers/anthos/AnthosStorage.h" +#endif + #ifdef HAS_IBM_STORAGE_PROVIDER #include "cloud/providers/ibm/IbmStorage.h" #endif @@ -47,6 +51,7 @@ static const std::string kAzureCloudProvider{"azure"}; static const std::string kAwsCloudProvider{"aws"}; static const std::string kIbmCloudProvider{"ibm"}; +static const std::string kAnthosCloudProvider{"anthos"}; static const bool UseAzureRest = seismicdrive::sdutils::getSwitchEnv("SDAPI_USE_OUTDATED_AZURE_REST"), UseGcsSdk = seismicdrive::sdutils::getSwitchEnv("SDAPI_USE_GCS_SDK"); @@ -137,11 +142,18 @@ namespace seismicdrive return new cpprest::AzureStorage(authProvider, sdResource, sdReadOnly); #endif } + else if (cloudProvider == kAnthosCloudProvider) + { + #ifdef HAS_ANTHOS_STORAGE_PROVIDER + return new AnthosStorage(authProvider, sdResource, sdReadOnly); + #endif + } else if (cloudProvider == kAwsCloudProvider) { #ifdef HAS_AWS_S3_STORAGE_PROVIDER return new AwsStorage(authProvider, sdResource, sdReadOnly); #endif + } else if (cloudProvider == kIbmCloudProvider) { diff --git a/src/src/lib/cloud/Storage.h b/src/src/lib/cloud/Storage.h index d70e798..13ae28b 100644 --- a/src/src/lib/cloud/Storage.h +++ b/src/src/lib/cloud/Storage.h @@ -46,6 +46,9 @@ namespace seismicdrive #ifdef HAS_IBM_STORAGE_PROVIDER class IbmStorage; #endif +#ifdef HAS_ANTHOS_STORAGE_PROVIDER + class AnthosStorage; +#endif #ifdef HAS_GCS_STORAGE_PROVIDER class GcsStorage; #endif diff --git a/src/src/lib/cloud/providers/anthos/AnthosStorage.cc b/src/src/lib/cloud/providers/anthos/AnthosStorage.cc new file mode 100644 index 0000000..fa55b79 --- /dev/null +++ b/src/src/lib/cloud/providers/anthos/AnthosStorage.cc @@ -0,0 +1,13 @@ +#include "shared/utils.h" + +#include "AnthosStorage.h" + +namespace seismicdrive +{ + AnthosStorage::AnthosStorage(AuthProvider *auth_provider, const std::string &sdresource, bool readonly) + : AwsStorage(auth_provider, sdresource, readonly) + { + endpoint_override = sdutils::getStringEnv("S3_ENDPOINT_OVERRIDE"); + } + +} diff --git a/src/src/lib/cloud/providers/anthos/AnthosStorage.h b/src/src/lib/cloud/providers/anthos/AnthosStorage.h new file mode 100644 index 0000000..9ed81dd --- /dev/null +++ b/src/src/lib/cloud/providers/anthos/AnthosStorage.h @@ -0,0 +1,11 @@ +#include "../aws/AwsStorage.h" + +namespace seismicdrive +{ + class AnthosStorage : public AwsStorage + { + public: + AnthosStorage(AuthProvider *auth_provider, const std::string &sdresource = "", bool readonly = false); + + }; +} \ No newline at end of file diff --git a/src/test/seismic-store-lib/CMakeLists.txt b/src/test/seismic-store-lib/CMakeLists.txt index cb72a46..dd79a29 100644 --- a/src/test/seismic-store-lib/CMakeLists.txt +++ b/src/test/seismic-store-lib/CMakeLists.txt @@ -48,6 +48,29 @@ if (IBM_PROVIDER_ENABLED) endif(IBM_PROVIDER_ENABLED) +if (ANTHOS_PROVIDER_ENABLED) + + find_package(Crc32c) + set(SOURCES anthos_storage.cc + ../shared/utils.cc) + + add_executable(anthos_storage ${SOURCES}) + + target_include_directories(anthos_storage PRIVATE ${AWSSDK_INCLUDE_DIR} Crc32c::crc32c) + + # explictly remove all other providers from this test... + # this is a hack, I should be retrieving and removing all the -D from the target compile definitions + target_compile_options(anthos_storage PRIVATE -UHAS_GCS_STORAGE_PROVIDER -UHAS_AZURE_BLOB_STORAGE_PROVIDER_CURL -UHAS_AZURE_BLOB_STORAGE_PROVIDER -UHAS_AWS_S3_STORAGE_PROVIDER) + target_link_libraries(anthos_storage ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} + ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CMAKE_DL_LIBS} + Crc32c::crc32c) + + add_executable(anthos_storage_static anthos_storage.cc) + target_link_libraries(anthos_storage_static ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) + +endif(ANTHOS_PROVIDER_ENABLED) + + # AZURE STORAGE TEST if (AZURE_PROVIDER_ENABLED OR AZURE_CURL_PROVIDER_ENABLED) set(SOURCES azure_storage.cc ../shared/utils.cc) -- GitLab