Commit dec1c2bd authored by Yan Sushchynski (EPAM)'s avatar Yan Sushchynski (EPAM)
Browse files

Merge branch 'anthos' into 'master'

feat: anthos

See merge request !147
parents c939d9bc e5302209
Pipeline #138783 failed
......@@ -29,6 +29,9 @@ include:
# aws
- local: "/devops/osdu/compile/aws.yml"
# anthos
- local: "/devops/osdu/compile/anthos.yml"
# ibm
- local: "/devops/osdu/compile/ibm.yml"
......@@ -61,6 +64,9 @@ include:
# aws
- local: "/devops/osdu/scanners/abi-check-sdapi-aws.yml"
# anthos
- local: "/devops/osdu/scanners/abi-check-sdapi-anthos.yml"
# gcs
- local: "/devops/osdu/scanners/abi-check-sdapi-gcs.yml"
......@@ -81,6 +87,9 @@ include:
# aws
- local: "/devops/osdu/deploy/aws.yml"
# anthos
- local: "/devops/osdu/deploy/anthos.yml"
# ibm
- local: "/devops/osdu/deploy/ibm.yml"
......
......@@ -141,6 +141,9 @@ options:
--ibm-provider
Also build the optional IBM cloud provider code. (non-functional, based on AWS C++ SDK)
--anthos-provider
Also build the optional Anthos provider code. (non-functional, based on AWS C++ SDK)
```
```bash
......
compile-and-unit-test-anthos:
variables:
cloud_provider: anthos-provider
extends: .compile-and-unit-test
compile-and-unit-test-polycloud:
extends: .compile-and-unit-test
script:
- ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --build-utest
- ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --anthos-provider --build-utest
- ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest
\ No newline at end of file
osdu-package-aws:
variables:
cloud_provider: anthos
extends: .osdu-package
dependencies:
- compile-and-unit-test-anthos
- abi-check-for-sdapi-anthos
abi-check-for-sdapi-anthos:
variables:
cloud_provider: anthos
extends: .abi-check-for-sdapi
needs: ['compile-and-unit-test-anthos']
# TODO: Delete this after anthos-cpp-lib package apper in Registry
rules:
- allow_failure: true
......@@ -4,7 +4,7 @@
image: $CI_REGISTRY_IMAGE/sdapi-osdu-centos7-static-deps
before_script:
- git submodule update --init --recursive
- ./devops/scripts/build-linux64.sh --build-mode=$build_mode --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --build-utest
- ./devops/scripts/build-linux64.sh --build-mode=$build_mode --azure-provider --azure-curl-provider --gcs-provider --aws-provider --anthos-provider --ibm-provider --build-utest
script:
- ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest
......
......@@ -84,6 +84,14 @@ for i in "$@"; do
cloud_providers="${cloud_providers};aws"
fi
;;
--anthos-provider)
shift
if [ -z "${cloud_providers}" ]; then
cloud_providers="anthos"
else
cloud_providers="${cloud_providers};anthos"
fi
;;
--ibm-provider)
shift
if [ -z "${cloud_providers}" ]; then
......@@ -148,6 +156,9 @@ options:
--aws-provider
Also build the AWS optional cloud provider code. ( non-functional, based on AWS C++ SDK )
--anthos-provider
Also build the ANTHOS optional cloud provider code. ( non-functional, based on AWS C++ SDK )
--ibm-provider
Also build the IBM optional cloud provider code. ( non-functional, based on AWS C++ SDK )
......
......@@ -132,6 +132,7 @@ endif()
option( AZURE_PROVIDER_ENABLED OFF )
option( AZURE_CURL_PROVIDER_ENABLED OFF)
option( AWS_PROVIDER_ENABLED OFF )
option( ANTHOS_PROVIDER_ENABLED OFF )
option( IBM_PROVIDER_ENABLED OFF )
option( GCS_PROVIDER_ENABLED OFF )
......@@ -167,6 +168,13 @@ else ()
message( STATUS "NOT BUILDING AWS PROVIDER" )
endif()
if( "anthos" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED )
set( ANTHOS_PROVIDER_ENABLED ON)
add_definitions( -DHAS_ANTHOS_STORAGE_PROVIDER )
else ()
message( STATUS "NOT BUILDING ANTHOS PROVIDER" )
endif()
if( "ibm" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED )
set( IBM_PROVIDER_ENABLED ON)
add_definitions( -DHAS_IBM_STORAGE_PROVIDER )
......@@ -178,6 +186,7 @@ message( STATUS "AZURE_PROVIDER_ENABLED=${AZURE_PROVIDER_ENABLED}" )
message( STATUS "AZURE_CURL_PROVIDER_ENABLED=${AZURE_CURL_PROVIDER_ENABLED}" )
message( STATUS "AWS_PROVIDER_ENABLED=${AWS_PROVIDER_ENABLED}" )
message( STATUS "IBM_PROVIDER_ENABLED=${IBM_PROVIDER_ENABLED}" )
message( STATUS "ANTHOS_PROVIDER_ENABLED=${ANTHOS_PROVIDER_ENABLED}" )
message( STATUS "GCS_PROVIDER_ENABLED=${GCS_PROVIDER_ENABLED}" )
# set polycloud definition
......@@ -239,6 +248,13 @@ if (IBM_PROVIDER_ENABLED)
list(APPEND SRC_LIB_PROVIDERS_IBM "${SRC_LIB_PROVIDERS_IBM_EXTRA}")
endif()
endif()
if (ANTHOS_PROVIDER_ENABLED)
file(GLOB SRC_LIB_PROVIDERS_ANTHOS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/anthos/*.cc)
if (NOT AWS_PROVIDER_ENABLED)
file(GLOB SRC_LIB_PROVIDERS_ANTHOS_EXTRA ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/aws/*.cc)
list(APPEND SRC_LIB_PROVIDERS_ANTHOS "${SRC_LIB_PROVIDERS_ANTHOS_EXTRA}")
endif()
endif()
if (GCS_PROVIDER_ENABLED)
file(GLOB SRC_LIB_PROVIDERS_GCS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/gcs/*.cc)
endif()
......@@ -375,6 +391,12 @@ if ( IBM_PROVIDER_ENABLED )
endif()
endif()
if ( ANTHOS_PROVIDER_ENABLED )
find_package( AWSSDK REQUIRED COMPONENTS s3 transfer)
if ( AWSSDK_FOUND )
set(SOURCES ${SOURCES} ${SRC_LIB_PROVIDERS_ANTHOS} )
endif()
endif()
if (GCS_PROVIDER_ENABLED)
# Google cloud storage c++ client library
......@@ -463,6 +485,11 @@ if(NOT WIN32)
target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
endif()
if( ANTHOS_PROVIDER_ENABLED )
target_include_directories( sdapi PRIVATE ${AWSSDK_INCLUDE_DIR} )
target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
endif()
if( GCS_PROVIDER_ENABLED )
target_link_libraries( sdapi PRIVATE storage_client )
endif()
......@@ -497,6 +524,12 @@ else() # WIN32 static lib
message( STATUS "[IBM Test] - WIN32 flow:target_include_directories:target_link_libraries")
endif()
if( ANTHOS_PROVIDER_ENABLED )
target_include_directories( sdapi PRIVATE ${AWSSDK_INCLUDE_DIR} )
target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
message( STATUS "[ANTHOS Test] - WIN32 flow:target_include_directories:target_link_libraries")
endif()
endif()
if(NOT WIN32)
......@@ -602,6 +635,11 @@ if(NOT WIN32)
target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
endif()
if( ANTHOS_PROVIDER_ENABLED )
target_include_directories( sdapi_static PRIVATE ${AWSSDK_INCLUDE_DIR} )
target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
endif()
if (GCS_PROVIDER_ENABLED)
target_link_libraries(sdapi_static PRIVATE storage_client)
endif()
......@@ -635,6 +673,12 @@ else()
message( STATUS "[IBM Test] - WIN32 flow:target_include_directories:target_link_libraries")
endif()
if( ANTHOS_PROVIDER_ENABLED )
target_include_directories( sdapi_static PRIVATE ${AWSSDK_INCLUDE_DIR} )
target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
message( STATUS "[Anthos Test] - WIN32 flow:target_include_directories:target_link_libraries")
endif()
SET(S "sdapi")
if (DEFINED LIB_VERSION_ON_NAME)
SET(V "_${LIB_VERSION_MAJOR}")
......
......@@ -36,6 +36,10 @@
#include "cloud/providers/aws/AwsStorage.h"
#endif
#ifdef HAS_ANTHOS_STORAGE_PROVIDER
#include "cloud/providers/anthos/AnthosStorage.h"
#endif
#ifdef HAS_IBM_STORAGE_PROVIDER
#include "cloud/providers/ibm/IbmStorage.h"
#endif
......@@ -47,6 +51,7 @@
static const std::string kAzureCloudProvider{"azure"};
static const std::string kAwsCloudProvider{"aws"};
static const std::string kIbmCloudProvider{"ibm"};
static const std::string kAnthosCloudProvider{"anthos"};
static const bool UseAzureRest = seismicdrive::sdutils::getSwitchEnv("SDAPI_USE_OUTDATED_AZURE_REST"),
UseGcsSdk = seismicdrive::sdutils::getSwitchEnv("SDAPI_USE_GCS_SDK");
......@@ -137,11 +142,18 @@ namespace seismicdrive
return new cpprest::AzureStorage(authProvider, sdResource, sdReadOnly);
#endif
}
else if (cloudProvider == kAnthosCloudProvider)
{
#ifdef HAS_ANTHOS_STORAGE_PROVIDER
return new AnthosStorage(authProvider, sdResource, sdReadOnly);
#endif
}
else if (cloudProvider == kAwsCloudProvider)
{
#ifdef HAS_AWS_S3_STORAGE_PROVIDER
return new AwsStorage(authProvider, sdResource, sdReadOnly);
#endif
}
else if (cloudProvider == kIbmCloudProvider)
{
......
......@@ -46,6 +46,9 @@ namespace seismicdrive
#ifdef HAS_IBM_STORAGE_PROVIDER
class IbmStorage;
#endif
#ifdef HAS_ANTHOS_STORAGE_PROVIDER
class AnthosStorage;
#endif
#ifdef HAS_GCS_STORAGE_PROVIDER
class GcsStorage;
#endif
......
#include "shared/utils.h"
#include "AnthosStorage.h"
namespace seismicdrive
{
AnthosStorage::AnthosStorage(AuthProvider *auth_provider, const std::string &sdresource, bool readonly)
: AwsStorage(auth_provider, sdresource, readonly)
{
endpoint_override = sdutils::getStringEnv("S3_ENDPOINT_OVERRIDE");
}
}
#include "../aws/AwsStorage.h"
namespace seismicdrive
{
class AnthosStorage : public AwsStorage
{
public:
AnthosStorage(AuthProvider *auth_provider, const std::string &sdresource = "", bool readonly = false);
};
}
\ No newline at end of file
......@@ -48,6 +48,29 @@ if (IBM_PROVIDER_ENABLED)
endif(IBM_PROVIDER_ENABLED)
if (ANTHOS_PROVIDER_ENABLED)
find_package(Crc32c)
set(SOURCES anthos_storage.cc
../shared/utils.cc)
add_executable(anthos_storage ${SOURCES})
target_include_directories(anthos_storage PRIVATE ${AWSSDK_INCLUDE_DIR} Crc32c::crc32c)
# explictly remove all other providers from this test...
# this is a hack, I should be retrieving and removing all the -D from the target compile definitions
target_compile_options(anthos_storage PRIVATE -UHAS_GCS_STORAGE_PROVIDER -UHAS_AZURE_BLOB_STORAGE_PROVIDER_CURL -UHAS_AZURE_BLOB_STORAGE_PROVIDER -UHAS_AWS_S3_STORAGE_PROVIDER)
target_link_libraries(anthos_storage ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}
${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CMAKE_DL_LIBS}
Crc32c::crc32c)
add_executable(anthos_storage_static anthos_storage.cc)
target_link_libraries(anthos_storage_static ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS})
endif(ANTHOS_PROVIDER_ENABLED)
# AZURE STORAGE TEST
if (AZURE_PROVIDER_ENABLED OR AZURE_CURL_PROVIDER_ENABLED)
set(SOURCES azure_storage.cc ../shared/utils.cc)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment