Merge branch 'anthos' into 'master'

feat: anthos

See merge request !147

.gitlab-ci.yml

@@ -29,6 +29,9 @@ include:
   # aws
   - local: "/devops/osdu/compile/aws.yml"
   
+  # anthos
+  - local: "/devops/osdu/compile/anthos.yml"
+
   # ibm
   - local: "/devops/osdu/compile/ibm.yml"
   
@@ -61,6 +64,9 @@ include:
   
   # aws
   - local: "/devops/osdu/scanners/abi-check-sdapi-aws.yml"
+
+  # anthos
+  - local: "/devops/osdu/scanners/abi-check-sdapi-anthos.yml"
   
   # gcs
   - local: "/devops/osdu/scanners/abi-check-sdapi-gcs.yml"
@@ -81,6 +87,9 @@ include:
   # aws
   - local: "/devops/osdu/deploy/aws.yml"
   
+  # anthos
+  - local: "/devops/osdu/deploy/anthos.yml"
+
   # ibm
   - local: "/devops/osdu/deploy/ibm.yml"
   

README.md

@@ -141,6 +141,9 @@ options:
 
           --ibm-provider
                Also build the optional IBM cloud provider code. (non-functional, based on AWS C++ SDK)
+        
+          --anthos-provider
+               Also build the optional Anthos provider code. (non-functional, based on AWS C++ SDK)
 ```
 
 ```bash

devops/osdu/compile/anthos.yml

+compile-and-unit-test-anthos:
+    variables:
+      cloud_provider: anthos-provider
+    extends: .compile-and-unit-test

devops/osdu/compile/polycloud.yml

 compile-and-unit-test-polycloud:
     extends: .compile-and-unit-test
     script:
-        - ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --build-utest
+        - ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --anthos-provider --build-utest
         - ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest
\ No newline at end of file

devops/osdu/deploy/anthos.yml

+osdu-package-aws:
+  variables:
+    cloud_provider: anthos
+  extends: .osdu-package
+  dependencies:
+    - compile-and-unit-test-anthos
+    - abi-check-for-sdapi-anthos

devops/osdu/scanners/abi-check-sdapi-anthos.yml

+abi-check-for-sdapi-anthos:
+  variables:
+    cloud_provider: anthos
+  extends: .abi-check-for-sdapi
+  needs: ['compile-and-unit-test-anthos']
+  # TODO: Delete this after anthos-cpp-lib package apper in Registry
+  rules:
+    - allow_failure: true

devops/osdu/scanners/sanitizer-global.yml

@@ -4,7 +4,7 @@
     image: $CI_REGISTRY_IMAGE/sdapi-osdu-centos7-static-deps
     before_script:
         - git submodule update --init --recursive
-        - ./devops/scripts/build-linux64.sh --build-mode=$build_mode --azure-provider --azure-curl-provider --gcs-provider --aws-provider --ibm-provider --build-utest
+        - ./devops/scripts/build-linux64.sh --build-mode=$build_mode --azure-provider --azure-curl-provider --gcs-provider --aws-provider --anthos-provider --ibm-provider --build-utest
     script:
         - ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest
         

devops/scripts/build-linux64.sh

@@ -84,6 +84,14 @@ for i in "$@"; do
             cloud_providers="${cloud_providers};aws"
           fi
           ;;
+        --anthos-provider)
+          shift
+          if [ -z "${cloud_providers}" ]; then
+            cloud_providers="anthos"
+          else
+            cloud_providers="${cloud_providers};anthos"
+          fi
+          ;;
         --ibm-provider)
           shift
           if [ -z "${cloud_providers}" ]; then
@@ -148,6 +156,9 @@ options:
         --aws-provider
             Also build the AWS optional cloud provider code. ( non-functional, based on AWS C++ SDK )
 
+        --anthos-provider
+            Also build the ANTHOS optional cloud provider code. ( non-functional, based on AWS C++ SDK )
+
         --ibm-provider
             Also build the IBM optional cloud provider code. ( non-functional, based on AWS C++ SDK )
 

src/CMakeLists.txt

@@ -132,6 +132,7 @@ endif()
 option( AZURE_PROVIDER_ENABLED OFF )
 option( AZURE_CURL_PROVIDER_ENABLED OFF)
 option( AWS_PROVIDER_ENABLED OFF )
+option( ANTHOS_PROVIDER_ENABLED OFF )
 option( IBM_PROVIDER_ENABLED OFF )
 option( GCS_PROVIDER_ENABLED OFF )
 
@@ -167,6 +168,13 @@ else ()
   message( STATUS "NOT BUILDING AWS PROVIDER" )
 endif()
 
+if( "anthos" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED )
+  set( ANTHOS_PROVIDER_ENABLED  ON)
+  add_definitions( -DHAS_ANTHOS_STORAGE_PROVIDER )
+else ()
+  message( STATUS "NOT BUILDING ANTHOS PROVIDER" )
+endif()
+
 if( "ibm" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED )
   set( IBM_PROVIDER_ENABLED ON)
   add_definitions( -DHAS_IBM_STORAGE_PROVIDER )
@@ -178,6 +186,7 @@ message( STATUS "AZURE_PROVIDER_ENABLED=${AZURE_PROVIDER_ENABLED}" )
 message( STATUS "AZURE_CURL_PROVIDER_ENABLED=${AZURE_CURL_PROVIDER_ENABLED}" )
 message( STATUS "AWS_PROVIDER_ENABLED=${AWS_PROVIDER_ENABLED}" )
 message( STATUS "IBM_PROVIDER_ENABLED=${IBM_PROVIDER_ENABLED}" )
+message( STATUS "ANTHOS_PROVIDER_ENABLED=${ANTHOS_PROVIDER_ENABLED}" )
 message( STATUS "GCS_PROVIDER_ENABLED=${GCS_PROVIDER_ENABLED}" )
 
 # set polycloud definition
@@ -239,6 +248,13 @@ if (IBM_PROVIDER_ENABLED)
     list(APPEND SRC_LIB_PROVIDERS_IBM "${SRC_LIB_PROVIDERS_IBM_EXTRA}")
   endif()
 endif()
+if (ANTHOS_PROVIDER_ENABLED)
+  file(GLOB SRC_LIB_PROVIDERS_ANTHOS  ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/anthos/*.cc)
+  if (NOT AWS_PROVIDER_ENABLED)
+    file(GLOB SRC_LIB_PROVIDERS_ANTHOS_EXTRA ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/aws/*.cc)
+    list(APPEND SRC_LIB_PROVIDERS_ANTHOS "${SRC_LIB_PROVIDERS_ANTHOS_EXTRA}")
+  endif()
+endif()
 if (GCS_PROVIDER_ENABLED)
   file(GLOB SRC_LIB_PROVIDERS_GCS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/gcs/*.cc)
 endif()
@@ -375,6 +391,12 @@ if ( IBM_PROVIDER_ENABLED )
   endif()
 endif()
 
+if ( ANTHOS_PROVIDER_ENABLED )
+  find_package( AWSSDK  REQUIRED COMPONENTS s3 transfer)
+  if ( AWSSDK_FOUND )
+    set(SOURCES ${SOURCES} ${SRC_LIB_PROVIDERS_ANTHOS} )
+  endif()
+endif()
 
 if (GCS_PROVIDER_ENABLED)
     # Google cloud storage c++ client library
@@ -463,6 +485,11 @@ if(NOT WIN32)
     target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
   endif()
 
+  if( ANTHOS_PROVIDER_ENABLED )
+    target_include_directories( sdapi PRIVATE ${AWSSDK_INCLUDE_DIR} )
+    target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
+  endif()
+
   if( GCS_PROVIDER_ENABLED )
     target_link_libraries( sdapi PRIVATE storage_client )
   endif()
@@ -497,6 +524,12 @@ else() # WIN32 static lib
 	  message( STATUS "[IBM Test] - WIN32 flow:target_include_directories:target_link_libraries")
   endif()
 
+  if( ANTHOS_PROVIDER_ENABLED )
+    target_include_directories( sdapi PRIVATE ${AWSSDK_INCLUDE_DIR} )
+    target_link_libraries( sdapi PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
+	  message( STATUS "[ANTHOS Test] - WIN32 flow:target_include_directories:target_link_libraries")
+  endif()
+
 endif()
 
 if(NOT WIN32)
@@ -602,6 +635,11 @@ if(NOT WIN32)
     target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
   endif()
 
+  if( ANTHOS_PROVIDER_ENABLED )
+    target_include_directories( sdapi_static PRIVATE ${AWSSDK_INCLUDE_DIR} )
+    target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
+  endif()
+
   if (GCS_PROVIDER_ENABLED)
     target_link_libraries(sdapi_static PRIVATE storage_client)
   endif()
@@ -635,6 +673,12 @@ else()
 	  message( STATUS "[IBM Test] - WIN32 flow:target_include_directories:target_link_libraries")
    endif()
 
+  if( ANTHOS_PROVIDER_ENABLED )
+    target_include_directories( sdapi_static PRIVATE ${AWSSDK_INCLUDE_DIR} )
+    target_link_libraries( sdapi_static PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} )
+	  message( STATUS "[Anthos Test] - WIN32 flow:target_include_directories:target_link_libraries")
+  endif()
+
     SET(S "sdapi")
     if (DEFINED LIB_VERSION_ON_NAME)
         SET(V "_${LIB_VERSION_MAJOR}")

src/src/lib/cloud/Storage.cc

@@ -36,6 +36,10 @@
     #include "cloud/providers/aws/AwsStorage.h"
 #endif
 
+#ifdef HAS_ANTHOS_STORAGE_PROVIDER
+    #include "cloud/providers/anthos/AnthosStorage.h"
+#endif
+
 #ifdef HAS_IBM_STORAGE_PROVIDER
     #include "cloud/providers/ibm/IbmStorage.h"
 #endif
@@ -47,6 +51,7 @@
 static const std::string kAzureCloudProvider{"azure"};
 static const std::string kAwsCloudProvider{"aws"};
 static const std::string kIbmCloudProvider{"ibm"};
+static const std::string kAnthosCloudProvider{"anthos"};
 
 static const bool UseAzureRest = seismicdrive::sdutils::getSwitchEnv("SDAPI_USE_OUTDATED_AZURE_REST"),
                   UseGcsSdk    = seismicdrive::sdutils::getSwitchEnv("SDAPI_USE_GCS_SDK");
@@ -137,11 +142,18 @@ namespace seismicdrive
                 return new cpprest::AzureStorage(authProvider, sdResource, sdReadOnly);
             #endif
         }
+        else if (cloudProvider == kAnthosCloudProvider)
+        {
+            #ifdef HAS_ANTHOS_STORAGE_PROVIDER
+                return new AnthosStorage(authProvider, sdResource, sdReadOnly);
+            #endif
+        }
         else if (cloudProvider == kAwsCloudProvider)
         {
             #ifdef HAS_AWS_S3_STORAGE_PROVIDER
                 return new AwsStorage(authProvider, sdResource, sdReadOnly);
             #endif
+            
         }
         else if (cloudProvider == kIbmCloudProvider)
         {

src/src/lib/cloud/Storage.h

@@ -46,6 +46,9 @@ namespace seismicdrive
 #ifdef HAS_IBM_STORAGE_PROVIDER
     class IbmStorage;
 #endif
+#ifdef HAS_ANTHOS_STORAGE_PROVIDER
+    class AnthosStorage;
+#endif
 #ifdef HAS_GCS_STORAGE_PROVIDER
     class GcsStorage;
 #endif

src/src/lib/cloud/providers/anthos/AnthosStorage.cc

+#include "shared/utils.h"
+
+#include "AnthosStorage.h"
+
+namespace seismicdrive
+{
+    AnthosStorage::AnthosStorage(AuthProvider *auth_provider, const std::string &sdresource, bool readonly)
+      : AwsStorage(auth_provider, sdresource, readonly)
+    {
+        endpoint_override = sdutils::getStringEnv("S3_ENDPOINT_OVERRIDE");
+    }
+
+}

src/src/lib/cloud/providers/anthos/AnthosStorage.h

+#include "../aws/AwsStorage.h"
+
+namespace seismicdrive
+{
+    class AnthosStorage : public AwsStorage
+    {
+    public:
+        AnthosStorage(AuthProvider *auth_provider, const std::string &sdresource = "", bool readonly = false);
+
+    };
+}
\ No newline at end of file

src/test/seismic-store-lib/CMakeLists.txt

@@ -48,6 +48,29 @@ if (IBM_PROVIDER_ENABLED)
 endif(IBM_PROVIDER_ENABLED)
 
 
+if (ANTHOS_PROVIDER_ENABLED)
+
+    find_package(Crc32c)
+    set(SOURCES anthos_storage.cc
+                ../shared/utils.cc)
+
+    add_executable(anthos_storage ${SOURCES})
+
+    target_include_directories(anthos_storage PRIVATE ${AWSSDK_INCLUDE_DIR} Crc32c::crc32c)
+
+    # explictly remove all other providers from this test...
+    # this is a hack, I should be retrieving and removing all the -D from the target compile definitions
+    target_compile_options(anthos_storage PRIVATE -UHAS_GCS_STORAGE_PROVIDER -UHAS_AZURE_BLOB_STORAGE_PROVIDER_CURL -UHAS_AZURE_BLOB_STORAGE_PROVIDER -UHAS_AWS_S3_STORAGE_PROVIDER)
+    target_link_libraries(anthos_storage ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}
+                                      ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CMAKE_DL_LIBS}
+                                      Crc32c::crc32c)
+
+    add_executable(anthos_storage_static anthos_storage.cc)
+    target_link_libraries(anthos_storage_static ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS})
+
+endif(ANTHOS_PROVIDER_ENABLED)
+
+
 # AZURE STORAGE TEST
 if (AZURE_PROVIDER_ENABLED OR AZURE_CURL_PROVIDER_ENABLED)
     set(SOURCES azure_storage.cc ../shared/utils.cc)