diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e894149586cb93e7d279b2d333363e83890a4f82..a6092df9dab2f9cf2548d7adfd5564b535e4bbed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,13 @@ compile-and-unit-test: stage: build - image: $CI_REGISTRY_IMAGE/sdapi-osdu-centos8 + image: $CI_REGISTRY_IMAGE/sdapi-osdu-ubuntu before_script: - git submodule update --init --recursive script: - - ./devops/scripts/build-linux64.sh --build-mode=Release --build-utest + - ./devops/scripts/build-linux64.sh --build-mode=Release --azure-provider --azure-curl-provider --build-utest --build-ftest --build-ptest --build-rtest - ./devops/scripts/run_unit_tests.sh --program=$(pwd)/build/test/utest/utest - cp -r dist/Release/sdapi-* $(pwd) artifacts: name: $(basename sdapi-*)-linux64 paths: - - sdapi-*/ \ No newline at end of file + - sdapi-*/ diff --git a/README.md b/README.md index 46878b555be599cb9cf24c90e9f77826d926e3d4..94c3a563ebe0059820aeea961c35b9ef17e3dea7 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ PS .\devops\scripts\build-win64.ps1 -build_type 'Release' -vcpkg_path 'D:/vcpkg' Library behaviours can be controlled via these environment variables: - DISABLE_SSL_CERTIFICATE_VERIFICATION=ON|OFF, enable or disable the SSL certificate verification (enabled by default) -- ENABLE_CURL_SIGNAL_HANDLERS=ON|OFF, enable or disable the internal curl signal handling (disabled by default) +- ENABLE_AZURE_CURL_BASED=ON|OFF, enable the usage of the azure sdk based on culr in case the library is built with both, --azure-provider(cpprest) --azure-curl-provider. (disabled by default) ## Usage Examples diff --git a/devops/docker/build.centos7.dockerfile b/devops/docker/build.centos7.dockerfile index 6d81ce2c348339d56accaaffba122301d9d054ec..0022964f6d440cc56921a9c9e221ef17f26f8e37 100644 --- a/devops/docker/build.centos7.dockerfile +++ b/devops/docker/build.centos7.dockerfile @@ -20,19 +20,6 @@ FROM centos/devtoolset-7-toolchain-centos7 # build as root USER 0 -# build dependencies -ARG CMAKE_VERSION=3.19.1 - -# library dependencies -ARG LIB_ZLIB_VERSION=1.2.11 -ARG LIB_OPENSSL_VERSION=1.1.1g -ARG LIB_CURL_VERSION=7.73.0 -ARG LIB_BOOST_VERSION=1.73.0 -ARG LIB_UUID_VERSION=1.0.3 -ARG LIB_XML2_VERSION=2.9.10 -ARG LIB_CPPREST_VERSION=2.10.16 -ARG LIB_AZURE_STORAGE_VERSION=7.5.0 - # library path ARG LIBRARIES_INSTALL_PATH=/usr/local @@ -43,6 +30,7 @@ RUN yum update -y && yum install -y wget which git libtool autoconf perl-devel WORKDIR /tmp # [CMAKE] +ARG CMAKE_VERSION=3.19.4 RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ && tar -zxvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -C /usr/local \ && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/bin/cmake \ @@ -50,6 +38,7 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cm && rm -rf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz # [ZLIB] +ARG LIB_ZLIB_VERSION=1.2.11 RUN wget https://www.zlib.net/zlib-${LIB_ZLIB_VERSION}.tar.gz \ && tar -zxvf zlib-${LIB_ZLIB_VERSION}.tar.gz \ && cd zlib-${LIB_ZLIB_VERSION} \ @@ -61,6 +50,7 @@ RUN wget https://www.zlib.net/zlib-${LIB_ZLIB_VERSION}.tar.gz \ && rm -rf zlib-${LIB_ZLIB_VERSION} # [OPENSSL] +ARG LIB_OPENSSL_VERSION=1.1.1g RUN wget https://www.openssl.org/source/openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && tar -zxvf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && cd openssl-${LIB_OPENSSL_VERSION} \ @@ -72,6 +62,7 @@ RUN wget https://www.openssl.org/source/openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && rm -rf openssl-${LIB_OPENSSL_VERSION} # [BOOST] +ARG LIB_BOOST_VERSION=1.73.0 RUN wget https://dl.bintray.com/boostorg/release/${LIB_BOOST_VERSION}/source/boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && tar -zxvf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && cd boost_$(echo "$LIB_BOOST_VERSION" | tr . _) \ @@ -84,7 +75,8 @@ RUN wget https://dl.bintray.com/boostorg/release/${LIB_BOOST_VERSION}/source/boo && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _) # [UUID] -RUN wget https://vorboss.dl.sourceforge.net/project/libuuid/libuuid-${LIB_UUID_VERSION}.tar.gz \ +ARG LIB_UUID_VERSION=1.0.3 +RUN wget https://sourceforge.net/projects/libuuid/files/libuuid-${LIB_UUID_VERSION}.tar.gz \ && tar -zxvf libuuid-${LIB_UUID_VERSION}.tar.gz \ && cd libuuid-${LIB_UUID_VERSION} \ && ./configure --prefix=${LIBRARIES_INSTALL_PATH} \ @@ -95,6 +87,7 @@ RUN wget https://vorboss.dl.sourceforge.net/project/libuuid/libuuid-${LIB_UUID_V && rm -rf libuuid-${LIB_UUID_VERSION} # [XML2] +ARG LIB_XML2_VERSION=2.9.10 RUN wget https://github.com/GNOME/libxml2/archive/v${LIB_XML2_VERSION}.tar.gz \ && tar -zxvf v${LIB_XML2_VERSION}.tar.gz \ && cd libxml2-${LIB_XML2_VERSION} \ @@ -106,6 +99,7 @@ RUN wget https://github.com/GNOME/libxml2/archive/v${LIB_XML2_VERSION}.tar.gz \ && rm -rf libxml2-${LIB_XML2_VERSION} # [CCPRESTSDK] +ARG LIB_CPPREST_VERSION=2.10.16 RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive \ && cd cpprestsdk \ && mkdir build \ @@ -117,6 +111,7 @@ RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk && rm -rf cpprestsdk # [AZURESTORAGECPP] +ARG LIB_AZURE_STORAGE_VERSION=7.5.0 RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive \ && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage \ && mkdir build.release \ @@ -128,6 +123,7 @@ RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-st && rm -rf azure-storage-cpp # [CURL] +ARG LIB_CURL_VERSION=7.73.0 RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive \ && cd curl \ && mkdir build \ @@ -138,6 +134,19 @@ RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/cu && cd ../.. \ && rm -rf curl +# [AZURE-STORAGE-FOR-CPP] +ARG AZURE_STORAGE_FOR_CPP_VERSION=12.0.0-beta.10 +RUN wget https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && tar -zxvf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && cd azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=YES .. \ + && make -j $(nproc) \ + && make install \ + && rm -rf azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && rm -rf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz + # finalize the build WORKDIR / USER 1001 \ No newline at end of file diff --git a/devops/docker/build.centos8.staticdeps.dockerfile b/devops/docker/build.centos7.staticdeps.dev.dockerfile similarity index 74% rename from devops/docker/build.centos8.staticdeps.dockerfile rename to devops/docker/build.centos7.staticdeps.dev.dockerfile index c40ec1dfcc7060fe161848f8743ff2dc903d7c9c..43746f4a57e908f45cb31ceff9dc85ee0b3f93a3 100644 --- a/devops/docker/build.centos8.staticdeps.dockerfile +++ b/devops/docker/build.centos7.staticdeps.dev.dockerfile @@ -15,25 +15,22 @@ # ============================================================================ # base image -FROM centos:8 AS dev -# centos 8 comes with gcc 8.3.1 as the default. -# centos 8 also has end of life of 2021 and has been abandoned by CentOS in favor of Centos Stream -# which is not a stable / long term support release but an upstream CI type of OS. -# strategy TBD. +FROM centos/devtoolset-7-toolchain-centos7 AS dev # build as root USER 0 -# build dependencies -ARG CMAKE_VERSION=3.19.1 +# library path +ARG LIBRARIES_INSTALL_PATH=/usr/local # install required build tools via packet manager -RUN yum update -y && yum install -y wget which git gcc-c++ make libtool autoconf perl-devel python38-devel xz-devel patch which file +RUN yum update -y && yum install -y wget which git libtool autoconf perl-devel python-devel xz-devel patch # temporary working directory WORKDIR /tmp # [CMAKE] +ARG CMAKE_VERSION=3.19.4 RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ && tar -zxvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -C /usr/local \ && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/bin/cmake \ @@ -42,140 +39,152 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cm && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/ccmake /usr/bin/ccmake \ && rm -rf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -FROM dev as rest-dev - -# library dependencies -ARG LIB_ZLIB_VERSION=1.2.11 -ARG LIB_OPENSSL_VERSION=1.1.1g -ARG LIB_CURL_VERSION=7.74.0 -ARG LIB_UUID_VERSION=1.0.3 - - -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local - - -# +# ---------------------------------------------------------------------------------------------------------------------------- # if we want to avoid adding shared libraries as depdendencies ( since in linux most of them export all their symbols) # we need to see if static libraries with Position Indepdent Code are built ( aka, .a that can be used to build .so later on) # # the analysis is done with : # for slib in /usr/local/lib*/*.a; do echo $slib; readelf --relocs ${slib} | egrep -c '(GOT|PLT|JUMP_SLO)'; done # -# aka : for all archives, read relocations, see if they are pointing to the global offset table, the procedure linkage table or are calls to shared libraries ( jump ) -# not quite sure this is exhaustive as this indicates _some relocatable code present_ not absence of relocations without indirections. +# aka : for all archives, read relocations, see if they are pointing to the global offset table, +# the procedure linkage table or are calls to shared libraries ( jump ) +# not quite sure this is exhaustive as this indicates _some relocatable code present_ +# not absence of relocations without indirections. # # otherwise we'd have to play with visibiliy at library build time, linker version scripts etc. +# ---------------------------------------------------------------------------------------------------------------------------- + +FROM dev as rest-dev # [ZLIB] -# build both static libraries (without PIC) and shared libraries. +ARG LIB_ZLIB_VERSION=1.2.11 RUN wget https://www.zlib.net/zlib-${LIB_ZLIB_VERSION}.tar.gz \ && tar -zxvf zlib-${LIB_ZLIB_VERSION}.tar.gz \ - && cd /tmp/zlib-${LIB_ZLIB_VERSION} \ + && cd zlib-${LIB_ZLIB_VERSION} \ && CFLAGS=-fPIC ./configure --prefix=${LIBRARIES_INSTALL_PATH} --static \ - && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf zlib-${LIB_ZLIB_VERSION}.tar.gz \ + && make -j $(nproc) \ + && make install \ + && cd .. \ + && rm -rf zlib-${LIB_ZLIB_VERSION}.tar.gz \ && rm -rf zlib-${LIB_ZLIB_VERSION} # [OPENSSL] -# uuuuh. this has TCC implications does it not ? not making it static then. -# build it statically anyways... +# this has TCC implications +# consider instead RUN apt-get install openssl-libs openssl-dev +ARG LIB_OPENSSL_VERSION=1.1.1g RUN wget https://www.openssl.org/source/openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && tar -zxvf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && cd /tmp/openssl-${LIB_OPENSSL_VERSION} \ && ./config --prefix=${LIBRARIES_INSTALL_PATH} no-shared \ && make -j $(nproc) \ - && make install_sw -WORKDIR /tmp -RUN rm -rf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ + && make install_sw \ + && cd .. \ + rm -rf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && rm -rf openssl-${LIB_OPENSSL_VERSION} -# consider instead -# RUN yum install -y openssl-libs.x86_64 openssl-devel.x86_64 -# and only use system ones - # [UUID] -# this builds both static libraries ( without fPIC ) and shared libraries -RUN wget https://vorboss.dl.sourceforge.net/project/libuuid/libuuid-${LIB_UUID_VERSION}.tar.gz \ +ARG LIB_UUID_VERSION=1.0.3 +RUN wget https://sourceforge.net/projects/libuuid/files/libuuid-${LIB_UUID_VERSION}.tar.gz \ && tar -zxvf libuuid-${LIB_UUID_VERSION}.tar.gz \ && cd /tmp/libuuid-${LIB_UUID_VERSION} \ && ./configure --prefix=${LIBRARIES_INSTALL_PATH} --with-pic --enable-shared=no --enable-static=yes \ && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf libuuid-${LIB_UUID_VERSION}.tar.gz \ + && make install \ + && cd .. \ + && rm -rf libuuid-${LIB_UUID_VERSION}.tar.gz \ && rm -rf libuuid-${LIB_UUID_VERSION} # [CURL] # this is not equivalent to the system one built with libtool ... soname, versions are not done the same way. # may require a patch for locally built nghttp2? or http3 libraries -RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive -WORKDIR /tmp/curl/build -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. -RUN make -j $(nproc) && make install -WORKDIR /tmp -RUN rm -rf curl +ARG LIB_CURL_VERSION=7.74.0 +RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive \ + && cd curl \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. \ + && make -j $(nrpoc) \ + && make install \ + && cd ../.. \ + && rm -rf curl FROM rest-dev as azure-dev -ARG LIB_BOOST_VERSION=1.73.0 -ARG LIB_XML2_VERSION=2.9.10 -ARG LIB_CPPREST_VERSION=2.10.16 -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local - # [BOOST] +ARG LIB_BOOST_VERSION=1.73.0 RUN wget https://dl.bintray.com/boostorg/release/${LIB_BOOST_VERSION}/source/boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && tar -zxvf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && cd boost_$(echo "$LIB_BOOST_VERSION" | tr . _) \ && ./bootstrap.sh --prefix=${LIBRARIES_INSTALL_PATH} \ && ./b2 stage -j $(nproc) threading=multi link=static cxxflags='-fPIC' linkflags='-pie' -q \ && ./b2 install threading=multi link=static cxxflags='-fPIC' linkflags='-pie' \ - && ln -svf detail/sha1.hpp ${LIBRARIES_INSTALL_PATH}/include/boost/uuid/sha1.hpp -WORKDIR /tmp -RUN rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ + && ln -svf detail/sha1.hpp ${LIBRARIES_INSTALL_PATH}/include/boost/uuid/sha1.hpp \ + && cd .. \ + && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _) +# [XZ] +# because we need liblzma that can be found and consumed by CMake +ARG LIB_XZ_COMMIT=e7da44d +RUN git clone https://git.tukaani.org/xz.git xz \ + && cd xz \ + && git checkout ${LIB_XZ_COMMIT} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=NO .. \ + && make -j $(nproc ) \ + && make install \ + && cd ../.. \ + && rm -rf xz # [XML2] +ARG LIB_XML2_VERSION=2.9.10 RUN wget https://github.com/GNOME/libxml2/archive/v${LIB_XML2_VERSION}.tar.gz \ && tar -zxvf v${LIB_XML2_VERSION}.tar.gz \ - && cd /tmp/libxml2-${LIB_XML2_VERSION} \ + && cd libxml2-${LIB_XML2_VERSION} \ && ./autogen.sh --prefix=${LIBRARIES_INSTALL_PATH} --with-pic --enable-static=yes --enable-shared=no \ && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf v${LIB_XML2_VERSION}.tar.gz && rm -rf libxml2-${LIB_XML2_VERSION} - + && make install \ + && cd .. \ + && rm -rf v${LIB_XML2_VERSION}.tar.gz \ + && rm -rf libxml2-${LIB_XML2_VERSION} # [CCPRESTSDK] -RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive -WORKDIR /tmp/cpprestsdk/build -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON .. -RUN make -j $(nproc ) && make install -WORKDIR /tmp -RUN rm -rf cpprestsdk - - -FROM azure-dev as azure-storage-dev - -ARG LIB_AZURE_STORAGE_VERSION=7.5.0 -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local +ARG LIB_CPPREST_VERSION=2.10.16 +RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive \ + && cd cpprestsdk \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON -DBUILD_SAMPLES=OFF .. \ + && make -j $(nrpoc) \ + && make install \ + && cd ../.. \ + && rm -rf cpprestsdk # [AZURESTORAGECPP] -# most likely needs a patch to build properly with non-system curl found. -RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive -WORKDIR /tmp/azure-storage-cpp/Microsoft.WindowsAzure.Storage/build.release -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON .. -RUN make -j $(nproc) && make install -WORKDIR /tmp -RUN rm -rf azure-storage-cpp - - -# packages depdending on libcurl +ARG LIB_AZURE_STORAGE_VERSION=7.5.0 +RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive \ + && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage \ + && mkdir build.release \ + && cd build.release \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON .. \ + && make -j $(nproc) \ + && make install \ + && cd ../../.. \ + && rm -rf azure-storage-cpp + +# [AZURE-STORAGE-FOR-CPP] +ARG AZURE_STORAGE_FOR_CPP_VERSION=12.0.0-beta.10 +RUN wget https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && tar -zxvf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && cd azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=NO -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DBUILD_TRANSPORT_CURL:BOOL=ON .. \ + && make -j $(nproc) \ + && make install \ + && rm -rf azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && rm -rf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz FROM azure-storage-dev AS aws-dev @@ -370,7 +379,7 @@ RUN cmake \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DgRPC_BUILD_TESTS=OFF \ -DgRPC_ZLIB_PROVIDER=package \ -DgRPC_SSL_PROVIDER=package \ @@ -437,14 +446,61 @@ RUN cmake \ RUN cmake --build /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/cmake-out \ --target install -- -j $( nproc ) +WORKDIR /tmp RUN rm -rf /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG} +ARG LIB_CPR_RELASE_TAG=v1.6.0 +ARG CROW_COMMIT=4e801d0 + FROM gcs-dev AS sdapi-dev +# [CPR] +# nice little third party packages for testing mainly. +WORKDIR /tmp +RUN git clone https://github.com/whoshuu/cpr.git cpr \ + && cd cpr \ + && git checkout ${LIB_CPR_RELEASE_TAG} \ + && cmake -H. -Bcmake-out \ + -DCPR_CURL_NOSIGNAL=ON \ + -DCPR_FORCE_USE_SYSTEM_CURL=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=YES \ + && cmake --build cmake-out --target install -- -j $(nproc) +WORKDIR /tmp +RUN rm -rf /tmp/cpr + +RUN yum install -y rh-python38.x86_64 rh-git218 +RUN echo source scl_source enable rh-python38 >> /opt/app-root/etc/scl_enable +RUN echo source scl_source enable rh-git218 >> /opt/app-root/etc/scl_enable + +# [ Crow ] +# header only produces an amalgated library +# has some static variables... +# will need to fork and patch it one day +WORKDIR /tmp +RUN source /opt/rh/rh-python38/enable \ + && git clone https://github.com/CrowCpp/crow.git crow \ + && cd crow \ + && git checkout ${CROW_COMMIT} \ + && cmake -H. -Bcmake-out \ + && cmake --build cmake-out --target install -- -j $(nproc ) + +WORKDIR /tmp +RUN rm -rf /tmp/crow +# allow tests to be statically linked +# careful with this and libraries that dlopen() internally. it may not be as +# portable as you assume ;-) -FROM sdapi-dev AS sdapi-cicd +RUN yum install -y -q glibc-static # finalize the build + +# fix issue with the base container we are using... +# root's home dir /opt/app-root ( duh ) does nto have proper owner +RUN chown -R root:root /opt/app-root/ + WORKDIR / + diff --git a/devops/docker/build.centos7.staticdeps.dockerfile b/devops/docker/build.centos7.staticdeps.dockerfile index a4953e59a56ce564bd260b583b697a41c64d7625..c80753d4c86566c91368f8da3fb67007a4a709cf 100644 --- a/devops/docker/build.centos7.staticdeps.dockerfile +++ b/devops/docker/build.centos7.staticdeps.dockerfile @@ -20,8 +20,8 @@ FROM centos/devtoolset-7-toolchain-centos7 AS dev # build as root USER 0 -# build dependencies -ARG CMAKE_VERSION=3.19.1 +# library path +ARG LIBRARIES_INSTALL_PATH=/usr/local # install required build tools via packet manager RUN yum update -y && yum install -y wget which git libtool autoconf perl-devel python-devel xz-devel patch @@ -30,6 +30,7 @@ RUN yum update -y && yum install -y wget which git libtool autoconf perl-devel WORKDIR /tmp # [CMAKE] +ARG CMAKE_VERSION=3.19.4 RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ && tar -zxvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -C /usr/local \ && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/bin/cmake \ @@ -38,409 +39,153 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cm && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/ccmake /usr/bin/ccmake \ && rm -rf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -FROM dev as rest-dev - -# library dependencies -ARG LIB_ZLIB_VERSION=1.2.11 -ARG LIB_OPENSSL_VERSION=1.1.1g -ARG LIB_CURL_VERSION=7.74.0 -ARG LIB_UUID_VERSION=1.0.3 - - -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local - - -# +# ---------------------------------------------------------------------------------------------------------------------------- # if we want to avoid adding shared libraries as depdendencies ( since in linux most of them export all their symbols) # we need to see if static libraries with Position Indepdent Code are built ( aka, .a that can be used to build .so later on) # # the analysis is done with : # for slib in /usr/local/lib*/*.a; do echo $slib; readelf --relocs ${slib} | egrep -c '(GOT|PLT|JUMP_SLO)'; done # -# aka : for all archives, read relocations, see if they are pointing to the global offset table, the procedure linkage table or are calls to shared libraries ( jump ) -# not quite sure this is exhaustive as this indicates _some relocatable code present_ not absence of relocations without indirections. +# aka : for all archives, read relocations, see if they are pointing to the global offset table, +# the procedure linkage table or are calls to shared libraries ( jump ) +# not quite sure this is exhaustive as this indicates _some relocatable code present_ +# not absence of relocations without indirections. # # otherwise we'd have to play with visibiliy at library build time, linker version scripts etc. +# ---------------------------------------------------------------------------------------------------------------------------- + +FROM dev as rest-dev # [ZLIB] -# build both static libraries (without PIC) and shared libraries. +ARG LIB_ZLIB_VERSION=1.2.11 RUN wget https://www.zlib.net/zlib-${LIB_ZLIB_VERSION}.tar.gz \ && tar -zxvf zlib-${LIB_ZLIB_VERSION}.tar.gz \ - && cd /tmp/zlib-${LIB_ZLIB_VERSION} \ + && cd zlib-${LIB_ZLIB_VERSION} \ && CFLAGS=-fPIC ./configure --prefix=${LIBRARIES_INSTALL_PATH} --static \ - && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf zlib-${LIB_ZLIB_VERSION}.tar.gz \ + && make -j $(nproc) \ + && make install \ + && cd .. \ + && rm -rf zlib-${LIB_ZLIB_VERSION}.tar.gz \ && rm -rf zlib-${LIB_ZLIB_VERSION} # [OPENSSL] -# uuuuh. this has TCC implications does it not ? not making it static then. -# build it statically anyways... +# this has TCC implications +# consider instead RUN apt-get install openssl-libs openssl-dev +ARG LIB_OPENSSL_VERSION=1.1.1g RUN wget https://www.openssl.org/source/openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && tar -zxvf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && cd /tmp/openssl-${LIB_OPENSSL_VERSION} \ && ./config --prefix=${LIBRARIES_INSTALL_PATH} no-shared \ && make -j $(nproc) \ - && make install_sw -WORKDIR /tmp -RUN rm -rf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ + && make install_sw \ + && cd .. \ + rm -rf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && rm -rf openssl-${LIB_OPENSSL_VERSION} -# consider instead -# RUN yum install -y openssl-libs.x86_64 openssl-devel.x86_64 -# and only use system ones - # [UUID] -# this builds both static libraries ( without fPIC ) and shared libraries -RUN wget https://vorboss.dl.sourceforge.net/project/libuuid/libuuid-${LIB_UUID_VERSION}.tar.gz \ +ARG LIB_UUID_VERSION=1.0.3 +RUN wget https://sourceforge.net/projects/libuuid/files/libuuid-${LIB_UUID_VERSION}.tar.gz \ && tar -zxvf libuuid-${LIB_UUID_VERSION}.tar.gz \ && cd /tmp/libuuid-${LIB_UUID_VERSION} \ && ./configure --prefix=${LIBRARIES_INSTALL_PATH} --with-pic --enable-shared=no --enable-static=yes \ && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf libuuid-${LIB_UUID_VERSION}.tar.gz \ + && make install \ + && cd .. \ + && rm -rf libuuid-${LIB_UUID_VERSION}.tar.gz \ && rm -rf libuuid-${LIB_UUID_VERSION} # [CURL] # this is not equivalent to the system one built with libtool ... soname, versions are not done the same way. # may require a patch for locally built nghttp2? or http3 libraries -RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive -WORKDIR /tmp/curl/build -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. -RUN make -j $(nproc) && make install -WORKDIR /tmp -RUN rm -rf curl +ARG LIB_CURL_VERSION=7.74.0 +RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive \ + && cd curl \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. \ + && make -j $(nrpoc) \ + && make install \ + && cd ../.. \ + && rm -rf curl FROM rest-dev as azure-dev -ARG LIB_BOOST_VERSION=1.73.0 -ARG LIB_XML2_VERSION=2.9.10 -ARG LIB_CPPREST_VERSION=2.10.16 -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local - # [BOOST] +ARG LIB_BOOST_VERSION=1.73.0 RUN wget https://dl.bintray.com/boostorg/release/${LIB_BOOST_VERSION}/source/boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && tar -zxvf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && cd boost_$(echo "$LIB_BOOST_VERSION" | tr . _) \ && ./bootstrap.sh --prefix=${LIBRARIES_INSTALL_PATH} \ && ./b2 stage -j $(nproc) threading=multi link=static cxxflags='-fPIC' linkflags='-pie' -q \ && ./b2 install threading=multi link=static cxxflags='-fPIC' linkflags='-pie' \ - && ln -svf detail/sha1.hpp ${LIBRARIES_INSTALL_PATH}/include/boost/uuid/sha1.hpp -WORKDIR /tmp -RUN rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ + && ln -svf detail/sha1.hpp ${LIBRARIES_INSTALL_PATH}/include/boost/uuid/sha1.hpp \ + && cd .. \ + && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _) +# [XZ] +# because we need liblzma that can be found and consumed by CMake +ARG LIB_XZ_COMMIT=e7da44d +RUN git clone https://git.tukaani.org/xz.git xz \ + && cd xz \ + && git checkout ${LIB_XZ_COMMIT} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=NO .. \ + && make -j $(nproc ) \ + && make install \ + && cd ../.. \ + && rm -rf xz # [XML2] +ARG LIB_XML2_VERSION=2.9.10 RUN wget https://github.com/GNOME/libxml2/archive/v${LIB_XML2_VERSION}.tar.gz \ && tar -zxvf v${LIB_XML2_VERSION}.tar.gz \ - && cd /tmp/libxml2-${LIB_XML2_VERSION} \ + && cd libxml2-${LIB_XML2_VERSION} \ && ./autogen.sh --prefix=${LIBRARIES_INSTALL_PATH} --with-pic --enable-static=yes --enable-shared=no \ && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf v${LIB_XML2_VERSION}.tar.gz && rm -rf libxml2-${LIB_XML2_VERSION} - + && make install \ + && cd .. \ + && rm -rf v${LIB_XML2_VERSION}.tar.gz \ + && rm -rf libxml2-${LIB_XML2_VERSION} # [CCPRESTSDK] -RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive -WORKDIR /tmp/cpprestsdk/build -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON .. -RUN make -j $(nproc ) && make install -WORKDIR /tmp -RUN rm -rf cpprestsdk - - -FROM azure-dev as azure-storage-dev - -ARG LIB_AZURE_STORAGE_VERSION=7.5.0 -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local +ARG LIB_CPPREST_VERSION=2.10.16 +RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive \ + && cd cpprestsdk \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON -DBUILD_SAMPLES=OFF .. \ + && make -j $(nrpoc) \ + && make install \ + && cd ../.. \ + && rm -rf cpprestsdk # [AZURESTORAGECPP] -# most likely needs a patch to build properly with non-system curl found. -RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive -WORKDIR /tmp/azure-storage-cpp/Microsoft.WindowsAzure.Storage/build.release -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON .. -RUN make -j $(nproc) && make install -WORKDIR /tmp -RUN rm -rf azure-storage-cpp - - -# packages depdending on libcurl - -FROM azure-storage-dev AS aws-dev - -ARG AWS_SDK_CPP_RELEASE_TAG=1.8.123 -# library path : args have to be repeated after every FROM ? or maybe I just need to declare it before the very first one? -ARG LIBRARIES_INSTALL_PATH=/usr/local - -# it requires a patch... -# [AWSCPPSDK] -# these are tags not branches. -#RUN git clone -b ${AWS_SDK_CPP_RELEASE_TAG} --recursive https://github.com/aws/aws-sdk-cpp aws-sdk-cpp -# a patch is required to correctly build non-system CURL built with CMake -ARG AWS_SDK_CPP_PATCHED_COMMIT=95b4403042458bb1de9697052d6da43c09579ee8 -RUN git clone --recursive https://github.com/alichnewsky/aws-sdk-cpp aws-sdk-cpp -WORKDIR /tmp/aws-sdk-cpp -RUN git checkout ${AWS_SDK_CPP_PATCHED_COMMIT} -WORKDIR /tmp/aws-sdk-cpp/build -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DBUILD_ONLY="s3" \ - .. -RUN make -j $(nproc) && make install -WORKDIR /tmp -RUN rm -rf aws-sdk-cpp - -FROM aws-dev AS gcloud-dev - -# plenty of patches in order to build from cmake packages. -# pretty old, to be reviewed. - -ARG LIB_CRC32C_VERSION=1.0.6 -ARG LIB_C_ARES_VERSION=1_14_0 -ARG LIB_NGHTTP2_RELEASE=1.42.0 -ARG LIB_PROTOBUF_VERSION=3.11.3 -ARG LIB_ABSEIL_RELEASE=20200225.2 -ARG LIB_GRPC_VERSION=1.29.1 - -# library path : args have to be repeated after every FROM ? or maybe I just need to declare it before the very first one? -ARG LIBRARIES_INSTALL_PATH=/usr/local - -# review sdapi build because of this. - -# [ CRC32C ] -WORKDIR /tmp -RUN wget -q https://github.com/google/crc32c/archive/${LIB_CRC32C_VERSION}.tar.gz -RUN tar -xf ${LIB_CRC32C_VERSION}.tar.gz - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCRC32C_BUILD_TESTS=OFF \ - -DCRC32C_BUILD_BENCHMARKS=OFF \ - -DCRC32C_USE_GLOG=OFF \ - -H/tmp/crc32c-${LIB_CRC32C_VERSION} -B/tmp/crc32c-${LIB_CRC32C_VERSION}/cmake-out/crc32c - -RUN cmake --build /tmp/crc32c-${LIB_CRC32C_VERSION}/cmake-out/crc32c \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/${LIB_CRC32C_VERSION}.tar.gz -RUN rm -rf /tmp/crc32c-${LIB_CRC32C_VERSION} - -# do I need this for a static lib ? -# centos does not have this -# as a result protoc, grpc code generators don't run... -# otherwise you need ENV LD_LIBRARY_PATH set... at build only... -#RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf -#RUN ldconfig - -# [C-ARES] : if we rebuild a modern one, it should go into our build of CURL as well, not just nghttp2 for gRPC - -RUN wget -q https://github.com/c-ares/c-ares/archive/cares-${LIB_C_ARES_VERSION}.tar.gz -RUN tar -xf cares-${LIB_C_ARES_VERSION}.tar.gz - -# this does not install c-ares as a CMake package -# -#RUN ./buildconf && ./configure && make -j $( nproc ) -#RUN make install -# installs in /usr/local/lib : not in /etc/ld.so.conf.d/local.conf - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DCARES_STATIC=ON \ - -DCARES_SHARED=OFF \ - -DCARES_STATIC_PIC=ON \ - -H/tmp/c-ares-cares-${LIB_C_ARES_VERSION} \ - -B/tmp/c-ares-cares-${LIB_C_ARES_VERSION}/cmake-out - -RUN cmake --build /tmp/c-ares-cares-${LIB_C_ARES_VERSION}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/cares-${LIB_C_ARES_VERSION}.tar.gz -RUN rm -rf /tmp/c-ares-cares-${LIB_C_ARES_VERSION} - -RUN echo $(which curl) -# centos does not have this -# as a result protoc, grpc code generators don't run... -# otherwise you need ENV LD_LIBRARY_PATH set... at build only... -RUN echo /usr/local/lib > /etc/ld.so.conf.d/local.conf -RUN echo /usr/local/lib64 >> /etc/ld.so.conf.d/local.conf -RUN ldconfig - -RUN curl -sL https://github.com/nghttp2/nghttp2/releases/download/v${LIB_NGHTTP2_RELEASE}/nghttp2-${LIB_NGHTTP2_RELEASE}.tar.gz -o nghttp2.tar.gz -RUN tar -xf nghttp2.tar.gz - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DENABLE_STATIC_LIB=ON \ - -DENABLE_SHARED_LIB=OFF \ - -DENABLE_LIB_ONLY=ON \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DENABLE_ARES=ON \ - -H/tmp/nghttp2-${LIB_NGHTTP2_RELEASE} \ - -B/tmp/nghttp2-${LIB_NGHTTP2_RELEASE}/cmake-out - -RUN cmake --build /tmp/nghttp2-${LIB_NGHTTP2_RELEASE}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/nghttp2.tar.gz -RUN rm -rf /tmp/nghttp2-${LIB_NGHTTP2_RELEASE} - -# rebuld CURL with ENABLE_ARES=ON USE_NGHTTP2=ON ? - -RUN wget -q https://github.com/google/protobuf/archive/v${LIB_PROTOBUF_VERSION}.tar.gz -RUN tar -xf v${LIB_PROTOBUF_VERSION}.tar.gz -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -Dprotobuf_BUILD_TESTS=OFF \ - -H/tmp/protobuf-${LIB_PROTOBUF_VERSION}/cmake \ - -B/tmp/protobuf-${LIB_PROTOBUF_VERSION}/cmake/cmake-out - -RUN cmake --build /tmp/protobuf-${LIB_PROTOBUF_VERSION}/cmake/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/v${LIB_PROTOBUF_VERSION}.tar.gz -RUN rm -rf /tmp/protobuf-${LIB_PROTOBUF_VERSION} - - -RUN curl -sL https://github.com/abseil/abseil-cpp/archive/${LIB_ABSEIL_RELEASE}.tar.gz -o ${LIB_ABSEIL_RELEASE}.tar.gz -RUN tar xf ${LIB_ABSEIL_RELEASE}.tar.gz -RUN rm -rf ${LIB_ABSEIL_RELEASE}.tar.gz - - -# 2020-06-12 -# google-cloud-cpp super build says absl/base/options.h needs to be patched -# -RUN perl -pi -e 's@#define ABSL_OPTION_USE_STD_ANY 2@#define ABSL_OPTION_USE_STD_ANY 0@g' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h -RUN perl -pi -e 's@#define ABSL_OPTION_USE_STD_OPTIONAL 2@#define ABSL_OPTION_USE_STD_OPTIONAL 0@g' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h -RUN perl -pi -e 's@#define ABSL_OPTION_USE_STD_STRING_VIEW 2@#define ABSL_OPTION_USE_STD_STRING_VIEW 0@g' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h -RUN perl -pi -e 's@#define ABSL_OPTION_USE_STD_VARIANT 2@#define ABSL_OPTION_USE_STD_VARIANT 0@g' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h -# verify the patch worked. -RUN [[ "$( egrep '^#define ABSL_OPTION_USE_STD_(.*) 2$' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h |wc -l )" == "0" ]] - - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_TESTING=OFF \ - -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -H/tmp/abseil-cpp-${LIB_ABSEIL_RELEASE} \ - -B/tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/cmake-out - -RUN cmake --build /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE} - -RUN curl -sL https://github.com/grpc/grpc/archive/v${LIB_GRPC_VERSION}.tar.gz -o v${LIB_GRPC_VERSION}.tar.gz -RUN tar -xf v${LIB_GRPC_VERSION}.tar.gz -RUN rm -rf v${LIB_GRPC_VERSION}.tar.gz - -# needed? -#ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig -#ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 -#ENV PATH=/usr/local/bin:${PATH} -# this does not install gRPC as a CMake package... -# -#RUN make -j $( nproc ) -#RUN make install - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DgRPC_BUILD_TESTS=OFF \ - -DgRPC_ZLIB_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package \ - -DgRPC_CARES_PROVIDER=package \ - -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_ABSL_PROVIDER=package \ - -H/tmp/grpc-${LIB_GRPC_VERSION} \ - -B/tmp/grpc-${LIB_GRPC_VERSION}/cmake-out - -RUN cmake --build /tmp/grpc-${LIB_GRPC_VERSION}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/grpc-${LIB_GRPC_VERSION} - - -FROM gcloud-dev AS gcs-dev -ARG LIB_NLOHMANN_JSON_RELEASE_TAG=v3.9.1 -ARG GOOGLE_CLOUD_CPP_RELEASE_TAG=1.22.0 -# library path : args have to be repeated after every FROM ? or maybe I just need to declare it before the very first one? -ARG LIBRARIES_INSTALL_PATH=/usr/local - -WORKDIR /tmp - -# this is a header-only library so, we don't ask for static libs with position independent code -RUN git clone --depth 1 --branch ${LIB_NLOHMANN_JSON_RELEASE_TAG} https://github.com/nlohmann/json -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -H/tmp/json -B/tmp/json/cmake-out -RUN cmake --build /tmp/json/cmake-out --target install -- -j $(nproc) -RUN rm -rf /tmp/json - - -# only build google-cloud-cpp with the storage_client api. no other, from code generation or otherwise - -# some benchmarks require CURL 7.64.0 features at compilation time but do not check. -# this is OK in this build _because_ we have a newer static libcurl - -RUN curl -sL https://github.com/googleapis/google-cloud-cpp/archive/v${GOOGLE_CLOUD_CPP_RELEASE_TAG}.tar.gz -o ${GOOGLE_CLOUD_CPP_RELEASE_TAG}.tar.gz -RUN tar -xf ${GOOGLE_CLOUD_CPP_RELEASE_TAG}.tar.gz -RUN rm -rf ${GOOGLE_CLOUD_CPP_RELEASE_TAG}.tar.gz - -# to run with GRPC in GOOGLE CLOUD STORAGE it looks like it needs a patch - -# NOT YET CONTRIBUTED -#RUN perl -pi -e 's/google_cloud_cpp_proto_library\(/include\(CompileProtos\)\ngoogle_cloud_cpp_proto_library\(/g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/google/cloud/storage/CMakeLists.txt -#RUN perl -pi -e 's/if \(\$\{GOOGLE_CLOUD_CPP_ENABLE_GRPC\}\)/if \(\$\{GOOGLE_CLOUD_CPP_ENABLE_GRPC\} OR \$\{GOOGLE_CLOUD_CPP_STORAGE_ENABLE_GRPC\}\)/g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/CMakeLists.txt -RUN perl -pi -e 's/foreach \(_library /foreach (_library storage /g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/CMakeLists.txt -RUN perl -pi -e 's/google_cloud_cpp_proto_library\(/file\(\MAKE_DIRECTORY \$\{CMAKE_CURRENT_BINARY_DIR\}\/internal)\ngoogle_cloud_cpp_proto_library\(/g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/google/cloud/storage/CMakeLists.txt -RUN perl -pi -e 's/ LOCAL_INCLUDE/ \$\{CMAKE_CURRENT_SOURCE_DIR\}\/internal LOCAL_INCLUDE/g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/google/cloud/storage/CMakeLists.txt - -RUN cmake \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DGOOGLE_CLOUD_CPP_DEPENDENCY_PROVIDER=package \ - -DGOOGLE_CLOUD_CPP_GMOCK_PROVIDER=external \ - -DGOOGLE_CLOUD_CPP_ENABLE=storage \ - -DBUILD_TESTING=OFF \ - -DGOOGLE_CLOUD_CPP_STORAGE_ENABLE_GRPC=ON \ - -H/tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG} \ - -B/tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/cmake-out - -RUN cmake --build /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG} - -FROM gcs-dev AS sdapi-dev - - - - -FROM sdapi-dev AS sdapi-cicd +ARG LIB_AZURE_STORAGE_VERSION=7.5.0 +RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive \ + && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage \ + && mkdir build.release \ + && cd build.release \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON .. \ + && make -j $(nproc) \ + && make install \ + && cd ../../.. \ + && rm -rf azure-storage-cpp + +# [AZURE-STORAGE-FOR-CPP] +ARG AZURE_STORAGE_FOR_CPP_VERSION=12.0.0-beta.10 +RUN wget https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && tar -zxvf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && cd azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=NO -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DBUILD_TRANSPORT_CURL:BOOL=ON .. \ + && make -j $(nproc) \ + && make install \ + && rm -rf azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && rm -rf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz # finalize the build WORKDIR / + diff --git a/devops/docker/build.centos8.dockerfile b/devops/docker/build.centos8.dockerfile deleted file mode 100644 index e043a613e543b374f5481460854a94959caa7396..0000000000000000000000000000000000000000 --- a/devops/docker/build.centos8.dockerfile +++ /dev/null @@ -1,143 +0,0 @@ -# ============================================================================ -# Copyright 2017-2021, Schlumberger -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -# base image -FROM centos:8 - -# build as root -USER 0 - -# build dependencies -ARG CMAKE_VERSION=3.19.1 - -# library dependencies -ARG LIB_ZLIB_VERSION=1.2.11 -ARG LIB_OPENSSL_VERSION=1.1.1g -ARG LIB_CURL_VERSION=7.73.0 -ARG LIB_BOOST_VERSION=1.73.0 -ARG LIB_UUID_VERSION=1.0.3 -ARG LIB_XML2_VERSION=2.9.10 -ARG LIB_CPPREST_VERSION=2.10.16 -ARG LIB_AZURE_STORAGE_VERSION=7.5.0 - -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local - -# install required build tools via packet manager -RUN dnf update -y && dnf install -y wget git gcc-c++ make libtool which xz-devel patch - -# temporary working directory -WORKDIR /tmp - -# [CMAKE] -RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ - && tar -zxvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -C /usr/local \ - && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/bin/cmake \ - && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/ctest /usr/bin/ctest \ - && rm -rf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz - -# [ZLIB] -RUN wget https://www.zlib.net/zlib-${LIB_ZLIB_VERSION}.tar.gz \ - && tar -zxvf zlib-${LIB_ZLIB_VERSION}.tar.gz \ - && cd zlib-${LIB_ZLIB_VERSION} \ - && ./configure --prefix=${LIBRARIES_INSTALL_PATH} \ - && make -j $(nproc) \ - && make install \ - && cd .. \ - && rm -rf zlib-${LIB_ZLIB_VERSION}.tar.gz \ - && rm -rf zlib-${LIB_ZLIB_VERSION} - -# [OPENSSL] -RUN wget https://www.openssl.org/source/openssl-${LIB_OPENSSL_VERSION}.tar.gz \ - && tar -zxvf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ - && cd openssl-${LIB_OPENSSL_VERSION} \ - && ./config --prefix=${LIBRARIES_INSTALL_PATH} \ - && make -j $(nproc) \ - && make install_sw \ - && cd .. \ - && rm -rf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ - && rm -rf openssl-${LIB_OPENSSL_VERSION} - -# [BOOST] -RUN wget https://dl.bintray.com/boostorg/release/${LIB_BOOST_VERSION}/source/boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ - && tar -zxvf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ - && cd boost_$(echo "$LIB_BOOST_VERSION" | tr . _) \ - && ./bootstrap.sh --prefix=${LIBRARIES_INSTALL_PATH} \ - && ./b2 stage -j $(nproc) threading=multi link=shared -q \ - && ./b2 install threading=multi link=shared \ - && ln -svf detail/sha1.hpp ${LIBRARIES_INSTALL_PATH}/include/boost/uuid/sha1.hpp \ - && cd .. \ - && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ - && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _) - -# [UUID] -RUN wget https://vorboss.dl.sourceforge.net/project/libuuid/libuuid-${LIB_UUID_VERSION}.tar.gz \ - && tar -zxvf libuuid-${LIB_UUID_VERSION}.tar.gz \ - && cd libuuid-${LIB_UUID_VERSION} \ - && ./configure --prefix=${LIBRARIES_INSTALL_PATH} \ - && make -j $(nproc) \ - && make install \ - && cd .. \ - && rm -rf libuuid-${LIB_UUID_VERSION}.tar.gz \ - && rm -rf libuuid-${LIB_UUID_VERSION} - -# [XML2] -RUN wget https://github.com/GNOME/libxml2/archive/v${LIB_XML2_VERSION}.tar.gz \ - && tar -zxvf v${LIB_XML2_VERSION}.tar.gz \ - && cd libxml2-${LIB_XML2_VERSION} \ - && ./autogen.sh --prefix=${LIBRARIES_INSTALL_PATH} \ - && make -j $(nproc) \ - && make install \ - && cd .. \ - && rm -rf v${LIB_XML2_VERSION}.tar.gz \ - && rm -rf libxml2-${LIB_XML2_VERSION} - -# [CCPRESTSDK] -RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive \ - && cd cpprestsdk \ - && mkdir build \ - && cd build \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} .. \ - && make -j $(nrpoc) \ - && make install \ - && cd ../.. \ - && rm -rf cpprestsdk - -# [AZURESTORAGECPP] -RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive \ - && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage \ - && mkdir build.release \ - && cd build.release \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} .. \ - && make -j $(nproc) \ - && make install \ - && cd ../../.. \ - && rm -rf azure-storage-cpp - -# [CURL] -RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive \ - && cd curl \ - && mkdir build \ - && cd build \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} .. \ - && make -j $(nrpoc) \ - && make install \ - && cd ../.. \ - && rm -rf curl - -# finalize the build -WORKDIR / -USER 1001 \ No newline at end of file diff --git a/devops/docker/build.ubuntu.dockerfile b/devops/docker/build.ubuntu.dockerfile index f2a115bc986c38a74b304ff6ece54a156bc9898c..b70c3cd09cbdaf775e740b56a9de62ffae42db2f 100644 --- a/devops/docker/build.ubuntu.dockerfile +++ b/devops/docker/build.ubuntu.dockerfile @@ -18,19 +18,6 @@ FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive -# build dependencies -ARG CMAKE_VERSION=3.19.1 - -# library dependencies -ARG LIB_ZLIB_VERSION=1.2.11 -ARG LIB_OPENSSL_VERSION=1.1.1g -ARG LIB_CURL_VERSION=7.73.0 -ARG LIB_BOOST_VERSION=1.73.0 -ARG LIB_UUID_VERSION=1.0.3 -ARG LIB_XML2_VERSION=2.9.10 -ARG LIB_CPPREST_VERSION=2.10.16 -ARG LIB_AZURE_STORAGE_VERSION=7.5.0 - # library path ARG LIBRARIES_INSTALL_PATH=/usr/local @@ -41,13 +28,15 @@ RUN apt-get -y update && apt-get -y install wget git g++ make libtool pkgconf li WORKDIR /tmp # [CMAKE] -RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ +ARG CMAKE_VERSION=3.19.4 +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz --no-check-certificate \ && tar -zxvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -C /usr/local \ && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/bin/cmake \ && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/ctest /usr/bin/ctest \ && rm -rf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz # [ZLIB] +ARG LIB_ZLIB_VERSION=1.2.11 RUN wget https://www.zlib.net/zlib-${LIB_ZLIB_VERSION}.tar.gz \ && tar -zxvf zlib-${LIB_ZLIB_VERSION}.tar.gz \ && cd zlib-${LIB_ZLIB_VERSION} \ @@ -59,6 +48,7 @@ RUN wget https://www.zlib.net/zlib-${LIB_ZLIB_VERSION}.tar.gz \ && rm -rf zlib-${LIB_ZLIB_VERSION} # [OPENSSL] +ARG LIB_OPENSSL_VERSION=1.1.1g RUN wget https://www.openssl.org/source/openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && tar -zxvf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && cd openssl-${LIB_OPENSSL_VERSION} \ @@ -70,6 +60,7 @@ RUN wget https://www.openssl.org/source/openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && rm -rf openssl-${LIB_OPENSSL_VERSION} # [BOOST] +ARG LIB_BOOST_VERSION=1.73.0 RUN wget https://dl.bintray.com/boostorg/release/${LIB_BOOST_VERSION}/source/boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && tar -zxvf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && cd boost_$(echo "$LIB_BOOST_VERSION" | tr . _) \ @@ -82,7 +73,8 @@ RUN wget https://dl.bintray.com/boostorg/release/${LIB_BOOST_VERSION}/source/boo && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _) # [UUID] -RUN wget https://vorboss.dl.sourceforge.net/project/libuuid/libuuid-${LIB_UUID_VERSION}.tar.gz \ +ARG LIB_UUID_VERSION=1.0.3 +RUN wget https://sourceforge.net/projects/libuuid/files/libuuid-${LIB_UUID_VERSION}.tar.gz \ && tar -zxvf libuuid-${LIB_UUID_VERSION}.tar.gz \ && cd libuuid-${LIB_UUID_VERSION} \ && ./configure --prefix=${LIBRARIES_INSTALL_PATH} \ @@ -92,7 +84,8 @@ RUN wget https://vorboss.dl.sourceforge.net/project/libuuid/libuuid-${LIB_UUID_V && rm -rf libuuid-${LIB_UUID_VERSION}.tar.gz \ && rm -rf libuuid-${LIB_UUID_VERSION} -# [XML2] +# [LIBXML2] +ARG LIB_XML2_VERSION=2.9.10 RUN wget https://github.com/GNOME/libxml2/archive/v${LIB_XML2_VERSION}.tar.gz \ && tar -zxvf v${LIB_XML2_VERSION}.tar.gz \ && cd libxml2-${LIB_XML2_VERSION} \ @@ -103,7 +96,8 @@ RUN wget https://github.com/GNOME/libxml2/archive/v${LIB_XML2_VERSION}.tar.gz \ && rm -rf v${LIB_XML2_VERSION}.tar.gz \ && rm -rf libxml2-${LIB_XML2_VERSION} -# [CCPRESTSDK] +# [CPPREST] +ARG LIB_CPPREST_VERSION=2.10.16 RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive \ && cd cpprestsdk \ && mkdir build \ @@ -114,7 +108,8 @@ RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk && cd ../.. \ && rm -rf cpprestsdk -# [AZURESTORAGECPP] +# [AZURE-STORAGE-CPP] +ARG LIB_AZURE_STORAGE_VERSION=7.5.0 RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive \ && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage \ && mkdir build.release \ @@ -126,6 +121,7 @@ RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-st && rm -rf azure-storage-cpp # [CURL] +ARG LIB_CURL_VERSION=7.73.0 RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive \ && cd curl \ && mkdir build \ @@ -136,6 +132,19 @@ RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/cu && cd ../.. \ && rm -rf curl +# [AZURE-STORAGE-FOR-CPP] +ARG AZURE_STORAGE_FOR_CPP_VERSION=12.0.0-beta.10 +RUN wget https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && tar -zxvf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && cd azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=YES .. \ + && make -j $(nproc) \ + && make install \ + && rm -rf azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && rm -rf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz + # finalize WORKDIR / ENV LD_LIBRARY_PATH=${LIBRARIES_INSTALL_PATH}/lib:$LD_LIBRARY_PATH diff --git a/devops/docker/build.ubuntu.staticdeps.dockerfile b/devops/docker/build.ubuntu.staticdeps.dockerfile index 9ebdab13a90f7b0c99a934aa9cb65121dcc6ae12..fa034c98946cd5ff7b95e51c703a12a392835e68 100644 --- a/devops/docker/build.ubuntu.staticdeps.dockerfile +++ b/devops/docker/build.ubuntu.staticdeps.dockerfile @@ -20,8 +20,8 @@ FROM ubuntu:20.04 AS dev # build as root USER 0 -# build dependencies -ARG CMAKE_VERSION=3.19.1 +# library path +ARG LIBRARIES_INSTALL_PATH=/usr/local # install required build tools via packet manager RUN apt-get -y update && apt-get -y install wget git g++ make libtool pkgconf liblzma-dev @@ -30,6 +30,7 @@ RUN apt-get -y update && apt-get -y install wget git g++ make libtool pkgconf li WORKDIR /tmp # [CMAKE] +ARG CMAKE_VERSION=3.19.4 RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ && tar -zxvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -C /usr/local \ && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/bin/cmake \ @@ -38,412 +39,138 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cm && ln -s /usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/ccmake /usr/bin/ccmake \ && rm -rf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -FROM dev as rest-dev - -# library dependencies -ARG LIB_ZLIB_VERSION=1.2.11 -ARG LIB_OPENSSL_VERSION=1.1.1g -ARG LIB_CURL_VERSION=7.74.0 -ARG LIB_UUID_VERSION=1.0.3 - - -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local - - -# +# ---------------------------------------------------------------------------------------------------------------------------- # if we want to avoid adding shared libraries as depdendencies ( since in linux most of them export all their symbols) # we need to see if static libraries with Position Indepdent Code are built ( aka, .a that can be used to build .so later on) # # the analysis is done with : # for slib in /usr/local/lib*/*.a; do echo $slib; readelf --relocs ${slib} | egrep -c '(GOT|PLT|JUMP_SLO)'; done # -# aka : for all archives, read relocations, see if they are pointing to the global offset table, the procedure linkage table or are calls to shared libraries ( jump ) -# not quite sure this is exhaustive as this indicates _some relocatable code present_ not absence of relocations without indirections. +# aka : for all archives, read relocations, see if they are pointing to the global offset table, +# the procedure linkage table or are calls to shared libraries ( jump ) +# not quite sure this is exhaustive as this indicates _some relocatable code present_ +# not absence of relocations without indirections. # # otherwise we'd have to play with visibiliy at library build time, linker version scripts etc. +# ---------------------------------------------------------------------------------------------------------------------------- + +FROM dev as rest-dev # [ZLIB] -# build both static libraries (without PIC) and shared libraries. +ARG LIB_ZLIB_VERSION=1.2.11 RUN wget https://www.zlib.net/zlib-${LIB_ZLIB_VERSION}.tar.gz \ && tar -zxvf zlib-${LIB_ZLIB_VERSION}.tar.gz \ - && cd /tmp/zlib-${LIB_ZLIB_VERSION} \ + && cd zlib-${LIB_ZLIB_VERSION} \ && CFLAGS=-fPIC ./configure --prefix=${LIBRARIES_INSTALL_PATH} --static \ && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf zlib-${LIB_ZLIB_VERSION}.tar.gz \ + && make install \ + && cd .. \ + && rm -rf zlib-${LIB_ZLIB_VERSION}.tar.gz \ && rm -rf zlib-${LIB_ZLIB_VERSION} # [OPENSSL] -# uuuuh. this has TCC implications does it not ? not making it static then. -# build it statically anyways... +# this has TCC implications +# consider instead RUN apt-get install openssl-libs openssl-dev +ARG LIB_OPENSSL_VERSION=1.1.1g RUN wget https://www.openssl.org/source/openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && tar -zxvf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ - && cd /tmp/openssl-${LIB_OPENSSL_VERSION} \ + && cd openssl-${LIB_OPENSSL_VERSION} \ && ./config --prefix=${LIBRARIES_INSTALL_PATH} no-shared \ && make -j $(nproc) \ - && make install_sw -WORKDIR /tmp -RUN rm -rf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ + && make install_sw \ + && cd .. \ + && rm -rf openssl-${LIB_OPENSSL_VERSION}.tar.gz \ && rm -rf openssl-${LIB_OPENSSL_VERSION} -# consider instead -# RUN apt-get install openssl-libs openssl-dev -# and only use system ones - # [UUID] -# this builds both static libraries ( without fPIC ) and shared libraries -RUN wget https://vorboss.dl.sourceforge.net/project/libuuid/libuuid-${LIB_UUID_VERSION}.tar.gz \ +ARG LIB_UUID_VERSION=1.0.3 +RUN wget https://sourceforge.net/projects/libuuid/files/libuuid-${LIB_UUID_VERSION}.tar.gz \ && tar -zxvf libuuid-${LIB_UUID_VERSION}.tar.gz \ && cd /tmp/libuuid-${LIB_UUID_VERSION} \ && ./configure --prefix=${LIBRARIES_INSTALL_PATH} --with-pic --enable-shared=no --enable-static=yes \ && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf libuuid-${LIB_UUID_VERSION}.tar.gz \ + && make install \ + && cd .. \ + && rm -rf libuuid-${LIB_UUID_VERSION}.tar.gz \ && rm -rf libuuid-${LIB_UUID_VERSION} # [CURL] # this is not equivalent to the system one built with libtool ... soname, versions are not done the same way. -# may require a patch for locally built nghttp2? or http3 libraries -RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive -WORKDIR /tmp/curl/build -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. -RUN make -j $(nproc) && make install -WORKDIR /tmp -RUN rm -rf curl - -FROM rest-dev as azure-dev - -ARG LIB_BOOST_VERSION=1.73.0 -ARG LIB_XML2_VERSION=2.9.10 -ARG LIB_CPPREST_VERSION=2.10.16 -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local +# may require a patch for locally built nghttp2? or http3 librarie +ARG LIB_CURL_VERSION=7.74.0 +RUN git clone -b curl-$(echo "$LIB_CURL_VERSION" | tr . _) https://github.com/curl/curl.git curl --recursive \ + && cd curl \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. \ + && make -j $(nrpoc) \ + && make install \ + && cd ../.. \ + && rm -rf curl + +FROM rest-dev as azure-storage-dev # [BOOST] +ARG LIB_BOOST_VERSION=1.73.0 RUN wget https://dl.bintray.com/boostorg/release/${LIB_BOOST_VERSION}/source/boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && tar -zxvf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && cd boost_$(echo "$LIB_BOOST_VERSION" | tr . _) \ && ./bootstrap.sh --prefix=${LIBRARIES_INSTALL_PATH} \ && ./b2 stage -j $(nproc) threading=multi link=static cxxflags='-fPIC' linkflags='-pie' -q \ - && ./b2 install threading=multi link=static cxxflags='-fPIC' linkflags='-pie' \ - && ln -svf detail/sha1.hpp ${LIBRARIES_INSTALL_PATH}/include/boost/uuid/sha1.hpp -WORKDIR /tmp -RUN rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ + && ./b2 install threading=multi link=static cxxflags='-fPIC' linkflags='-pie' \ + && ln -svf detail/sha1.hpp ${LIBRARIES_INSTALL_PATH}/include/boost/uuid/sha1.hpp \ + && cd .. \ + && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _).tar.gz \ && rm -rf boost_$(echo "$LIB_BOOST_VERSION" | tr . _) - # [XML2] +ARG LIB_XML2_VERSION=2.9.10 RUN wget https://github.com/GNOME/libxml2/archive/v${LIB_XML2_VERSION}.tar.gz \ && tar -zxvf v${LIB_XML2_VERSION}.tar.gz \ - && cd /tmp/libxml2-${LIB_XML2_VERSION} \ + && cd libxml2-${LIB_XML2_VERSION} \ && ./autogen.sh --prefix=${LIBRARIES_INSTALL_PATH} --with-pic --enable-static=yes --enable-shared=no \ && make -j $(nproc) \ - && make install -WORKDIR /tmp -RUN rm -rf v${LIB_XML2_VERSION}.tar.gz && rm -rf libxml2-${LIB_XML2_VERSION} - -# THERE IS AN ISSUE HERE WITH OpenSSL discovery via pkg-config or FindOpenSSL not adding libdl to the OpenSSL::SSL target -# and samples not building. + && make install \ + && cd .. \ + && rm -rf v${LIB_XML2_VERSION}.tar.gz \ + && rm -rf libxml2-${LIB_XML2_VERSION} # [CCPRESTSDK] -RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive -WORKDIR /tmp/cpprestsdk/build -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON -DBUILD_SAMPLES=OFF .. -RUN make -j $(nproc ) && make install -WORKDIR /tmp -RUN rm -rf cpprestsdk - - -FROM azure-dev as azure-storage-dev - -ARG LIB_AZURE_STORAGE_VERSION=7.5.0 -# library path -ARG LIBRARIES_INSTALL_PATH=/usr/local +ARG LIB_CPPREST_VERSION=2.10.16 +RUN git clone -b v${LIB_CPPREST_VERSION} https://github.com/microsoft/cpprestsdk.git cpprestsdk --recursive \ + && cd cpprestsdk \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON -DBUILD_SAMPLES=OFF .. \ + && make -j $(nrpoc) \ + && make install \ + && cd ../.. \ + && rm -rf cpprestsdk # [AZURESTORAGECPP] -# most likely needs a patch to build properly with non-system curl found. -RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive -WORKDIR /tmp/azure-storage-cpp/Microsoft.WindowsAzure.Storage/build.release -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON .. -RUN make -j $(nproc) && make install -WORKDIR /tmp -RUN rm -rf azure-storage-cpp - - -# packages depdending on libcurl - -FROM azure-storage-dev AS aws-dev - -ARG AWS_SDK_CPP_RELEASE_TAG=1.8.123 -# library path : args have to be repeated after every FROM ? or maybe I just need to declare it before the very first one? -ARG LIBRARIES_INSTALL_PATH=/usr/local - -# it requires a patch... -# [AWSCPPSDK] -# these are tags not branches. -#RUN git clone -b ${AWS_SDK_CPP_RELEASE_TAG} --recursive https://github.com/aws/aws-sdk-cpp aws-sdk-cpp -# a patch is required to correctly build non-system CURL built with CMake -ARG AWS_SDK_CPP_PATCHED_COMMIT=95b4403042458bb1de9697052d6da43c09579ee8 -RUN git clone --recursive https://github.com/alichnewsky/aws-sdk-cpp aws-sdk-cpp -WORKDIR /tmp/aws-sdk-cpp -RUN git checkout ${AWS_SDK_CPP_PATCHED_COMMIT} -WORKDIR /tmp/aws-sdk-cpp/build -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DBUILD_ONLY="s3" \ - .. -RUN make -j $(nproc) && make install -WORKDIR /tmp -RUN rm -rf aws-sdk-cpp - -FROM aws-dev AS gcloud-dev - -# plenty of patches in order to build from cmake packages. -# pretty old, to be reviewed. - -ARG LIB_CRC32C_VERSION=1.0.6 -ARG LIB_C_ARES_VERSION=1_14_0 -ARG LIB_NGHTTP2_RELEASE=1.42.0 -ARG LIB_PROTOBUF_VERSION=3.11.3 -ARG LIB_ABSEIL_RELEASE=20200225.2 -ARG LIB_GRPC_VERSION=1.29.1 - -# library path : args have to be repeated after every FROM ? or maybe I just need to declare it before the very first one? -ARG LIBRARIES_INSTALL_PATH=/usr/local - -# review sdapi build because of this. - -# [ CRC32C ] -WORKDIR /tmp -RUN wget -q https://github.com/google/crc32c/archive/${LIB_CRC32C_VERSION}.tar.gz -RUN tar -xf ${LIB_CRC32C_VERSION}.tar.gz - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCRC32C_BUILD_TESTS=OFF \ - -DCRC32C_BUILD_BENCHMARKS=OFF \ - -DCRC32C_USE_GLOG=OFF \ - -H/tmp/crc32c-${LIB_CRC32C_VERSION} -B/tmp/crc32c-${LIB_CRC32C_VERSION}/cmake-out/crc32c - -RUN cmake --build /tmp/crc32c-${LIB_CRC32C_VERSION}/cmake-out/crc32c \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/${LIB_CRC32C_VERSION}.tar.gz -RUN rm -rf /tmp/crc32c-${LIB_CRC32C_VERSION} - -# do I need this for a static lib ? -# centos does not have this -# as a result protoc, grpc code generators don't run... -# otherwise you need ENV LD_LIBRARY_PATH set... at build only... -#RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf -#RUN ldconfig - -# [C-ARES] : if we rebuild a modern one, it should go into our build of CURL as well, not just nghttp2 for gRPC - -RUN wget -q https://github.com/c-ares/c-ares/archive/cares-${LIB_C_ARES_VERSION}.tar.gz -RUN tar -xf cares-${LIB_C_ARES_VERSION}.tar.gz - -# this does not install c-ares as a CMake package -# -#RUN ./buildconf && ./configure && make -j $( nproc ) -#RUN make install -# installs in /usr/local/lib : not in /etc/ld.so.conf.d/local.conf - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DCARES_STATIC=ON \ - -DCARES_SHARED=OFF \ - -DCARES_STATIC_PIC=ON \ - -H/tmp/c-ares-cares-${LIB_C_ARES_VERSION} \ - -B/tmp/c-ares-cares-${LIB_C_ARES_VERSION}/cmake-out - -RUN cmake --build /tmp/c-ares-cares-${LIB_C_ARES_VERSION}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/cares-${LIB_C_ARES_VERSION}.tar.gz -RUN rm -rf /tmp/c-ares-cares-${LIB_C_ARES_VERSION} - -RUN echo $(which curl) -# centos does not have this -# as a result protoc, grpc code generators don't run... -# otherwise you need ENV LD_LIBRARY_PATH set... at build only... -RUN echo /usr/local/lib > /etc/ld.so.conf.d/local.conf -RUN echo /usr/local/lib64 >> /etc/ld.so.conf.d/local.conf -RUN ldconfig - -RUN curl -sL https://github.com/nghttp2/nghttp2/releases/download/v${LIB_NGHTTP2_RELEASE}/nghttp2-${LIB_NGHTTP2_RELEASE}.tar.gz -o nghttp2.tar.gz -RUN tar -xf nghttp2.tar.gz - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DENABLE_STATIC_LIB=ON \ - -DENABLE_SHARED_LIB=OFF \ - -DENABLE_LIB_ONLY=ON \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DENABLE_ARES=ON \ - -H/tmp/nghttp2-${LIB_NGHTTP2_RELEASE} \ - -B/tmp/nghttp2-${LIB_NGHTTP2_RELEASE}/cmake-out - -RUN cmake --build /tmp/nghttp2-${LIB_NGHTTP2_RELEASE}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/nghttp2.tar.gz -RUN rm -rf /tmp/nghttp2-${LIB_NGHTTP2_RELEASE} - -# rebuld CURL with ENABLE_ARES=ON USE_NGHTTP2=ON ? - -RUN wget -q https://github.com/google/protobuf/archive/v${LIB_PROTOBUF_VERSION}.tar.gz -RUN tar -xf v${LIB_PROTOBUF_VERSION}.tar.gz -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -Dprotobuf_BUILD_TESTS=OFF \ - -H/tmp/protobuf-${LIB_PROTOBUF_VERSION}/cmake \ - -B/tmp/protobuf-${LIB_PROTOBUF_VERSION}/cmake/cmake-out - -RUN cmake --build /tmp/protobuf-${LIB_PROTOBUF_VERSION}/cmake/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/v${LIB_PROTOBUF_VERSION}.tar.gz -RUN rm -rf /tmp/protobuf-${LIB_PROTOBUF_VERSION} - - -RUN curl -sL https://github.com/abseil/abseil-cpp/archive/${LIB_ABSEIL_RELEASE}.tar.gz -o ${LIB_ABSEIL_RELEASE}.tar.gz -RUN tar xf ${LIB_ABSEIL_RELEASE}.tar.gz -RUN rm -rf ${LIB_ABSEIL_RELEASE}.tar.gz - - -# 2020-06-12 -# google-cloud-cpp super build says absl/base/options.h needs to be patched -# -RUN perl -pi -e 's@#define ABSL_OPTION_USE_STD_ANY 2@#define ABSL_OPTION_USE_STD_ANY 0@g' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h -RUN perl -pi -e 's@#define ABSL_OPTION_USE_STD_OPTIONAL 2@#define ABSL_OPTION_USE_STD_OPTIONAL 0@g' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h -RUN perl -pi -e 's@#define ABSL_OPTION_USE_STD_STRING_VIEW 2@#define ABSL_OPTION_USE_STD_STRING_VIEW 0@g' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h -RUN perl -pi -e 's@#define ABSL_OPTION_USE_STD_VARIANT 2@#define ABSL_OPTION_USE_STD_VARIANT 0@g' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h -# verify the patch worked. -# ubuntu has dash and not bash aliased as /bin/sh ... -RUN bash -c "[[ \"$( egrep '^#define ABSL_OPTION_USE_STD_(.*) 2$' /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/absl/base/options.h |wc -l )\" == \"0\" ]]" - - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_TESTING=OFF \ - -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -H/tmp/abseil-cpp-${LIB_ABSEIL_RELEASE} \ - -B/tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/cmake-out - -RUN cmake --build /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/abseil-cpp-${LIB_ABSEIL_RELEASE} - -RUN curl -sL https://github.com/grpc/grpc/archive/v${LIB_GRPC_VERSION}.tar.gz -o v${LIB_GRPC_VERSION}.tar.gz -RUN tar -xf v${LIB_GRPC_VERSION}.tar.gz -RUN rm -rf v${LIB_GRPC_VERSION}.tar.gz - -# needed? -#ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig -#ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 -#ENV PATH=/usr/local/bin:${PATH} -# this does not install gRPC as a CMake package... -# -#RUN make -j $( nproc ) -#RUN make install - -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DgRPC_BUILD_TESTS=OFF \ - -DgRPC_ZLIB_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package \ - -DgRPC_CARES_PROVIDER=package \ - -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_ABSL_PROVIDER=package \ - -H/tmp/grpc-${LIB_GRPC_VERSION} \ - -B/tmp/grpc-${LIB_GRPC_VERSION}/cmake-out - -RUN cmake --build /tmp/grpc-${LIB_GRPC_VERSION}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/grpc-${LIB_GRPC_VERSION} - - -FROM gcloud-dev AS gcs-dev -ARG LIB_NLOHMANN_JSON_RELEASE_TAG=v3.9.1 -ARG GOOGLE_CLOUD_CPP_RELEASE_TAG=1.22.0 -# library path : args have to be repeated after every FROM ? or maybe I just need to declare it before the very first one? -ARG LIBRARIES_INSTALL_PATH=/usr/local - -WORKDIR /tmp - -# this is a header-only library so, we don't ask for static libs with position independent code -RUN git clone --depth 1 --branch ${LIB_NLOHMANN_JSON_RELEASE_TAG} https://github.com/nlohmann/json -RUN cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -H/tmp/json -B/tmp/json/cmake-out -RUN cmake --build /tmp/json/cmake-out --target install -- -j $(nproc) -RUN rm -rf /tmp/json - - -# only build google-cloud-cpp with the storage_client api. no other, from code generation or otherwise - -# some benchmarks require CURL 7.64.0 features at compilation time but do not check. -# this is OK in this build _because_ we have a newer static libcurl - -RUN curl -sL https://github.com/googleapis/google-cloud-cpp/archive/v${GOOGLE_CLOUD_CPP_RELEASE_TAG}.tar.gz -o ${GOOGLE_CLOUD_CPP_RELEASE_TAG}.tar.gz -RUN tar -xf ${GOOGLE_CLOUD_CPP_RELEASE_TAG}.tar.gz -RUN rm -rf ${GOOGLE_CLOUD_CPP_RELEASE_TAG}.tar.gz - -# to run with GRPC in GOOGLE CLOUD STORAGE it looks like it needs a patch - -# NOT YET CONTRIBUTED -#RUN perl -pi -e 's/google_cloud_cpp_proto_library\(/include\(CompileProtos\)\ngoogle_cloud_cpp_proto_library\(/g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/google/cloud/storage/CMakeLists.txt -#RUN perl -pi -e 's/if \(\$\{GOOGLE_CLOUD_CPP_ENABLE_GRPC\}\)/if \(\$\{GOOGLE_CLOUD_CPP_ENABLE_GRPC\} OR \$\{GOOGLE_CLOUD_CPP_STORAGE_ENABLE_GRPC\}\)/g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/CMakeLists.txt -RUN perl -pi -e 's/foreach \(_library /foreach (_library storage /g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/CMakeLists.txt -RUN perl -pi -e 's/google_cloud_cpp_proto_library\(/file\(\MAKE_DIRECTORY \$\{CMAKE_CURRENT_BINARY_DIR\}\/internal)\ngoogle_cloud_cpp_proto_library\(/g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/google/cloud/storage/CMakeLists.txt -RUN perl -pi -e 's/ LOCAL_INCLUDE/ \$\{CMAKE_CURRENT_SOURCE_DIR\}\/internal LOCAL_INCLUDE/g' /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/google/cloud/storage/CMakeLists.txt - -RUN cmake \ - -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} \ - -DBUILD_SHARED_LIBS=NO \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DGOOGLE_CLOUD_CPP_DEPENDENCY_PROVIDER=package \ - -DGOOGLE_CLOUD_CPP_GMOCK_PROVIDER=external \ - -DGOOGLE_CLOUD_CPP_ENABLE=storage \ - -DBUILD_TESTING=OFF \ - -DGOOGLE_CLOUD_CPP_STORAGE_ENABLE_GRPC=ON \ - -H/tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG} \ - -B/tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/cmake-out - -RUN cmake --build /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG}/cmake-out \ - --target install -- -j $( nproc ) - -RUN rm -rf /tmp/google-cloud-cpp-${GOOGLE_CLOUD_CPP_RELEASE_TAG} - -FROM gcs-dev AS sdapi-dev - - - - -FROM sdapi-dev AS sdapi-cicd +ARG LIB_AZURE_STORAGE_VERSION=7.5.0 +RUN git clone -b v${LIB_AZURE_STORAGE_VERSION} https://github.com/Azure/azure-storage-cpp.git azure-storage-cpp --recursive \ + && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage \ + && mkdir build.release \ + && cd build.release \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LIBRARIES_INSTALL_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBoost_USE_STATIC_LIBS=ON .. \ + && make -j $(nproc) \ + && make install \ + && cd ../../.. \ + && rm -rf azure-storage-cpp + +# [AZURE-STORAGE-FOR-CPP] +ARG AZURE_STORAGE_FOR_CPP_VERSION=12.0.0-beta.10 +RUN wget https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && tar -zxvf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz \ + && cd azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=NO -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DBUILD_TRANSPORT_CURL:BOOL=ON .. \ + && make -j $(nproc) \ + && make install \ + && rm -rf azure-sdk-for-cpp-azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION} \ + && rm -rf azure-storage-blobs_${AZURE_STORAGE_FOR_CPP_VERSION}.tar.gz # finalize the build WORKDIR / diff --git a/devops/scripts/build-linux64.sh b/devops/scripts/build-linux64.sh index 3bf5014ebf75abda0e934cf5ea749c4f0dd3308d..7e418068e6b45f4ee6d5a54ee72a7bbdd6ce88bc 100755 --- a/devops/scripts/build-linux64.sh +++ b/devops/scripts/build-linux64.sh @@ -62,15 +62,23 @@ for i in "$@"; do ;; --azure-provider) shift - if [ -z "${cloud_providers}"]; then + if [ -z "${cloud_providers}" ]; then cloud_providers="azure" else cloud_providers="${cloud_providers};azure" fi ;; + --azure-curl-provider) + shift + if [ -z "${cloud_providers}" ]; then + cloud_providers="azure-curl" + else + cloud_providers="${cloud_providers};azure-curl" + fi + ;; --aws-provider) shift - if [ -z "${cloud_providers}"]; then + if [ -z "${cloud_providers}" ]; then cloud_providers="aws" else cloud_providers="${cloud_providers};aws" @@ -78,7 +86,7 @@ for i in "$@"; do ;; --gcp-provider) shift - if [ -z "${cloud_providers}"]; then + if [ -z "${cloud_providers}" ]; then cloud_providers="gcp" else cloud_providers="${cloud_providers};gcp" @@ -94,7 +102,7 @@ options: print this message --build-mode=VALUE - CMake CMAKE_BUILD_TYPE variable value ( Release, Debug, ... ) + CMake CMAKE_BUILD_TYPE variable value ( Release, Debug, Veracode... ) --mnt-volume=path Location of the source code @@ -124,7 +132,10 @@ options: Just build SDAPI with the default GCS implementation. No optional Cloud provider --azure-provider - Also build the Azure optional cloud provider code. ( functional, based on Azure Storage C++ SDK ) + Also build the Azure optional cloud provider code. ( functional, based on Azure Storage C++ SDK, cpprest based ) + + --azure-curl-provider + Also build the Azure optional cloud provider code. ( functional, based on Azure Storage C++ SDK, libcurl based ) --aws-provider Also build the AWS optional cloud provider code. ( non-functional, based on AWS C++ SDK ) @@ -155,8 +166,8 @@ if [ -z "${build_mode}" ]; then fi # check if the required build mode is supported -if [[ "${build_mode}" != "Release" && "${build_mode}" != "Debug" ]]; then - echo "The build mode ${build_mode} is not supported. Valid values are Release or Debug" +if [[ "${build_mode}" != "Release" && "${build_mode}" != "Debug" && "${build_mode}" != "Veracode" ]]; then + echo "The build mode ${build_mode} is not supported. Valid values are Release | Debug | Veracode" exit 1 fi diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 339063ba6a538e0680c2126fb3b8af520422608d..0aad5625869855409f73ab07742bc00ac34471a4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,22 +24,50 @@ project(sdapi) # cmake modules set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/modules) +#list(APPEND CMAKE_MODULE_PATH ${CMAKE_ROOT}/Modules) + if(WIN32) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/modules/version_info_win32) + +else() + # cmake config packages. + # adapt to third party container sdk? + list( APPEND CMAKE_PREFIX_PATH /usr/local/share ) + list( APPEND CMAKE_PREFIX_PATH /usr/local/lib/cmake ) + list( APPEND CMAKE_PREFIX_PATH /usr/local/lib64/cmake ) + + # when cmake attempts to use pkgconfig from dependencies + # aka libxml2 may be troublesome on centos / static builds + set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig") + endif() +message( STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") +message( STATUS "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}") +message( STATUS "CMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" ) + # set compilers flags if(NOT WIN32) set (CMAKE_CXX_STANDARD 14) set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_CXX_FLAGS "-std=c++14 -pthread ${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic") + if( CMAKE_BUILD_TYPE STREQUAL "Coverage" OR CMAKE_BUILD_TYPE STREQUAL "Debug" ) - set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g3 -Og " ) + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g3 -Og " ) elseif( CMAKE_BUILD_TYPE STREQUAL "Veracode" ) - set (CMAKE_CXX_FLAGS_VERACODE "${CMAKE_CXX_FLAGS} -gdwarf-2 -g3 -O0 -fno-builtin " ) + set (CMAKE_CXX_FLAGS_VERACODE "${CMAKE_CXX_FLAGS} -gdwarf-2 -g3 -O0 -fno-builtin " ) elseif( CMAKE_BUILD_TYPE STREQUAL "Release" ) - set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O3 " ) + set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O3 " ) endif() + + + # enforce no undefined symbols in shared libraries produced by the linker + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") + + # enforce no undefined symbols in shared libraries consumed by the linker + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-allow-shlib-undefined") + + else() set (CMAKE_CXX_STANDARD 14) set (CMAKE_CXX_STANDARD_REQUIRED ON) @@ -57,6 +85,7 @@ if(NOT WIN32 AND CMAKE_BUILD_TYPE STREQUAL "Coverage" ) endif() option( AZURE_PROVIDER_ENABLED OFF ) +option( AZURE_CURL_PROVIDER_ENABLED OFF) option( AWS_PROVIDER_ENABLED OFF ) option( GCP_PROVIDER_ENABLED OFF ) @@ -70,6 +99,13 @@ else() message( STATUS "NOT BUILDING AZURE PROVIDER") endif() +if ( "azure-curl" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED ) + set( AZURE_CURL_PROVIDER_ENABLED ON) + add_definitions( -DHAS_AZURE_BLOB_STORAGE_PROVIDER_CURL ) +else() + message( STATUS "NOT BUILDING AZURE PROVIDER CURL") +endif() + if( "gcp" IN_LIST OPTIONAL_STORAGE_PROVIDERS_ENABLED ) set( GCP_PROVIDER_ENABLED ON ) @@ -86,6 +122,7 @@ else () endif() message( STATUS "AZURE_PROVIDER_ENABLED=${AZURE_PROVIDER_ENABLED}" ) +message( STATUS "AZURE_CURL_PROVIDER_ENABLED=${AZURE_CURL_PROVIDER_ENABLED}" ) message( STATUS "AWS_PROVIDER_ENABLED=${AWS_PROVIDER_ENABLED}" ) message( STATUS "GCP_PROVIDER_ENABLED=${GCP_PROVIDER_ENABLED}" ) @@ -125,7 +162,10 @@ file(GLOB SRC_LIB_ACCESSORS ${sdapi_SOURCE_DIR}/src/lib/accessors/*.cc) file(GLOB SRC_LIB_PROVIDERS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/*.cc) if(AZURE_PROVIDER_ENABLED) - file(GLOB SRC_LIB_PROVIDERS_AZURE ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/azure/*.cc) + file(GLOB SRC_LIB_PROVIDERS_AZURE ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/azure/cpprest/*.cc) +endif() +if(AZURE_CURL_PROVIDER_ENABLED) + file(GLOB SRC_LIB_PROVIDERS_AZURE_NEW ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/azure/curl/*.cc) endif() if (AWS_PROVIDER_ENABLED) file(GLOB SRC_LIB_PROVIDERS_AWS ${sdapi_SOURCE_DIR}/src/lib/cloud/providers/aws/*.cc) @@ -144,6 +184,10 @@ if(AZURE_PROVIDER_ENABLED) set(SOURCES ${SOURCES} ${SRC_LIB_PROVIDERS_AZURE} ) endif() +if(AZURE_CURL_PROVIDER_ENABLED) + set(SOURCES ${SOURCES} ${SRC_LIB_PROVIDERS_AZURE_NEW} ) +endif() + # import curl module if(NOT LIBCURL_VERSION) set (LIBCURL_VERSION "7.73.0") @@ -159,12 +203,48 @@ else() set (OPENSSL_LIBRARY "") endif() +if(AZURE_CURL_PROVIDER_ENABLED) + + if(NOT WIN32) + + # cmake configs : + # there may be issues with LibXml2 ... and libxml2 built in container + + find_library( XML2_LIBRARY xml2 ) + find_library( LZMA_LIBRARY lzma ) + find_library( Z_LIBRARY z ) + + find_package( libxml2 CONFIG REQUIRED) + set( LIBXML2_LIBRARY ${XML2_LIBRARY} CACHE FILEPATH "path to libxml2 library" ) + + find_package(LibXml2 CONFIG REQUIRED ) + + if(LibXml2_FOUND AND NOT TARGET LibXml2::LibXml2) + message( STATUS "manually constructing LibXml2::LibXml2") + + add_library(LibXml2::LibXml2 UNKNOWN IMPORTED) + set_target_properties(LibXml2::LibXml2 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBXML2_INCLUDE_DIRS}") + set_target_properties(LibXml2::LibXml2 PROPERTIES INTERFACE_COMPILE_OPTIONS "${LIBXML2_DEFINITIONS}") + set_property(TARGET LibXml2::LibXml2 APPEND PROPERTY IMPORTED_LOCATION "${LIBXML2_LIBRARY}" ) + + # the magic line that makes statically linked libxml2 with lzma and lz work is here... + set_property(TARGET LibXml2::LibXml2 PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES "${LZMA_LIBRARY};${Z_LIBRARY}") + + endif() + endif() + + find_package(azure-storage-common-cpp CONFIG REQUIRED ) + find_package(azure-storage-blobs-cpp CONFIG REQUIRED) + +endif() + if(AZURE_PROVIDER_ENABLED) # import azure storage cpp module # for now that does not get bundled into the static library bundle because it's not a cmake package # and its depdendenceis are not declared. find_path(WASTORAGE_INCLUDE_DIR was/blob.h) include_directories(${WASTORAGE_INCLUDE_DIR}) + if (NOT WIN32) find_library(WASTORAGE_LIBRARY azurestorage) find_library(CPPREST_LIBRARY cpprest) @@ -255,6 +335,10 @@ if(NOT WIN32) ${CRC32C_LIBRARY} ) + if(AZURE_CURL_PROVIDER_ENABLED) + target_link_libraries(sdapi PRIVATE Azure::azure-storage-blobs LibXml2::LibXml2 ) + endif(AZURE_CURL_PROVIDER_ENABLED) + if(AZURE_PROVIDER_ENABLED) #stuff with no proper targets defined in CMake module or package? # we will have to reap all the libraries at some point though. @@ -291,12 +375,17 @@ if(NOT WIN32) else() # WIN32 - target_link_libraries(sdapi + target_link_libraries(sdapi PRIVATE ${CURL_LIBRARY} ${CRC32C_LIBRARY} ) + + if(AZURE_CURL_PROVIDER_ENABLED) + target_link_libraries(sdapi PRIVATE Azure::azure-storage-blobs) + endif() + if(AZURE_PROVIDER_ENABLED) - target_link_libraries(sdapi + target_link_libraries(sdapi PRIVATE ${WASTORAGE_LIBRARY} ${CPPREST_LIBRARY} ) @@ -322,9 +411,11 @@ if(NOT WIN32) set(LN_FLAGS "-sfT" ) file(GLOB version_script src/Version.txt) - set_target_properties(sdapi PROPERTIES - LINK_FLAGS "-Wl,--version-script=${version_script}" - ) + set_target_properties(sdapi PROPERTIES LINK_FLAGS "-Wl,--version-script=${version_script}") + + # if the linker version script changes. I want to link again. + set_target_properties( sdapi PROPERTIES LINK_DEPENDS ${version_script}) + endif() add_custom_command(TARGET sdapi @@ -364,6 +455,11 @@ if(NOT WIN32) ${OPENSSL_LIBRARY} ${CRC32C_LIBRARY} ) + + if(AZURE_CURL_PROVIDER_ENABLED) + target_link_libraries(sdapi_static PRIVATE Azure::azure-storage-blobs LibXml2::LibXml2) + endif() + if( AZURE_PROVIDER_ENABLED) target_link_libraries(sdapi_static PRIVATE ${WASTORAGE_LIBRARY} @@ -396,14 +492,20 @@ if(NOT WIN32) target_link_libraries( sdapi_static PRIVATE storage_client ) endif() +# WIN32 else() - target_link_libraries(sdapi_static + target_link_libraries(sdapi_static PRIVATE ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CRC32C_LIBRARY} ) + + if(AZURE_CURL_PROVIDER_ENABLED) + target_link_libraries(sdapi_static PRIVATE Azure::azure-storage-blobs) + endif() + if(AZURE_PROVIDER_ENABLED) - target_link_libraries(sdapi_static + target_link_libraries(sdapi_static PRIVATE ${WASTORAGE_LIBRARY} ${CPPREST_LIBRARY} ) @@ -484,4 +586,4 @@ if(BUILD_BENCHMARKS) COMMENT "Running Performance Benchmarks" ) -endif() +endif() \ No newline at end of file diff --git a/src/src/lib/cloud/providers/Storage.cc b/src/src/lib/cloud/providers/Storage.cc index fc87adb317ef6555292abd19996f7493c92127cb..29d329b97728d58baffc2e817eae7d442c281a77 100644 --- a/src/src/lib/cloud/providers/Storage.cc +++ b/src/src/lib/cloud/providers/Storage.cc @@ -21,9 +21,14 @@ #include "Storage.h" #include "SDException.h" #include "accessors/GcsAccessor.h" +#include "shared/utils.h" #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER -#include "cloud/providers/azure/AzureStorage.h" +#include "cloud/providers/azure/cpprest/AzureStorage.h" +#endif + +#ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL +#include "cloud/providers/azure/curl/AzureStorage.h" #endif #ifdef HAS_AWS_S3_STORAGE_PROVIDER @@ -38,6 +43,10 @@ static const std::string kAzureCloudProvider{"azure"}; static const std::string kAwsCloudProvider{"aws"}; +namespace { + const bool _isAzureCurlEnabled = seismicdrive::sdutils::getSwitchEnv("ENABLE_AZURE_CURL_BASED"); +} + namespace seismicdrive { // -------------------------------- @@ -49,10 +58,22 @@ class StorageObjectInfoIteratorImpl { public: #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER - StorageObjectInfoIteratorImpl(std::shared_ptr accessor, const std::string& storagePath, const GcsContext* pContext, bool recursive): + StorageObjectInfoIteratorImpl(std::shared_ptr accessor, const std::string& storagePath, const GcsContext* pContext, bool recursive): + _storagePath(storagePath), + _httpContext(pContext), + _azureAccessor(accessor), + _recursive(recursive) { + + getNextItems(); + + } +#endif + +#ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + StorageObjectInfoIteratorImpl(std::shared_ptr accessor, const std::string& storagePath, const GcsContext* pContext, bool recursive): _storagePath(storagePath), - _http_context(pContext), - _azaccessor(accessor), + _httpContext(pContext), + _azureAccessorCurl(accessor), _recursive(recursive) { getNextItems(); @@ -63,21 +84,20 @@ public: #ifdef HAS_AWS_S3_STORAGE_PROVIDER StorageObjectInfoIteratorImpl(std::shared_ptr accessor, const std::string& storagePath, const GcsContext* pContext, bool recursive): _storagePath(storagePath), - _http_context(pContext), - _awsaccessor(accessor), + _httpContext(pContext), + _awsAccessor(accessor), _recursive(recursive) { getNextItems(); } - #endif #ifdef HAS_GCP_GCS_STORAGE_PROVIDER StorageObjectInfoIteratorImpl(std::shared_ptr accessor, const std::string& storagePath, const GcsContext* pContext, bool recursive): _storagePath(storagePath), - _http_context(pContext), - _gcpaccessor(accessor), + _httpContext(pContext), + _gcpAccessor(accessor), _recursive(recursive) { getNextItems(); @@ -88,8 +108,8 @@ public: StorageObjectInfoIteratorImpl(std::shared_ptr accessor, const std::string& storagePath, const GcsContext* pContext, bool recursive): _storagePath(storagePath), - _http_context(pContext), - _gcsaccessor(accessor), + _httpContext(pContext), + _gcsAccessor(accessor), _recursive(recursive) { getNextItems(); } @@ -112,46 +132,68 @@ private: std::unique_lock lock(_mu); _nextItem = 0; - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER - if(_azaccessor) { - _azaccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _http_context); + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if(_isAzureCurlEnabled) { + if(_azureAccessorCurl) { + _azureAccessorCurl->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _httpContext); + return; + } + } else { + if(_azureAccessor) { + _azureAccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _httpContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if(_azureAccessor) { + _azureAccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _httpContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if(_azureAccessorCurl) { + _azureAccessorCurl->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _httpContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER - if(_awsaccessor) { - _awsaccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _http_context); + if(_awsAccessor) { + _awsAccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _httpContext); return; } #endif + #ifdef HAS_GCP_GCS_STORAGE_PROVIDER - if(_gcpaccessor) { - _gcpaccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _http_context); + if(_gcpAccessor) { + _gcpAccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &_items, _httpContext); return; } #endif // November 2020 -> can't break the exported GcsAccessor Interface (conversion is needed here) std::vector itemsGcs; - _gcsaccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &itemsGcs, _http_context); + _gcsAccessor->getChildren(_storagePath, _recursive, &_nextPageToken, &itemsGcs, _httpContext); for(auto &&item: itemsGcs) _items.emplace_back(StorageObjectInfo(item.name, item.size)); } std::string _storagePath; - const GcsContext* _http_context; + const GcsContext* _httpContext; #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER - std::shared_ptr _azaccessor; + std::shared_ptr _azureAccessor; +#endif +#ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + std::shared_ptr _azureAccessorCurl; #endif #ifdef HAS_AWS_S3_STORAGE_PROVIDER - std::shared_ptr _awsaccessor; + std::shared_ptr _awsAccessor; #endif #ifdef HAS_GCP_GCS_STORAGE_PROVIDER - std::shared_ptr _gcpaccessor; + std::shared_ptr _gcpAccessor; #endif - std::shared_ptr _gcsaccessor; + std::shared_ptr _gcsAccessor; std::vector _items; size_t _nextItem; std::string _nextPageToken; @@ -186,8 +228,16 @@ private: _provider(cloud_provider) { if (_provider == kAzureCloudProvider) { -#ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER - _azstorage = std::make_shared(auth_provider, sdresource, readonly); +#if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if(_isAzureCurlEnabled) { + _azureStorageCurl = std::make_shared(auth_provider, sdresource, readonly); + } else { + _azureStorage = std::make_shared(auth_provider, sdresource, readonly); + } +#elif HAS_AZURE_BLOB_STORAGE_PROVIDER + _azureStorage = std::make_shared(auth_provider, sdresource, readonly); +#elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + _azureStorageCurl = std::make_shared(auth_provider, sdresource, readonly); #else throw std::logic_error( "azure / blob: not supported : no client library code compiled" ); #endif @@ -196,7 +246,7 @@ private: if ( _provider == kAwsCloudProvider ){ #ifdef HAS_AWS_S3_STORAGE_PROVIDER - _awsstorage = std::make_shared(auth_provider, sdresource, readonly); + _awsStorage = std::make_shared(auth_provider, sdresource, readonly); #else throw std::logic_error( "aws / s3 : not supported : no client library code compiled" ); #endif @@ -205,322 +255,654 @@ private: if ( _provider == "experimental-gcp" ){ #ifdef HAS_GCP_GCS_STORAGE_PROVIDER - _gcsstorage = std::make_shared(auth_provider, sdresource, readonly); + _gcsStorage = std::make_shared(auth_provider, sdresource, readonly); #else throw std::logic_error( "google-cloud-cpp sdk not yet supported : no client library code compiled"); #endif return; } - _gcsaccessor = std::make_shared(auth_provider, sdresource, readonly); + _gcsAccessor = std::make_shared(auth_provider, sdresource, readonly); } Storage::~Storage() = default; bool Storage::objectExists(const std::string& storage_path, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) if (_provider == kAzureCloudProvider) { - return _azstorage->objectExists(storage_path, pContext); + if(_isAzureCurlEnabled) { + return _azureStorageCurl->objectExists(storage_path, pContext); + } else { + return _azureStorage->objectExists(storage_path, pContext); + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + return _azureStorage->objectExists(storage_path, pContext); + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + return _azureStorageCurl->objectExists(storage_path, pContext); } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if ( _provider == kAwsCloudProvider ){ - return _awsstorage->objectExists( storage_path, pContext ); + return _awsStorage->objectExists( storage_path, pContext ); } #endif + #ifdef HAS_GCP_GCS_STORAGE_PROVIDER if ( _provider == "experimental-gcp" ){ - return _gcsstorage->objectExists( storage_path, pContext ); + return _gcsStorage->objectExists( storage_path, pContext ); } #endif - return _gcsaccessor->objectExists(storage_path, pContext); + return _gcsAccessor->objectExists(storage_path, pContext); } StorageObjectInfoIterator Storage::getIterator(const std::string &storage_path, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + return StorageObjectInfoIterator( + new StorageObjectInfoIteratorImpl( + _azureStorageCurl, storage_path, pContext, false)); + } else { + return StorageObjectInfoIterator( + new StorageObjectInfoIteratorImpl( + _azureStorage, storage_path, pContext, false)); + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER if (_provider == kAzureCloudProvider) { return StorageObjectInfoIterator( new StorageObjectInfoIteratorImpl( - _azstorage, storage_path, pContext, false)); + _azureStorage, storage_path, pContext, false)); + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + return StorageObjectInfoIterator( + new StorageObjectInfoIteratorImpl( + _azureStorageCurl, storage_path, pContext, false)); } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { return StorageObjectInfoIterator( new StorageObjectInfoIteratorImpl( - _awsstorage, storage_path, pContext, false)); + _awsStorage, storage_path, pContext, false)); } #endif + return StorageObjectInfoIterator( new StorageObjectInfoIteratorImpl( - _gcsaccessor, storage_path, pContext, false)); + _gcsAccessor, storage_path, pContext, false)); } StorageObjectInfoIterator Storage::getIterator(const std::string &storage_path, bool recursive, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + return StorageObjectInfoIterator( + new StorageObjectInfoIteratorImpl( + _azureStorageCurl, storage_path, pContext, recursive)); + } else { + return StorageObjectInfoIterator( + new StorageObjectInfoIteratorImpl( + _azureStorage, storage_path, pContext, recursive)); + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + return StorageObjectInfoIterator( + new StorageObjectInfoIteratorImpl( + _azureStorage, storage_path, pContext, recursive)); + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL if (_provider == kAzureCloudProvider) { return StorageObjectInfoIterator( new StorageObjectInfoIteratorImpl( - _azstorage, storage_path, pContext, recursive)); + _azureStorageCurl, storage_path, pContext, recursive)); } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { return StorageObjectInfoIterator( new StorageObjectInfoIteratorImpl( - _awsstorage, storage_path, pContext, recursive)); + _awsStorage, storage_path, pContext, recursive)); } #endif + return StorageObjectInfoIterator( new StorageObjectInfoIteratorImpl( - _gcsaccessor, storage_path, pContext, recursive)); + _gcsAccessor, storage_path, pContext, recursive)); } long long Storage::objectSize(const std::string &storage_path, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + return _azureStorageCurl->objectSize(storage_path, pContext); + } else { + return _azureStorage->objectSize(storage_path, pContext); + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER if (_provider == kAzureCloudProvider) { - return _azstorage->objectSize(storage_path, pContext); + return _azureStorage->objectSize(storage_path, pContext); + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + return _azureStorageCurl->objectSize(storage_path, pContext); } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - return _awsstorage->objectSize(storage_path, pContext); + return _awsStorage->objectSize(storage_path, pContext); } #endif - return _gcsaccessor->objectSize(storage_path, pContext); + + return _gcsAccessor->objectSize(storage_path, pContext); } std::vector Storage::objectsSize(const std::vector &storage_paths, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) if (_provider == kAzureCloudProvider) { - return _azstorage->objectsSize(storage_paths, pContext); + if(_isAzureCurlEnabled) { + return _azureStorageCurl->objectsSize(storage_paths, pContext); + } else { + return _azureStorage->objectsSize(storage_paths, pContext); + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + return _azureStorage->objectsSize(storage_paths, pContext); + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + return _azureStorageCurl->objectsSize(storage_paths, pContext); } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - return _awsstorage->objectsSize(storage_paths, pContext); + return _awsStorage->objectsSize(storage_paths, pContext); } #endif - return _gcsaccessor->objectsSize(storage_paths, pContext); + + return _gcsAccessor->objectsSize(storage_paths, pContext); } void Storage::uploadObject(const std::string &storage_path, const void *data, size_t len, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->uploadObject(storage_path, data, len, pContext); + return; + } else { + _azureStorage->uploadObject(storage_path, data, len, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + _azureStorage->uploadObject(storage_path, data, len, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL if (_provider == kAzureCloudProvider) { - _azstorage->uploadObject(storage_path, data, len, pContext); + _azureStorageCurl->uploadObject(storage_path, data, len, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->uploadObject(storage_path, data, len, pContext); + _awsStorage->uploadObject(storage_path, data, len, pContext); return; } #endif - _gcsaccessor->uploadObject(storage_path, data, len, pContext); + + _gcsAccessor->uploadObject(storage_path, data, len, pContext); } void Storage::uploadObject(const std::string &storage_path, const std::string &file_name, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) if (_provider == kAzureCloudProvider) { - _azstorage->uploadObject(storage_path, file_name, pContext); + if(_isAzureCurlEnabled) { + _azureStorageCurl->uploadObject(storage_path, file_name, pContext); + return; + } else { + _azureStorage->uploadObject(storage_path, file_name, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + _azureStorage->uploadObject(storage_path, file_name, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + _azureStorageCurl->uploadObject(storage_path, file_name, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->uploadObject(storage_path, file_name, pContext); + _awsStorage->uploadObject(storage_path, file_name, pContext); return; } #endif - _gcsaccessor->uploadObject(storage_path, file_name, pContext); + + _gcsAccessor->uploadObject(storage_path, file_name, pContext); } void Storage::downloadObject(const std::string &storage_path, void *data, size_t offset, size_t len, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->downloadObject(storage_path, data, offset, len, pContext); + return; + } else { + _azureStorage->downloadObject(storage_path, data, offset, len, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER if (_provider == kAzureCloudProvider) { - _azstorage->downloadObject(storage_path, data, offset, len, pContext); + _azureStorage->downloadObject(storage_path, data, offset, len, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + _azureStorageCurl->downloadObject(storage_path, data, offset, len, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->downloadObject(storage_path, data, offset, len, pContext); + _awsStorage->downloadObject(storage_path, data, offset, len, pContext); return; } #endif - _gcsaccessor->downloadObject(storage_path, data, offset, len, pContext); + + _gcsAccessor->downloadObject(storage_path, data, offset, len, pContext); } void Storage::downloadObject(const std::string &storage_path, void *data, size_t& len, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) if (_provider == kAzureCloudProvider) { - _azstorage->downloadObject(storage_path, data, len, pContext); + if(_isAzureCurlEnabled) { + _azureStorageCurl->downloadObject(storage_path, data, len, pContext); + return; + } else { + _azureStorage->downloadObject(storage_path, data, len, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + _azureStorage->downloadObject(storage_path, data, len, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + _azureStorageCurl->downloadObject(storage_path, data, len, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->downloadObject(storage_path, data, len, pContext); + _awsStorage->downloadObject(storage_path, data, len, pContext); return; } #endif - _gcsaccessor->downloadObject(storage_path, data, len, pContext); + + _gcsAccessor->downloadObject(storage_path, data, len, pContext); } void Storage::downloadObject(const std::string &storage_path, void **data, size_t& len, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->downloadObject(storage_path, data, len, pContext); + return; + } else { + _azureStorage->downloadObject(storage_path, data, len, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER if (_provider == kAzureCloudProvider) { - _azstorage->downloadObject(storage_path, data, len, pContext); + _azureStorage->downloadObject(storage_path, data, len, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + _azureStorageCurl->downloadObject(storage_path, data, len, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->downloadObject(storage_path, data, len, pContext); + _awsStorage->downloadObject(storage_path, data, len, pContext); return; } #endif - _gcsaccessor->downloadObject(storage_path, data, len, pContext); + + _gcsAccessor->downloadObject(storage_path, data, len, pContext); } void Storage::deleteObject(const std::string &storage_path, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) if (_provider == kAzureCloudProvider) { - _azstorage->deleteObject(storage_path, pContext); + if(_isAzureCurlEnabled) { + _azureStorageCurl->deleteObject(storage_path, pContext); + return; + } else { + _azureStorage->deleteObject(storage_path, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + _azureStorage->deleteObject(storage_path, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + _azureStorageCurl->deleteObject(storage_path, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->deleteObject(storage_path, pContext); + _awsStorage->deleteObject(storage_path, pContext); return; } #endif - _gcsaccessor->deleteObject(storage_path, pContext); + + _gcsAccessor->deleteObject(storage_path, pContext); } void Storage::lockObject(const std::string &storage_path, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->lockObject(storage_path, pContext); + return; + } else { + _azureStorage->lockObject(storage_path, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + _azureStorage->lockObject(storage_path, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL if (_provider == kAzureCloudProvider) { - _azstorage->lockObject(storage_path, pContext); + _azureStorageCurl->lockObject(storage_path, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->lockObject(storage_path, pContext); + _awsStorage->lockObject(storage_path, pContext); return; } #endif - _gcsaccessor->lockObject(storage_path, pContext); + + _gcsAccessor->lockObject(storage_path, pContext); } void Storage::waitLockObject(const std::string &storage_path, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->waitLockObject(storage_path, pContext); + return; + } else { + _azureStorage->waitLockObject(storage_path, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER if (_provider == kAzureCloudProvider) { - _azstorage->waitLockObject(storage_path, pContext); + _azureStorage->waitLockObject(storage_path, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + _azureStorageCurl->waitLockObject(storage_path, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->waitLockObject(storage_path, pContext); + _awsStorage->waitLockObject(storage_path, pContext); return; } #endif - _gcsaccessor->waitLockObject(storage_path, pContext); + + _gcsAccessor->waitLockObject(storage_path, pContext); } void Storage::unlockObject(const std::string &storage_path, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->unlockObject(storage_path, pContext); + return; + } else { + _azureStorage->unlockObject(storage_path, pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER if (_provider == kAzureCloudProvider) { - _azstorage->unlockObject(storage_path, pContext); + _azureStorage->unlockObject(storage_path, pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + _azureStorageCurl->unlockObject(storage_path, pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->unlockObject(storage_path, pContext); + _awsStorage->unlockObject(storage_path, pContext); return; } #endif - _gcsaccessor->unlockObject(storage_path, pContext); + + _gcsAccessor->unlockObject(storage_path, pContext); } void Storage::setHttpContext(const HttpContext *pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->setHttpContext(pContext); + return; + } else { + _azureStorage->setHttpContext(pContext); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER if (_provider == kAzureCloudProvider) { - _azstorage->setHttpContext(pContext); + _azureStorage->setHttpContext(pContext); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + _azureStorageCurl->setHttpContext(pContext); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->setHttpContext(pContext); + _awsStorage->setHttpContext(pContext); return; } #endif - _gcsaccessor->setHttpContext(pContext); + + _gcsAccessor->setHttpContext(pContext); } void Storage::setExponentialRetryBackoffPolicy(const ExponentialRetryBackoffPolicy* policy) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->setExponentialRetryBackoffPolicy(policy); + return; + } else { + _azureStorage->setExponentialRetryBackoffPolicy(policy); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + _azureStorage->setExponentialRetryBackoffPolicy(policy); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL if (_provider == kAzureCloudProvider) { - _azstorage->setExponentialRetryBackoffPolicy(policy); + _azureStorageCurl->setExponentialRetryBackoffPolicy(policy); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER throw seismicdrive::SDException("Not yet implemented"); #endif - _gcsaccessor->setExponentialRetryBackoffPolicy(policy); + + _gcsAccessor->setExponentialRetryBackoffPolicy(policy); } std::pair Storage::getCachedStorageAccessToken(const std::string& resourceRef) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + return _azureStorageCurl->getCachedStorageAccessToken(resourceRef); + } else { + return _azureStorage->getCachedStorageAccessToken(resourceRef); + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + return _azureStorage->getCachedStorageAccessToken(resourceRef); + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL if (_provider == kAzureCloudProvider) { - return _azstorage->getCachedStorageAccessToken(resourceRef); + return _azureStorageCurl->getCachedStorageAccessToken(resourceRef); } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - return _awsstorage->getCachedStorageAccessToken(resourceRef); + return _awsStorage->getCachedStorageAccessToken(resourceRef); } #endif - return _gcsaccessor->getCachedStorageAccessToken(resourceRef); + + return _gcsAccessor->getCachedStorageAccessToken(resourceRef); } void Storage::setCachedStorageAccessToken(const std::string& resourceRef, std::pair credentials) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + _azureStorageCurl->setCachedStorageAccessToken(resourceRef, credentials); + return; + } else { + _azureStorage->setCachedStorageAccessToken(resourceRef, credentials); + return; + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER + if (_provider == kAzureCloudProvider) { + _azureStorage->setCachedStorageAccessToken(resourceRef, credentials); + return; + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL if (_provider == kAzureCloudProvider) { - _azstorage->setCachedStorageAccessToken(resourceRef, credentials); + _azureStorageCurl->setCachedStorageAccessToken(resourceRef, credentials); return; } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - _awsstorage->setCachedStorageAccessToken(resourceRef, credentials); + _awsStorage->setCachedStorageAccessToken(resourceRef, credentials); return; } #endif - _gcsaccessor->setCachedStorageAccessToken(resourceRef, credentials); + + _gcsAccessor->setCachedStorageAccessToken(resourceRef, credentials); } std::vector> Storage::ls(const std::string &storage_path, const GcsContext* pContext) { - #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER + + #if defined(HAS_AZURE_BLOB_STORAGE_PROVIDER) && defined(HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL) + if (_provider == kAzureCloudProvider) { + if(_isAzureCurlEnabled) { + return _azureStorageCurl->ls(storage_path, pContext); + } else { + return _azureStorage->ls(storage_path, pContext); + } + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER if (_provider == kAzureCloudProvider) { - return _azstorage->ls(storage_path, pContext); + return _azureStorage->ls(storage_path, pContext); + } + #elif HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + if (_provider == kAzureCloudProvider) { + return _azureStorageCurl->ls(storage_path, pContext); } #endif + #ifdef HAS_AWS_S3_STORAGE_PROVIDER if (_provider == kAwsCloudProvider) { - return _awsstorage->ls(storage_path, pContext); + return _awsStorage->ls(storage_path, pContext); } #endif - return _gcsaccessor->ls(storage_path, pContext); + + return _gcsAccessor->ls(storage_path, pContext); } -} // namespace seismicdrive +} // namespace seismicdrive \ No newline at end of file diff --git a/src/src/lib/cloud/providers/Storage.h b/src/src/lib/cloud/providers/Storage.h index 9759a424a8dae6237dc24cf886bfb7e7e4e4b534..3f59d270846ae3a560d3a59d6b6dcddc2d1f7876 100644 --- a/src/src/lib/cloud/providers/Storage.h +++ b/src/src/lib/cloud/providers/Storage.h @@ -25,7 +25,14 @@ namespace seismicdrive { #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER -class AzureStorage; +namespace cpprest { + class AzureStorage; +} +#endif +#ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL +namespace curl { + class AzureStorage; +} #endif #ifdef HAS_AWS_S3_STORAGE_PROVIDER class AwsStorage; @@ -403,23 +410,27 @@ class Storage: public StorageIF { private: - std::string _provider; - std::shared_ptr _gcsaccessor; + std::string _provider; + std::shared_ptr _gcsAccessor; #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER - std::shared_ptr _azstorage; + std::shared_ptr _azureStorage; +#endif + +#ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + std::shared_ptr _azureStorageCurl; #endif #ifdef HAS_AWS_S3_STORAGE_PROVIDER - std::shared_ptr _awsstorage; + std::shared_ptr _awsStorage; #endif #ifdef HAS_GCP_GCS_STORAGE_PROVIDER // experiment using google-cloud-cpp storage_client C++ library. - std::shared_ptr _gcsstorage; + std::shared_ptr _gcsStorage; #endif }; } // namespace seismicdrive -#endif // SDAPI_STORAGE_H +#endif // SDAPI_STORAGE_H \ No newline at end of file diff --git a/src/src/lib/cloud/providers/azure/AzureStorage.cc b/src/src/lib/cloud/providers/azure/cpprest/AzureStorage.cc similarity index 97% rename from src/src/lib/cloud/providers/azure/AzureStorage.cc rename to src/src/lib/cloud/providers/azure/cpprest/AzureStorage.cc index 7e9096e6de296bd4a1ccadb97aac5d5bb1b7646b..6f522d6c9d99f23967a359851187f99bf812ba51 100644 --- a/src/src/lib/cloud/providers/azure/AzureStorage.cc +++ b/src/src/lib/cloud/providers/azure/cpprest/AzureStorage.cc @@ -17,7 +17,7 @@ #define BUILDING_DLL #include "auth/auth_provider.h" -#include "cloud/providers/azure/AzureStorage.h" +#include "cloud/providers/azure/cpprest/AzureStorage.h" #include "accessors/GcsAccessor.h" #include "shared/config.h" #include "shared/mex.h" @@ -41,7 +41,7 @@ #include #include -namespace seismicdrive { +namespace seismicdrive { namespace cpprest { template @@ -822,4 +822,5 @@ namespace seismicdrive { std::vector> AzureStorage::ls(const std::string &objPath, const GcsContext* pContext) { return _impl->ls(objPath, pContext); } -} + +} } // namespace seismicdrive::cpprest diff --git a/src/src/lib/cloud/providers/azure/AzureStorage.h b/src/src/lib/cloud/providers/azure/cpprest/AzureStorage.h similarity index 97% rename from src/src/lib/cloud/providers/azure/AzureStorage.h rename to src/src/lib/cloud/providers/azure/cpprest/AzureStorage.h index 1541decbcfde19d6751d045acaff8b39b01e4d0c..24f782c2e14162c0aa51d59aacd642fde2df9d69 100644 --- a/src/src/lib/cloud/providers/azure/AzureStorage.h +++ b/src/src/lib/cloud/providers/azure/cpprest/AzureStorage.h @@ -23,7 +23,7 @@ #include -namespace seismicdrive { +namespace seismicdrive { namespace cpprest { class AzureStorageImpl; @@ -349,6 +349,6 @@ private: }; -} // namespace seismicdrive +} } // namespace seismicdrive::cpprest #endif // SDAPI_AZURESTORAGE_H diff --git a/src/src/lib/cloud/providers/azure/curl/AzureStorage.cc b/src/src/lib/cloud/providers/azure/curl/AzureStorage.cc new file mode 100644 index 0000000000000000000000000000000000000000..39a6990f94701b52b56d1c5d127151f3af7fd9a4 --- /dev/null +++ b/src/src/lib/cloud/providers/azure/curl/AzureStorage.cc @@ -0,0 +1,1102 @@ +// ============================================================================ +// Copyright 2017-2021, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ +#define BUILDING_DLL + +#include "auth/auth_provider.h" +#include "cloud/providers/azure/curl/AzureStorage.h" +#include "accessors/GcsAccessor.h" +#include "shared/config.h" +#include "shared/mex.h" +#include "SDException.h" +#include "shared/big_endian.h" +#include "shared/utils.h" +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Azure::Storage::Blobs; + +// TODO: The timeout for download should be based +// on the size of the data and the bandwidth of the network. +// The value should be flexible based on the above two factors. +// Timout in seconds +const int DOWNLOAD_TIMEOUT = 1800; + +namespace seismicdrive { namespace curl { + + template + void exponentialRetry( + const ExponentialRetryBackoffPolicy *retryPolicy, + const std::string operation, + std::function work, + std::function retryable) + { + if (retryPolicy->enabled) + { + std::string what; + for (unsigned i = 0; i < retryPolicy->maxRetry; i++) + { + if (i) + { + std::cout << operation << " retry " << i << '\n' + << what << '\n'; + } + try + { + work(); + return; + } + catch (const Catch &e) + { + what = e.what(); + if (!retryable(e)) + { + throw Throw(what); + } + std::uint32_t micros = seismicdrive::sdutils::generateRandomNumber(0, 1000000); + std::uint64_t delay_micros = std::min(((retryPolicy->initialWaitingTimeMicroSec << i) + micros), retryPolicy->maxWaitingTimeMicroSec); + std::this_thread::sleep_for(std::chrono::microseconds(delay_micros)); + } + } + throw Throw(operation + " failed " + std::to_string(retryPolicy->maxRetry) + " times. Error: " + what); + } + else + { + work(); + } + } + + void azureStorageExponentialRetry( + const ExponentialRetryBackoffPolicy *retryPolicy, + const std::string operation, + std::function work, + const std::unordered_set &retryStatusCodes, + std::function refreshToken) + { + using Ex = Azure::Storage::StorageException; + exponentialRetry( + retryPolicy, + operation, + work, + + [&retryStatusCodes, refreshToken](const Ex &e) -> bool { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::Unauthorized) + { + refreshToken(); + return true; + } + + return retryStatusCodes.find(e.StatusCode) != retryStatusCodes.end(); + }); + } + + using TokenInfo = GcsAuthProvider::TokenInfo; + + namespace + { + void parse_azstorage_path( + const std::string &cs_path, + std::string &container, + std::string &block_blob) + { + auto path = cs_path; + container.clear(); + block_blob.clear(); + + auto res = path.compare(0, sdconfig::GCSPREFIX.length(), sdconfig::GCSPREFIX) == 0; + if (!res) + { + throw seismicdrive::SDException(sdmex::gcs::PrefixError(path)); + } + path = path.substr(sdconfig::GCSPREFIX.length()); + + auto pos = path.find('/'); + if (pos == std::string::npos) + { + container = path; + } + else + { + container = path.substr(0, pos); + block_blob = path.substr(pos + 1); + } + } + + void parse_sas_uri( + const std::string &sas_uri, + std::string &account_name, + std::string &container_name, + std::string &sas_token) + { + account_name.clear(); + container_name.clear(); + sas_token.clear(); + + const std::regex re("https://(.*).blob.core.windows.net/(.*)\\?(.*)"); + std::smatch match; + if (!std::regex_match(sas_uri, match, re)) + { + throw seismicdrive::SDException("Invalid sas uri"); + } + + account_name = match[1].str(); + container_name = match[2].str(); + sas_token = match[3].str(); + } + } // namespace + + // -------------------------------- + // CloudBlobClient Definition + // -------------------------------- + + class CloudBlobClient : public CloudBlobClientIF + { + + public: + CloudBlobClient(const std::string &account_name, const std::string &sas_token); + ~CloudBlobClient(); + + bool exists(const std::string &container_name, const std::string &blob_name, const GcsContext *pContext) override; + void upload_from_file(const std::string &container_name, const std::string &blob_name, const std::string &file_name, const GcsContext * pContext); + void upload_from(const std::string &container_name, const std::string &blob_name, const char *source, std::uint64_t length, const GcsContext *pContext) override; + void download_to_range(const std::string &container_name, const std::string &blob_name, char *target, std::uint64_t offset, std::uint64_t length, const GcsContext *pContext) override; + void download_to(const std::string &container_name, const std::string &blob_name, char *target, std::uint64_t length, const GcsContext *pContext) override; + long long get_blob_size(const std::string &container_name, const std::string &blob_name, const GcsContext * pContext) override; + void delete_blob(const std::string &container_name, const std::string &blob_name, const GcsContext *pContext) override; + void list_blobs_segmented(const std::string &container_name, const std::string &prefix, bool recursiveList, std::string *nextPageToken, std::vector *result, const GcsContext *pContext) override; + void list_blobs_segmented(const std::string &container_name, const std::string &prefix, bool recursiveList, std::string *nextPageToken, std::vector> *result, const GcsContext *pContext) override; + + private: + static std::string convert_sas_token_to_connection_string(const std::string &account_name, const std::string &sas_token); + BlobServiceClient _blob_service_client; + }; + + std::string CloudBlobClient::convert_sas_token_to_connection_string( + const std::string &account_name, + const std::string &sas_token) + { + std::ostringstream oss; + oss << "https://" << account_name << ".blob.core.windows.net/" + << "?" << sas_token; + return oss.str(); + } + + CloudBlobClient::CloudBlobClient( + const std::string &account_name, + const std::string &sas_token) + : _blob_service_client(BlobServiceClient(convert_sas_token_to_connection_string(account_name, sas_token))) + { + } + + CloudBlobClient::~CloudBlobClient() + { + } + + bool CloudBlobClient::exists( + const std::string &container_name, + const std::string &blob_name, + const GcsContext * /*pContext*/) + { + try + { + auto blobContainerClient = _blob_service_client.GetBlobContainerClient(container_name); + auto blockBlobCllient = blobContainerClient.GetBlockBlobClient(blob_name); + blockBlobCllient.GetProperties(); + return true; + } + catch (const Azure::Storage::StorageException &e) + { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) + { + return false; + } + throw seismicdrive::SDExceptionAzureStorageError(e.what()); + } + } + + void CloudBlobClient::upload_from_file( + const std::string &container_name, + const std::string &blob_name, + const std::string &file_name, + const GcsContext * /*pContext*/) + { + auto containerClient = _blob_service_client.GetBlobContainerClient(container_name); + auto blobClient = containerClient.GetBlockBlobClient(blob_name); + blobClient.UploadFrom(file_name); + } + + void CloudBlobClient::upload_from( + const std::string &container_name, + const std::string &blob_name, + const char *buffer, + std::uint64_t bufferSize, + const GcsContext * /*pContext*/) + { + auto containerClient = _blob_service_client.GetBlobContainerClient(container_name); + auto blobClient = containerClient.GetBlockBlobClient(blob_name); + blobClient.UploadFrom(reinterpret_cast(buffer), bufferSize); + } + + void CloudBlobClient::download_to_range( + const std::string &container_name, + const std::string &blob_name, + char *target, + std::uint64_t offset, + std::uint64_t length, + const GcsContext * /*pContext*/) + { + auto containerClient = _blob_service_client.GetBlobContainerClient(container_name); + auto blobClient = containerClient.GetBlockBlobClient(blob_name); + + DownloadBlobOptions options; + + if (offset <= std::numeric_limits::max() && length <= std::numeric_limits::max()) + { + options.Range = Azure::Core::Http::HttpRange(); + options.Range.Value().Offset = offset; + options.Range.Value().Length = length; + options.RangeHashAlgorithm = Azure::Storage::HashAlgorithm::Crc64; + } + else + { + throw seismicdrive::SDExceptionAzureStorageError("download_to_range() : overflow for offset or length parameter"); + } + + // Download data + Azure::Core::Context context; + context = context.WithDeadline(std::chrono::system_clock::now() + std::chrono::seconds(DOWNLOAD_TIMEOUT)); + auto res = blobClient.Download(options, context); + res.Value.BodyStream->ReadToCount(reinterpret_cast(target), length, context); + + // Check data integrity + Azure::Storage::Crc64Hash instance; + auto crcPassed = res.Value.TransactionalContentHash.Value().Value == instance.Final(reinterpret_cast(target), length); + if (!crcPassed) + { + throw seismicdrive::SDExceptionAzureStorageError("download_to_range() : data integrity check failed"); + } + } + + void CloudBlobClient::download_to( + const std::string &container_name, + const std::string &blob_name, + char *target, + std::uint64_t length, + const GcsContext *pContext) + { + download_to_range(container_name, blob_name, target, 0, length, pContext); + } + + long long CloudBlobClient::get_blob_size( + const std::string &container_name, + const std::string &blob_name, + const GcsContext * /*pContext*/) + { + auto containerClient = _blob_service_client.GetBlobContainerClient(container_name); + auto blobClient = containerClient.GetBlockBlobClient(blob_name); + try + { + auto properties = blobClient.GetProperties(); + return properties.Value.BlobSize; + } + catch (const Azure::Storage::StorageException &e) + { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) + { + throw seismicdrive::SDExceptionAzureStorageError("Blob does not exist"); + } + throw seismicdrive::SDExceptionAzureStorageError(e.what()); + } + } + + void CloudBlobClient::delete_blob( + const std::string &container_name, + const std::string &blob_name, + const GcsContext * /*pContext*/) + { + auto containerClient = _blob_service_client.GetBlobContainerClient(container_name); + auto blobClient = containerClient.GetBlockBlobClient(blob_name); + auto result = blobClient.DeleteIfExists(); + } + + void CloudBlobClient::list_blobs_segmented( + const std::string &container_name, + const std::string &prefix, + bool recursiveList, + std::string *nextPageToken, + std::vector> *result, + const GcsContext * /*pContext*/) + { + if (result == nullptr) + { + throw seismicdrive::SDException("Invalid argument. Result is nullptr"); + } + + const std::string delimiter = "/"; + + std::string sanitized_prefix(prefix); + if (prefix.back() != '/') + sanitized_prefix += "/"; + + ListBlobsOptions options; + options.Prefix = sanitized_prefix; + + if (nextPageToken && !nextPageToken->empty()) + { + options.ContinuationToken = *nextPageToken; + } + + auto blobContainerClient = _blob_service_client.GetBlobContainerClient(container_name); + + if (!recursiveList) + { + auto pageResult = blobContainerClient.ListBlobsByHierarchy(delimiter, options); + for (const auto &blob : pageResult.Blobs) + { + std::pair obi; + obi.first = blob.Name.substr(prefix.length()); + obi.second = blob.BlobSize; + result->emplace_back(obi); + } + if (!pageResult.NextPageToken.empty()) + { + *nextPageToken = pageResult.NextPageToken; + } + } + else + { + auto pageResult = blobContainerClient.ListBlobs(options); + for (const auto &blob : pageResult.Blobs) + { + std::pair obi; + obi.first = blob.Name.substr(prefix.length()); + obi.second = blob.BlobSize; + result->emplace_back(obi); + } + if (!pageResult.NextPageToken.empty()) + { + *nextPageToken = pageResult.NextPageToken; + } + } + } + + void CloudBlobClient::list_blobs_segmented( + const std::string &container_name, + const std::string &prefix, + bool recursiveList, + std::string *nextPageToken, + std::vector *result, + const GcsContext * /*pContext*/) + { + if (result == nullptr) + { + throw seismicdrive::SDException("Invalid argument. Result is nullptr"); + } + + const std::string delimiter = "/"; + + std::string sanitized_prefix(prefix); + if (prefix.back() != '/') + sanitized_prefix += "/"; + + ListBlobsOptions options; + options.Prefix = sanitized_prefix; + + if (nextPageToken && !nextPageToken->empty()) + { + options.ContinuationToken = *nextPageToken; + } + + auto blobContainerClient = _blob_service_client.GetBlobContainerClient(container_name); + auto storage_name_prefix = "gs://" + container_name + "/"; + + if (!recursiveList) + { + auto pageResult = blobContainerClient.ListBlobsByHierarchy(delimiter, options); + for (const auto &blob : pageResult.Blobs) + { + StorageObjectInfo obi; + obi.name = storage_name_prefix + blob.Name; + obi.size = blob.BlobSize; + result->emplace_back(obi); + } + if (!pageResult.NextPageToken.empty()) + { + *nextPageToken = pageResult.NextPageToken; + } + } + else + { + auto pageResult = blobContainerClient.ListBlobs(options); + for (const auto &blob : pageResult.Blobs) + { + StorageObjectInfo obi; + obi.name = storage_name_prefix + blob.Name; + obi.size = blob.BlobSize; + result->emplace_back(obi); + } + if (!pageResult.NextPageToken.empty()) + { + *nextPageToken = pageResult.NextPageToken; + } + } + } + + // -------------------------------- + // CloudBlobClientFactory Definition + // -------------------------------- + + class CloudBlobClientFactory : public CloudBlobClientFactoryIF + { + public: + CloudBlobClientFactory(); + ~CloudBlobClientFactory(); + std::shared_ptr create(const std::string &account_name, const std::string &sas_token) override; + }; + + CloudBlobClientFactory::CloudBlobClientFactory() {} + + CloudBlobClientFactory::~CloudBlobClientFactory() {} + + std::shared_ptr CloudBlobClientFactory::create( + const std::string &account_name, + const std::string &sas_token) + { + + return std::make_shared(account_name, sas_token); + } + + // -------------------------------- + // AzureStorageImpl Definition + // -------------------------------- + + using namespace Azure::Core::Http; + + // Source: https://docs.microsoft.com/en-us/azure/architecture/best-practices/retry-service-specific + std::unordered_set RetryHttpStatusCodes({ + HttpStatusCode::RequestTimeout, // 408 + HttpStatusCode::TooManyRequests, // 429 + HttpStatusCode::InternalServerError, // 500 + HttpStatusCode::BadGateway, // 502 + HttpStatusCode::ServiceUnavailable, // 503 + HttpStatusCode::GatewayTimeout // 504 + }); + + class AzureStorageImpl + { + + public: + AzureStorageImpl(void *auth_provider, const std::string &sdresource = "", const bool readonly = false); + ~AzureStorageImpl(); + AzureStorageImpl(const AzureStorageImpl &rhs) = delete; + AzureStorageImpl &operator=(const AzureStorageImpl &rhs) = delete; + bool ObjectExists(const std::string &cs_path, const GcsContext *pContext); + std::size_t objectSize(const std::string &cs_path, const GcsContext *pContext = nullptr); + std::vector objectsSize(const std::vector &objects, const GcsContext *pContext); + void uploadObject(const std::string &cs_path, const void *data, std::size_t len, const GcsContext *pContext); + void uploadObject(const std::string &cs_path, const std::string &file_name, const GcsContext *pContext); + void downloadObject(const std::string &cs_path, void *data, std::size_t offset, std::size_t len, const GcsContext *pContext); + void downloadObject(const std::string &cs_path, void *data, std::size_t &len, const GcsContext *pContext); + void downloadObject(const std::string &cs_path, void **data, std::size_t &len, const GcsContext *pContext); + void deleteObject(const std::string &cs_path, const GcsContext *pContext); + void lockObject(const std::string &cs_path, const GcsContext *pContext); + void waitLockObject(const std::string &cs_path, const GcsContext *pContext); + void unlockObject(const std::string &cs_path, const GcsContext *pContext); + void getChildren(const std::string &objPath, bool recursiveList, std::string *nextPageToken, std::vector *result, const GcsContext *pContext); + void setHttpContext(const HttpContext *pContext); + void setExponentialRetryBackoffPolicy(const ExponentialRetryBackoffPolicy *policy); + std::shared_ptr getBlobClient(); + TokenInfo getCachedStorageAccessToken(const std::string &resourceRef); + void setCachedStorageAccessToken(const std::string &resourceRef, const TokenInfo &credentials); + std::vector> ls(const std::string &objPath, const GcsContext *pContext); + + private: + std::unique_ptr _context; + std::unique_ptr _expContext; + + AuthProvider *_auth_provider; + const std::shared_ptr _storage_auth_provider; + const std::string _sdresource; + const bool _sdreadonly; + std::string _sas_token; + std::shared_ptr _blob_client; + std::shared_ptr _blob_client_factory; + std::mutex mutex; + + friend class AzureStorageCurlTest; + + void setDefaultCallbacks(HTTPManager &); + + void setRequestContext(HTTPRequest &req, const GcsContext *pContext); + }; + + // -------------------------------- + // AzureStorageImpl Declaration + // -------------------------------- + + AzureStorageImpl::AzureStorageImpl( + void *auth_provider, + const std::string &sdresource, + const bool readonly) + : _auth_provider(reinterpret_cast(auth_provider)), + _storage_auth_provider(std::make_shared()), + _sdresource(sdresource), + _sdreadonly(readonly), + _blob_client_factory(std::make_shared()) + { + // Enable logging + // azure::storage::operation_context::set_default_log_level(azure::storage::client_log_level::log_level_verbose); + auto httpContext = new HttpContext(); + httpContext->timeoutSecs = 3600; + httpContext->lowSpeedLimitBytesPerSec = 30; + httpContext->lowSpeedTimeSecs = 15; + _context.reset(httpContext); + + _expContext.reset(new ExponentialRetryBackoffPolicy()); + } + + AzureStorageImpl::~AzureStorageImpl() {} + + void AzureStorageImpl::setDefaultCallbacks( + HTTPManager & /*http*/) + { + throw seismicdrive::SDException("Not supported"); + } + + void AzureStorageImpl::setRequestContext( + HTTPRequest &req, + const GcsContext *pContext) + { + req.setHttpContext(pContext ? pContext : _context.get()); + } + + TokenInfo AzureStorageImpl::getCachedStorageAccessToken( + const std::string& resourceRef) + { + auto credentials = _storage_auth_provider->getCachedToken(resourceRef); + if (!credentials.token.empty()) + return { credentials.token, credentials.expire }; + _auth_provider->getGcsAuthToken(_sdresource, _sdreadonly, _storage_auth_provider); + auto t = _storage_auth_provider->getCachedToken(resourceRef); + return { t.token, t.expire }; + } + + void AzureStorageImpl::setCachedStorageAccessToken( + const std::string& resourceRef, + const TokenInfo &credentials) + { + _storage_auth_provider->setCachedToken(resourceRef, credentials); + } + + bool AzureStorageImpl::ObjectExists( + const std::string &cs_path, + const GcsContext * pContext) + { + bool temp = false; + std::string container, block_blob; + parse_azstorage_path(cs_path, container, block_blob); + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::ObjectExists()", + [&]() -> void { + temp = blob_client->exists(container, block_blob, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + return temp; + } + + void AzureStorageImpl::uploadObject( + const std::string &cs_path, + const void *data, + std::size_t len, + const GcsContext * pContext) + { + std::string container, block_blob; + parse_azstorage_path(cs_path, container, block_blob); + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::uploadObject()", + [&]() -> void { + blob_client->upload_from(container, block_blob, reinterpret_cast(data), len, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + } + + void AzureStorageImpl::uploadObject( + const std::string &cs_path, + const std::string &file_name, + const GcsContext * pContext) + { + std::string container, block_blob; + parse_azstorage_path(cs_path, container, block_blob); + + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::uploadObject()", + [&]() -> void { + blob_client->upload_from_file(container, block_blob, file_name, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + } + + void AzureStorageImpl::downloadObject( + const std::string &cs_path, + void *data, + std::size_t offset, + std::size_t len, + const GcsContext * pContext) + { + std::string container, block_blob; + parse_azstorage_path(cs_path, container, block_blob); + + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::downloadObject()", + [&]() -> void { + blob_client->download_to_range(container, block_blob, reinterpret_cast(data), offset, len, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + } + + void AzureStorageImpl::downloadObject( + const std::string &cs_path, + void *data, + std::size_t &len, + const GcsContext * pContext) + { + std::string container, block_blob; + parse_azstorage_path(cs_path, container, block_blob); + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::downloadObject()", + [&]() -> void { + blob_client->download_to(container, block_blob, reinterpret_cast(data), len, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + } + + void AzureStorageImpl::downloadObject( + const std::string &cs_path, + void **data, + std::size_t &len, + const GcsContext * pContext) + { + std::string container, block_blob; + parse_azstorage_path(cs_path, container, block_blob); + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::downloadObject()", + [&]() -> void { + blob_client->download_to(container, block_blob, reinterpret_cast(*data), len, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + } + + void AzureStorageImpl::deleteObject( + const std::string &cs_path, + const GcsContext * pContext) + { + std::string container, block_blob; + parse_azstorage_path(cs_path, container, block_blob); + + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::deleteObject()", + [&]() -> void { + blob_client->delete_blob(container, block_blob, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + } + + std::size_t AzureStorageImpl::objectSize( + const std::string &cs_path, + const GcsContext * pContext) + { + std::size_t temp = 0; + std::string container, block_blob; + parse_azstorage_path(cs_path, container, block_blob); + + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::objectSize()", + [&]() -> void { + temp = blob_client->get_blob_size(container, block_blob, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + return temp; + } + + std::vector AzureStorageImpl::objectsSize( + const std::vector &cs_paths, + const GcsContext * pContext) + { + // TODO: Replace with batch API when Azure Storage Client Library for C++ supports it + + const std::size_t cs_paths_size = cs_paths.size(); + std::vector sizes(cs_paths_size, -1); + auto blob_client = getBlobClient(); + for (std::size_t i = 0; i < cs_paths_size; ++i) + { + std::string container, block_blob; + parse_azstorage_path(cs_paths[i], container, block_blob); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::objectsSize()", + [&]() -> void { + sizes[i] = blob_client->get_blob_size(container, block_blob, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + } + return sizes; + } + + void AzureStorageImpl::getChildren( + const std::string &objPath, + bool recursiveList, + std::string *nextPageToken, + std::vector *result, + const GcsContext * pContext) + { + if (!result) + { + throw seismicdrive::SDExceptionGCSAccessorError(sdmex::gcs::GetChildrenEmptyResult()); + } + // Duo: need to verify if this line is needed + result->clear(); + + std::string sanitized_dirname = objPath; + if (!objPath.empty() && objPath.back() != '/') + { + sanitized_dirname += "/"; + } + + std::string container, object_prefix; + parse_azstorage_path(objPath, container, object_prefix); + + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::getChildren()", + [&]() -> void { + blob_client->list_blobs_segmented(container, object_prefix, recursiveList, nextPageToken, result, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + } + + std::vector> AzureStorageImpl::ls( + const std::string &objPath, + const GcsContext * pContext) + { + + std::string sanitized_dirname = objPath; + if (!objPath.empty() && objPath.back() != '/') + { + sanitized_dirname += "/"; + } + + std::string container, object_prefix; + parse_azstorage_path(objPath, container, object_prefix); + + std::vector> result; + + auto blob_client = getBlobClient(); + azureStorageExponentialRetry( + _expContext.get(), + "AzureStorage::getChildren()", + [&]() -> void { + blob_client->list_blobs_segmented(container, object_prefix, false, nullptr, &result, pContext); + }, + RetryHttpStatusCodes, + [&]() -> void { + blob_client = getBlobClient(); + }); + return result; + } + + void AzureStorageImpl::lockObject( + const std::string & /*cs_path*/, + const GcsContext * /*pContext*/) + { + + throw seismicdrive::SDException("Not supported"); + } + + void AzureStorageImpl::waitLockObject( + const std::string & /*cs_path*/, + const GcsContext * /*pContext*/) + { + + throw seismicdrive::SDException("Not supported"); + } + + void AzureStorageImpl::unlockObject( + const std::string & /*cs_path*/, + const GcsContext * /*pContext*/) + { + + throw seismicdrive::SDException("Not supported"); + } + + void AzureStorageImpl::setHttpContext( + const HttpContext *pContext) + { + + if (pContext) + { + pContext = new HttpContext(*pContext); + } + _context.reset(pContext); + } + + void AzureStorageImpl::setExponentialRetryBackoffPolicy( + const ExponentialRetryBackoffPolicy *policy) + { + if (policy) + { + policy = new ExponentialRetryBackoffPolicy(*policy); + } + _expContext.reset(policy); + } + + std::shared_ptr AzureStorageImpl::getBlobClient() + { + std::lock_guard guard(mutex); + + // Obtain user delegation SAS token from seismic-store-service. + const std::string sas_uri = _auth_provider->getGcsAuthToken(_sdresource, _sdreadonly, _storage_auth_provider); + + std::string account_name, + container_name, + new_sas_token; + parse_sas_uri(sas_uri, account_name, container_name, new_sas_token); + + if (_sas_token.empty() || _sas_token != new_sas_token) + { + // Re-init the blob client + _blob_client = _blob_client_factory->create(account_name, new_sas_token); + _sas_token = new_sas_token; + } + return _blob_client; + } + + // -------------------------------- + // AzureStorage + // -------------------------------- + + AzureStorage::AzureStorage() + : _impl(new AzureStorageImpl(NULL)) + { + } + + AzureStorage::AzureStorage( + void *auth_provider, + const std::string &sdresource, + const bool readonly) + : _impl(new AzureStorageImpl(auth_provider, sdresource, readonly)) + { + } + + AzureStorage::~AzureStorage() = default; + + bool AzureStorage::objectExists( + const std::string &cs_path, + const GcsContext *pContext) + { + return _impl->ObjectExists(cs_path, pContext); + } + + long long AzureStorage::objectSize( + const std::string &cs_path, + const GcsContext *pContext) + { + return _impl->objectSize(cs_path, pContext); + } + + void AzureStorage::uploadObject( + const std::string &cs_path, + const void *data, + std::size_t len, + const GcsContext *pContext) + { + return _impl->uploadObject(cs_path, data, len, pContext); + } + + void AzureStorage::downloadObject( + const std::string &cs_path, + void *data, + std::size_t offset, + std::size_t len, + const GcsContext *pContext) + { + _impl->downloadObject(cs_path, data, offset, len, pContext); + } + + void AzureStorage::downloadObject( + const std::string &cs_path, + void *data, + std::size_t &len, + const GcsContext *pContext) + { + _impl->downloadObject(cs_path, data, len, pContext); + } + + void AzureStorage::downloadObject( + const std::string &cs_path, + void **data, + std::size_t &len, + const GcsContext *pContext) + { + _impl->downloadObject(cs_path, data, len, pContext); + } + + void AzureStorage::deleteObject( + const std::string &cs_path, + const GcsContext *pContext) + { + _impl->deleteObject(cs_path, pContext); + } + + std::vector AzureStorage::objectsSize( + const std::vector &cs_paths, + const GcsContext *pContext) + { + return _impl->objectsSize(cs_paths, pContext); + } + + void AzureStorage::uploadObject( + const std::string &cs_path, + const std::string &file_name, + const GcsContext *pContext) + { + _impl->uploadObject(cs_path, file_name, pContext); + } + + void AzureStorage::lockObject( + const std::string &cs_path, + const GcsContext *pContext) + { + _impl->lockObject(cs_path, pContext); + } + + void AzureStorage::waitLockObject( + const std::string &cs_path, + const GcsContext *pContext) + { + _impl->waitLockObject(cs_path, pContext); + } + + void AzureStorage::unlockObject( + const std::string &cs_path, + const GcsContext *pContext) + { + _impl->unlockObject(cs_path, pContext); + } + + void AzureStorage::setHttpContext( + const HttpContext *pContext) + { + _impl->setHttpContext(pContext); + } + + void AzureStorage::setExponentialRetryBackoffPolicy( + const ExponentialRetryBackoffPolicy *policy) + { + _impl->setExponentialRetryBackoffPolicy(policy); + } + + std::pair AzureStorage::getCachedStorageAccessToken( + const std::string &resourceRef) + { + auto t = _impl->getCachedStorageAccessToken(resourceRef); + return { t.token, t.expire }; + } + + void AzureStorage::setCachedStorageAccessToken( + const std::string &resourceRef, + std::pair credentials) + { + _impl->setCachedStorageAccessToken(resourceRef, { credentials.first, credentials.second }); + } + + void AzureStorage::getChildren( + const std::string &objPath, + bool recursiveList, + std::string *nextPageToken, + std::vector *result, + const GcsContext *pContext) + { + return _impl->getChildren(objPath, recursiveList, nextPageToken, result, pContext); + } + + std::vector> AzureStorage::ls( + const std::string &objPath, + const GcsContext *pContext) + { + return _impl->ls(objPath, pContext); + } + +} } // namespace seismicdrive::curl \ No newline at end of file diff --git a/src/src/lib/cloud/providers/azure/curl/AzureStorage.h b/src/src/lib/cloud/providers/azure/curl/AzureStorage.h new file mode 100644 index 0000000000000000000000000000000000000000..452906e4b5daba125581b0e572b6d8a4c2f80028 --- /dev/null +++ b/src/src/lib/cloud/providers/azure/curl/AzureStorage.h @@ -0,0 +1,327 @@ +// ============================================================================ +// Copyright 2017-2021, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#pragma once + +#include "cloud/providers/Storage.h" + +namespace seismicdrive { namespace curl { + + // For the only purpose of mocking + class AzureStorageImpl; + + // -------------------------------- + // Azure Cloud Blob Client Interface + // -------------------------------- + + struct CloudBlobClientIF + { + virtual ~CloudBlobClientIF() = default; + virtual bool exists(const std::string &container_name, const std::string &blob_name, const GcsContext *pContext) = 0; + virtual void upload_from_file(const std::string &container_name, const std::string &blob_name, const std::string &file_name, const GcsContext *pContext) = 0; + virtual void upload_from(const std::string &container_name, const std::string &blob_name, const char *source, std::uint64_t length, const GcsContext *pContext) = 0; + virtual void download_to_range(const std::string &container_name, const std::string &blob_name, char *target, std::uint64_t offset, std::uint64_t length, const GcsContext *pContext) = 0; + virtual void download_to(const std::string &container_name, const std::string &blob_name, char *target, std::uint64_t length, const GcsContext *pContext) = 0; + virtual long long get_blob_size(const std::string &container_name, const std::string &blob_name, const GcsContext *pContext) = 0; + virtual void delete_blob(const std::string &container_name, const std::string &blob_name, const GcsContext *pContext) = 0; + virtual void list_blobs_segmented(const std::string &container_name, const std::string &prefix, bool recursiveList, std::string *nextPageToken, std::vector *result, const GcsContext *pContext) = 0; + virtual void list_blobs_segmented(const std::string &container_name, const std::string &prefix, bool recursiveList, std::string *nextPageToken, std::vector> *result, const GcsContext *pContext) = 0; + }; + struct CloudBlobClientFactoryIF + { + virtual ~CloudBlobClientFactoryIF() = default; + virtual std::shared_ptr create(const std::string &account_name, const std::string &sas_token) = 0; + }; + + /*******************************************************************************/ /** + * @class AzureStorage AzureStorage.h + * + * @brief This class is an accessor for the Azure blob storage objects. + * + * This class handles objects located in a given Azure blob storage path. + * + ******************************************************************************/ + + class AzureStorage : public StorageIF + { + + public: + /*******************************************************************************/ /** + * @brief Default constructor. + * + * This contructor creates a AzureStorage object and + * initialize the dataset info. + * + ******************************************************************************/ + AzureStorage(); + + /*******************************************************************************/ /** + * @brief Parameterized contructor + * + * This constructor initializes the accessor object using user-defined arguments. + * + * @param auth_provider is the auth provider + * @param sdresource is the seismic drive subproject resource (optional if the auth provider is a google default one) + * @param readonly define the access policy (read or readwrite) + ******************************************************************************/ + AzureStorage(void *auth_provider, const std::string &sdresource = "", const bool readonly = false); + + /*******************************************************************************/ /** + * @brief Copy Constructor. + * + ******************************************************************************/ + AzureStorage(const AzureStorage &rhs) = delete; + + /*******************************************************************************/ /** + * @brief Copy Assignment Operator. + * + ******************************************************************************/ + AzureStorage &operator=(const AzureStorage &) = delete; + + /*******************************************************************************/ /** + * @brief Move Constructor. + * + ******************************************************************************/ + AzureStorage(AzureStorage &&op) noexcept; + + /*******************************************************************************/ /** + * @brief Move Assignment Operator. + * + ******************************************************************************/ + AzureStorage &operator=(AzureStorage &&op) noexcept; + + /*******************************************************************************/ /** + * @brief Destructor + * + * This destructor deletes the AzureStorage object and + * free up the memory after it goes out of scope. + * + ******************************************************************************/ + ~AzureStorage(); + + /*******************************************************************************/ /** + * @brief Checks if an object exists + * + * This method checks if an object exists on Azure in a certain path. + * + * @param cs_path is the cloud storage URI of the resource + * @param pContext is the cloud storage context of the access request + * + * @return true if tge object exists, false if the object not exist + ******************************************************************************/ + bool objectExists(const std::string &cs_path, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Gets the size of an object + * + * This method gets the size of an object in a given cloud storage path + * + * @param cs_path is the cloud storage URI of the resource object + * @param pContext is the cloud storage context of the access request + * + * @return the size of the object in bytes + ******************************************************************************/ + long long objectSize(const std::string &cs_path, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Upload/Create an object from a buffer + * + * This method creates an object from a given data (buffer) and upload it to Azure. + * + * @param cs_path is the cloud storage URI of the resource object + * @param data start of data to store in the object + * @param len number of bytes to store + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void uploadObject(const std::string &cs_path, const void *data, std::size_t len, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Download all or part of an object + * + * This method downloads from an Azure cloud storage object given the start of the buffer with the wanted offset. + * This method doesnt check crc32c on partial download + * + * @param cs_path is the cloud storage URI of the resource object + * @param data start of data buffer to receive object contents + * @param offset first byte in the object to return + * @param len number of bytes to download + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void downloadObject(const std::string &cs_path, void *data, std::size_t offset, std::size_t len, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Download an object from Azure + * + * This method download an object from Azure from start to end. + * The crc32c is checked on downloaded object. + * + * @param cs_path is the cloud storage URI of the resource object + * @param data start of data buffer to receive object contents + * @param len the read length will be saved/returned in this parameter + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void downloadObject(const std::string &cs_path, void *data, std::size_t &len, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Download an object from Azure + * + * This method download an object from Azure by dynamically allocating the memory of the "data" buffer with the right object size. + * Memory is allocated in the method and the size will be saved and returned in the len parameter. + * The crc32c is checked on downloaded object. + * + * @param cs_path is the cloud storage URI of the resource object + * @param data start of data buffer to receive object contents + * @param len the length of the downloaded object will be saved/returned in the len parameter + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void downloadObject(const std::string &cs_path, void **data, std::size_t &len, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Delete an object + * + * This method deletes an object from Azure given its path. + * + * @param cs_path is the cloud storage URI of the resource object to delete + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void deleteObject(const std::string &cs_path, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Return sizes of several objects + * + * This method gets the sizes of a list of objects given their Azure cloud storage paths + * + * @param cs_paths is a vector of cloud storage objects URI + * @param pContext is the cloud storage context of the access request + * + * @return size of corresponding object or -1 + ******************************************************************************/ + std::vector objectsSize(const std::vector &cs_paths, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Upload (create) an object from a local file + * + * This method creates an object identical to a local file and upload it to Azure. + * + * @param cs_path name of object to create + * @param file_name name of file to be uploaded to object + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void uploadObject(const std::string &cs_path, const std::string &file_name, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Creates a lock file at a Azure cloud storage path + * + * This method locks the object located in the given path. + * + * @param cs_path is the cloud storage URI of the resource object + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void lockObject(const std::string &cs_path, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief waits for the lock to be released + * + * This method waits for the lock placed on the object in the given Azure cloud storage path to be released. + * + * @param cs_path is the cloud storage URI of the resource object + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void waitLockObject(const std::string &cs_path, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Unlock file at a Azure cloud storage path + * + * This method removes the lock from the object in the given path. + * + * @param cs_path is the cloud storage URI of the resource object + * @param pContext is the cloud storage context of the access request + ******************************************************************************/ + void unlockObject(const std::string &cs_path, const GcsContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Set a default http context for access requests + * + * This method sets a default http context for access requests. + * + * @param pContext is the http context of the access request + ******************************************************************************/ + void setHttpContext(const HttpContext *pContext = nullptr) override; + + /*******************************************************************************/ /** + * @brief Set a default retry policy strategy for http requests + * + * This method sets a default http retry policy strategy. + * + * @param policy is the back-off retry policy to apply on request + ******************************************************************************/ + void setExponentialRetryBackoffPolicy(const ExponentialRetryBackoffPolicy *policy); + + /*******************************************************************************/ /** + * @brief get the storage access token for a resource + * + * This method get the storage access token for a resource + * + * @param resourceRef the reference resource (subproject path string + readonly as "true"/"false"); + ******************************************************************************/ + std::pair getCachedStorageAccessToken(const std::string &resourceRef); + + /*******************************************************************************/ /** + * @brief set the storage access token for a resource + * + * This method get the internal storage access token for a specific resource + * + * @param resourceRef the reference resource (subproject path string + readonly as "true"/"false"); + * @param credentials the credentials as + ******************************************************************************/ + void setCachedStorageAccessToken( + const std::string &resourceRef, std::pair credentials); + + /*******************************************************************************/ /** + * @brief return the object children list + * + * This method return the object children list for a specified storage path prefix + * + * @param objPath the reference resource + * @param recursiveList true to return the recursive object children list + * @param nextPageToken continuation token + * @param result results will be returned in this vector + * @param pContext is the http context of the access request + ******************************************************************************/ + void getChildren( + const std::string &objPath, bool recursiveList, std::string *nextPageToken, + std::vector *result, const GcsContext *pContext); + + /*******************************************************************************/ /** + * @brief get all objects in a container resource (prefix) + * + * This method takes a storage path and return a list of all the objects that exists in that path. + * + * @param objPath the reference resource + * @param pContext is the http context of the access request + * + * @return a list of the object names in the given storage path. + ******************************************************************************/ + std::vector> ls(const std::string &objPath, const GcsContext *pContext = nullptr) override; + + private: + std::shared_ptr _impl; + + friend class AzureStorageCurlTest; + }; + +} } // namespace seismicdrive::curl + diff --git a/src/src/lib/shared/config.h b/src/src/lib/shared/config.h index 3901f221d4e9bd3616256c240db23db379d99d20..27da4702e929c7f8e132b7912cb4d18dd700d2d7 100644 --- a/src/src/lib/shared/config.h +++ b/src/src/lib/shared/config.h @@ -48,9 +48,7 @@ namespace seismicdrive { // Disable SSL Certificate Symbol static const std::string DISABLE_SSL_CERTIFICATE_VERIFICATION{"DISABLE_SSL_CERTIFICATE_VERIFICATION"}; - - // Set to allow curl to use signal handlers. - static const std::string ENABLE_CURL_SIGNAL_HANDLERS{"ENABLE_CURL_SIGNAL_HANDLERS"}; + } } diff --git a/src/test/seismic-store-lib/CMakeLists.txt b/src/test/seismic-store-lib/CMakeLists.txt index 7dacc616f8e68b177736d7e5193755a9312cb67b..6008e93f602976ad9244c4ad01572cba91e5230f 100644 --- a/src/test/seismic-store-lib/CMakeLists.txt +++ b/src/test/seismic-store-lib/CMakeLists.txt @@ -28,7 +28,7 @@ target_link_libraries(gcs_accessor ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CRC32C_L # AZURE STORAGE TEST if(AZURE_PROVIDER_ENABLED) - add_executable(azure_storage ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/lib/cloud/providers/azure/AzureStorage.cc ../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/direct_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp azure_storage.cc) + add_executable(azure_storage ../../src/lib/accessors/GcsAccessor.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../shared/utils.cc ../../src/lib/cloud/providers/azure/cpprest/AzureStorage.cc ../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/auth/direct_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp azure_storage.cc) add_dependencies(azure_storage crc32c) set_target_properties(azure_storage PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib") target_include_directories(azure_storage PRIVATE ${CRC32CLIB_DIR}/include) @@ -67,6 +67,45 @@ if(AZURE_PROVIDER_ENABLED) endif() endif ( AZURE_PROVIDER_ENABLED ) +# AZURE STORAGE TEST +if(AZURE_CURL_PROVIDER_ENABLED) + + add_executable(azure_storage_curl + ../../src/lib/accessors/GcsAccessor.cc + ../../src/lib/shared/base64.cc + ../../src/lib/shared/utils.cc + ../shared/utils.cc + ../../src/lib/cloud/providers/azure/curl/AzureStorage.cc + ../../src/core/SDException.cc ../../src/lib/auth/gcs_auth_provider.cc + ../../src/lib/auth/direct_auth_provider.cc + ../../src/lib/auth/service_account.cc + ../../src/lib/auth/google_auth_provider.cc + ../../src/lib/http/http_request.cc + ../../src/lib/json/jsoncpp.cpp + azure_storage_curl.cc) + add_dependencies(azure_storage_curl crc32c) + set_target_properties(azure_storage_curl PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib") + target_include_directories(azure_storage_curl PRIVATE ${CRC32CLIB_DIR}/include) + + if(NOT WIN32) + target_link_libraries(azure_storage_curl PRIVATE + Azure::azure-storage-blobs + LibXml2::LibXml2 + ${CURL_LIBRARY} # CURL::libcurl + ${OPENSSL_LIBRARY} + ${CRC32C_LIBRARY} # Crc32c::crc32c + ${CMAKE_DL_LIBS}) + else() + target_link_libraries(azure_storage_curl + PRIVATE + Azure::azure-storage-blobs + ${CURL_LIBRARY} + ${OPENSSL_LIBRARY} + ${CRC32C_LIBRARY}) + endif() +endif ( AZURE_CURL_PROVIDER_ENABLED ) + + if ( AWS_PROVIDER_ENABLED ) find_package( Crc32c ) @@ -143,4 +182,4 @@ endif() # AUTH PROVIDER TEST add_executable(auth_provider ../../src/lib/auth/gcs_auth_provider.cc ../../src/lib/shared/base64.cc ../../src/lib/shared/utils.cc ../../src/lib/shared/utils_ext.cc ../shared/utils.cc ../../src/core/SDException.cc ../../src/lib/auth/google_auth_provider.cc ../../src/lib/auth/service_account.cc ../../src/lib/auth/imp_auth_provider.cc ../../src/lib/http/http_request.cc ../../src/lib/json/jsoncpp.cpp auth_provider.cc) set_target_properties(auth_provider PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/seismic-store-lib") -target_link_libraries(auth_provider ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CMAKE_DL_LIBS}) +target_link_libraries(auth_provider ${CURL_LIBRARY} ${OPENSSL_LIBRARY} ${CMAKE_DL_LIBS}) \ No newline at end of file diff --git a/src/test/seismic-store-lib/azure_storage.cc b/src/test/seismic-store-lib/azure_storage.cc index 68d775d7b5f4e13837c20d8d8990ae710450d7f4..45ae053163136b9cdaf971600cf4e4a2ef2f3595 100644 --- a/src/test/seismic-store-lib/azure_storage.cc +++ b/src/test/seismic-store-lib/azure_storage.cc @@ -18,7 +18,7 @@ #define NO_DLL_PUBLIC #endif -#include "cloud/providers/azure/AzureStorage.h" +#include "cloud/providers/azure/cpprest/AzureStorage.h" #include "auth/direct_auth_provider.h" #include "auth/google_auth_provider.h" #include "../utest/mocks/auth_provider_mock.h" @@ -35,7 +35,7 @@ #include #include -using namespace seismicdrive; +using namespace seismicdrive::cpprest; std::string SDURL(""); std::string SDAPIKEY(""); @@ -162,7 +162,7 @@ int main() { SUBPROJECT = subproject; DATASET = dataset; - DirectAuthProvider auth(SDURL, SDAPIKEY, TOKEN); + seismicdrive::DirectAuthProvider auth(SDURL, SDAPIKEY, TOKEN); std::string sdresource = "sd://" + TENANT + '/' + SUBPROJECT + '/' + DATASET; AzureStorage accessor(&auth, sdresource); const std::string sas_uri = auth.getGcsAuthToken(sdresource, false); diff --git a/src/test/seismic-store-lib/azure_storage_curl.cc b/src/test/seismic-store-lib/azure_storage_curl.cc new file mode 100644 index 0000000000000000000000000000000000000000..3f9eec51fc1ad39eaea9b02f5ad1ec5b6c40d3da --- /dev/null +++ b/src/test/seismic-store-lib/azure_storage_curl.cc @@ -0,0 +1,221 @@ +// ============================================================================ +// Copyright 2017-2021, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifdef _MSC_VER + #define NO_DLL_PUBLIC +#endif + +#include "cloud/providers/azure/curl/AzureStorage.h" +#include "auth/direct_auth_provider.h" +#include "auth/google_auth_provider.h" +#include "../utest/mocks/auth_provider_mock.h" +#include "../shared/utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace seismicdrive::curl; + +std::string SDURL(""); +std::string SDAPIKEY(""); +std::string TOKEN(""); +std::string TENANT(""); +std::string SUBPROJECT(""); +std::string DATASET(""); + +inline void div() { + std::cout << line(40); +} + +inline std::string to_string_t(const std::string &str) { +#ifdef _WIN32 + // Windows uses wstring + std::wstring_convert> converter; + std::wstring wstr = converter.from_bytes(str); + return wstr; +#else + return str; +#endif +} + +void upload(AzureStorage *accessor, const std::vector &objs, const std::string &data) { + for (auto &obj : objs) { + std::cout << "Uploading object: " << obj << '\n'; + accessor->uploadObject(obj, data.c_str(), data.length()); + } +} + +void exist(AzureStorage *accessor, const std::vector &objs, bool test_fake = false) { + for (auto &obj : objs) { + std::cout << "Exist " << obj << ": " << accessor->objectExists(obj) << '\n'; + } + // test not exist + if (test_fake) { + std::cout << "Exist " << (objs[0] + "_fake") << ": " << accessor->objectExists(objs[0] + "_fake") << '\n'; + } +} + +void size(AzureStorage *accessor, const std::vector &objs) { + for (auto &obj : objs) { + std::cout << "Size " << obj << ": " << accessor->objectSize(obj) << '\n'; + } +} + +void download(AzureStorage *accessor, const std::vector &objs, size_t size) { + std::string rbuf(size, '\0'); + for (auto &obj : objs) { + std::cout << "Downloading object: " << obj << " -> "; + accessor->downloadObject(obj, &rbuf[0], 0, size); + std::cout << rbuf << '\n'; + } +} + +void downloadpartial(AzureStorage *accessor, const std::vector &objs, size_t offset, size_t size) { + for (auto &obj : objs) { + char rbuf[4]; + std::cout << "Downloading object: " << obj << " from byte " << offset << " to " << (offset + size) << " -> "; + accessor->downloadObject(obj, rbuf, offset, size); + std::cout << std::string(rbuf, size) << '\n'; + } +} + +void deleteobj(AzureStorage *accessor, const std::vector &objs) { + for (auto &obj : objs) { + std::cout << "Delete Object " << obj << '\n'; + accessor->deleteObject(obj); + } +} + +void ls(AzureStorage *accessor, const std::string &uri) { + auto list = accessor->ls(uri); + for (auto &item : list) { + std::cout << "Item list: " << item.first << '\n'; + } +} + +void uploadfromfile(AzureStorage *accessor, const std::string &obj, const std::string &filename, const std::string &data) { + std::ofstream fs; + fs.open(filename); + fs << data << '\n'; + fs.close(); + std::cout << "Uploading object: " << obj << '\n'; + accessor->uploadObject(obj, filename); + std::remove(filename.c_str()); +} + +int main() { + + try { + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable: 4996) +#endif + // Load environment variables + const char *sdurl = getenv("SDURL"), + *sdapikey = getenv("SDAPIKEY"), + *token = getenv("TOKEN"), + *tenant = getenv("TENANT"), + *subproject = getenv("SUBPROJECT"), + *dataset = getenv("DATASET"); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + if (!sdurl || !sdapikey || !token || !tenant || !subproject || !dataset) { + std::cout << line(50) + << "- Missing environment variables: \n" + << "- SDURL\n" + << "- SDAPIKEY\n" + << "- TOKEN\n" + << "- TENANT\n" + << "- SUBPROJECT\n" + << "- DATASET\n" + << line(50); + exit(1); + } + + SDURL = sdurl; + SDAPIKEY = sdapikey; + TOKEN = token; + TENANT = tenant; + SUBPROJECT = subproject; + DATASET = dataset; + + seismicdrive::DirectAuthProvider auth(SDURL, SDAPIKEY, TOKEN); + std::string sdresource = "sd://" + TENANT + '/' + SUBPROJECT + '/' + DATASET; + AzureStorage accessor(&auth, sdresource); + const std::string sas_uri = auth.getGcsAuthToken(sdresource, false); + const std::regex re("https://(.*).blob.core.windows.net/(.*)\\?(.*)"); + std::smatch match; + std::regex_match(sas_uri, match, re); + std::string base_uri = "gs://" + match[2].str() + "/azapitest/"; + + const int datasize = 32, + nobj = 5; + + std::vector objs(5, base_uri); + std::string data = "1234abcd5678efgh1234abcd5678efgh", + composeobj = base_uri + "compose", + filename = "test_azure_storage.txt", + fileobj = base_uri + "test_azure_storage.txt", + lockobj = base_uri + "lockfile"; + for (int i = 0; i < nobj; i++) { + objs[i] += std::to_string(i); + } + auto objscomp = objs; objscomp.emplace_back(composeobj); + std::vector> metadata_put; + metadata_put.emplace_back(std::make_pair("k1", "v1")); + metadata_put.emplace_back(std::make_pair("k2", "v2")); + metadata_put.emplace_back(std::make_pair("k3", "v3")); + + std::cout << frame("TEST SINGLE OBJECT METHODS", '='); + upload(&accessor, objs, data); + div(); exist(&accessor, objs, true); + div(); size(&accessor, objs); + div(); download(&accessor, objs, datasize); + div(); downloadpartial(&accessor, objs, 16, 4); + div(); deleteobj(&accessor, objs); + div(); exist(&accessor, objs); + + std::cout << frame("TEST UPLOAD FROM FILE", '='); + uploadfromfile(&accessor, fileobj, filename, data); + div(); exist(&accessor, { fileobj }, true); + div(); size(&accessor, { fileobj }); + div(); download(&accessor, { fileobj }, datasize); + div(); downloadpartial(&accessor, { fileobj }, 16, 4); + div(); deleteobj(&accessor, { fileobj }); + div(); exist(&accessor, { fileobj }); + + // -------------------------- + // Test list + // -------------------------- + objs.emplace_back(base_uri + "testdir/it_must_no_ret_in_ls.txt"); + div(); upload(&accessor, objs, data); + ls(&accessor, base_uri); + deleteobj(&accessor, { objs }); + } + catch(std::runtime_error &exc) { + std::cout << exc.what() << '\n'; + } + + return 0; +} diff --git a/src/test/utest/main.cc b/src/test/utest/main.cc index 02a63eeffef21845d2a78da58e332aafbe4a8d51..0a9323eaa2f0c94268a3fe62ecc50dc963227c46 100644 --- a/src/test/utest/main.cc +++ b/src/test/utest/main.cc @@ -21,6 +21,9 @@ #ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER #include "utest/utest_lib_accessors_azure.h" #endif +#ifdef HAS_AZURE_BLOB_STORAGE_PROVIDER_CURL + #include "utest/utest_lib_accessors_azure_curl.h" +#endif #include "utest/utest_lib_shared.h" #include "utest/utest_core_dataset.h" #include "utest/utest_core_sd_manager.h" diff --git a/src/test/utest/mocks/azure_cloud_blob_client_curl_mock.h b/src/test/utest/mocks/azure_cloud_blob_client_curl_mock.h new file mode 100644 index 0000000000000000000000000000000000000000..571ba8e391c8c71c12c4a15e05f5f24d8794b1a2 --- /dev/null +++ b/src/test/utest/mocks/azure_cloud_blob_client_curl_mock.h @@ -0,0 +1,69 @@ +// ============================================================================ +// Copyright 2017-2021, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + +#ifndef SDAPI_CLOUD_BLOB_CLIENT_NEW_MOCK_H +#define SDAPI_CLOUD_BLOB_CLIENT_NEW_MOCK_H + +#include "gmock/gmock.h" +#include "cloud/providers/azure/curl/AzureStorage.h" + +namespace seismicdrive { namespace curl { + +class CloudBlobClientNewMock: public CloudBlobClientIF { + +public: + + CloudBlobClientNewMock() { } + + CloudBlobClientNewMock(const std::string &account_name, const std::string &sas_token) : + account_name(account_name), + sas_token(sas_token) { + } + + ~CloudBlobClientNewMock() { } + + MOCK_METHOD3(exists, bool(const std::string &container_name, const std::string &blob_name, const GcsContext *pContext)); + MOCK_METHOD4(upload_from_file, void(const std::string &container_name, const std::string &blob_name, const std::string &file_name, const GcsContext *pContext)); + MOCK_METHOD5(upload_from, void(const std::string &container_name, const std::string &blob_name, const char *source, std::uint64_t length, const GcsContext *pContext)); + MOCK_METHOD6(download_to_range, void(const std::string &container_name, const std::string &blob_name, char *target, uint64_t offset, uint64_t length, const GcsContext *pContext)); + MOCK_METHOD5(download_to, void(const std::string &container_name, const std::string &blob_name, char *target, std::uint64_t length, const GcsContext *pContext)); + MOCK_METHOD3(get_blob_size, long long(const std::string &container_name, const std::string &blob_name, const GcsContext *pContext)); + MOCK_METHOD3(delete_blob, void(const std::string &container_name, const std::string &blob_name, const GcsContext *pContext)); + MOCK_METHOD6(list_blobs_segmented, void(const std::string &container_name, const std::string &prefix, bool recursiveList, std::string *nextPageToken, std::vector *result, const GcsContext *pContext)); + MOCK_METHOD6(list_blobs_segmented, void(const std::string &container_name, const std::string &prefix, bool recursiveList, std::string *nextPageToken, std::vector> *result, const GcsContext *pContext)); + + std::string account_name; + std::string sas_token; + +}; + +class CloudBlobClientNewFactoryMock: public CloudBlobClientFactoryIF { + +public: + + CloudBlobClientNewFactoryMock() { } + + ~CloudBlobClientNewFactoryMock() { } + + std::shared_ptr create(const std::string& account_name, const std::string& sas_token) override { + + return std::make_shared(account_name, sas_token); + } +}; + +} } // namespace seismicdrive::curl + +#endif // SDAPI_CLOUD_BLOB_CLIENT_NEW_MOCK_H \ No newline at end of file diff --git a/src/test/utest/mocks/cloud_blob_client_mock.h b/src/test/utest/mocks/azure_cloud_blob_client_mock.h similarity index 94% rename from src/test/utest/mocks/cloud_blob_client_mock.h rename to src/test/utest/mocks/azure_cloud_blob_client_mock.h index be065946511af83cfd3789aec6993c73eca683e1..96febe8f536a228ba152622e71a546ede7e04bd4 100644 --- a/src/test/utest/mocks/cloud_blob_client_mock.h +++ b/src/test/utest/mocks/azure_cloud_blob_client_mock.h @@ -18,11 +18,11 @@ #define SDAPI_CLOUD_BLOB_CLIENT_MOCK_H #include "gmock/gmock.h" -#include "cloud/providers/azure/AzureStorage.h" +#include "cloud/providers/azure/cpprest/AzureStorage.h" #include -namespace seismicdrive { +namespace seismicdrive { namespace cpprest { class CloudBlobClientMock: public CloudBlobClientIF { @@ -66,6 +66,6 @@ public: }; -} // namespace seismicdrive +} } // namespace seismicdrive::cpprest #endif // SDAPI_CLOUD_BLOB_CLIENT_MOCK_H \ No newline at end of file diff --git a/src/test/utest/utest_auth_google.h b/src/test/utest/utest_auth_google.h index c42abb172876f952ff73bd24c411e62eb629244b..b28957ef98505086388c4844dcac6c03ad914123 100644 --- a/src/test/utest/utest_auth_google.h +++ b/src/test/utest/utest_auth_google.h @@ -19,6 +19,7 @@ #include "auth/google_auth_provider.h" #include +#include namespace seismicdrive { @@ -69,7 +70,8 @@ TEST(GoogleAuthProviderUserJsonTest, google_auth_user_json) { GoogleAuthProvider auth2; GoogleAuthProvider *auth3 = new GoogleAuthProvider(); delete auth3; - + + std::remove(usr_file_key.c_str()); } TEST(GoogleAuthProviderSvcAccountTest, google_auth_svc_account_json) { @@ -117,6 +119,7 @@ TEST(GoogleAuthProviderSvcAccountTest, google_auth_svc_account_json) { // Test Auth Token From JSON file auth2->getAuthToken(http_req_mock); + std::remove(svc_file_token.c_str()); } } diff --git a/src/test/utest/utest_lib_accessors_azure.h b/src/test/utest/utest_lib_accessors_azure.h index e48ca0f52ca90ec2b82d3aa740ae17e03080884c..7bed3afcbb4e336b05e62a91614fa64891026e10 100644 --- a/src/test/utest/utest_lib_accessors_azure.h +++ b/src/test/utest/utest_lib_accessors_azure.h @@ -15,17 +15,17 @@ // ============================================================================ -#include "cloud/providers/azure/AzureStorage.h" -#include "cloud/providers/azure/AzureStorage.cc" +#include "cloud/providers/azure/cpprest/AzureStorage.h" +#include "cloud/providers/azure/cpprest/AzureStorage.cc" #include "SDManager.h" #include "gtest/gtest.h" #include "utest/mocks/auth_provider_mock.h" -#include "utest/mocks/cloud_blob_client_mock.h" +#include "utest/mocks/azure_cloud_blob_client_mock.h" #include -namespace seismicdrive { +namespace seismicdrive { namespace cpprest { class AzureStorageTest : public ::testing::Test { @@ -348,4 +348,4 @@ TEST_F(AzureStorageTest, azstorage_objects_size) { EXPECT_THROW(azstorage.objectsSize({"gs://b/o"}), SDException); } -} // namespace seismicdrive +} } // namespace seismicdrive::ccprest diff --git a/src/test/utest/utest_lib_accessors_azure_curl.h b/src/test/utest/utest_lib_accessors_azure_curl.h new file mode 100644 index 0000000000000000000000000000000000000000..ab5f9d295572ac87ffc5379def8177454e036967 --- /dev/null +++ b/src/test/utest/utest_lib_accessors_azure_curl.h @@ -0,0 +1,287 @@ +// ============================================================================ +// Copyright 2017-2021, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================ + + +#include "cloud/providers/azure/curl/AzureStorage.h" +#include "cloud/providers/azure/curl/AzureStorage.cc" +#include "SDManager.h" + +#include "gtest/gtest.h" +#include "utest/mocks/auth_provider_mock.h" +#include "utest/mocks/azure_cloud_blob_client_curl_mock.h" + +#include +#include + +namespace seismicdrive { namespace curl { + +class AzureStorageCurlTest : public ::testing::Test { + +protected: + + AzureStorageCurlTest () { + auth.mock_gcstoken("https://mocksa.blob.core.windows.net/mockcontainer?mocktoken"); + azstorage._impl->_auth_provider = &auth; + azstorage._impl->_blob_client_factory = std::make_shared(); + update_blob_client(); + } + + void update_blob_client() { + blob_client = (CloudBlobClientNewMock*)azstorage._impl->getBlobClient().get(); + } + + std::string get_sas_token() { + return azstorage._impl->_sas_token; + } + + AzureStorage azstorage{new SDManager{"url", "key"}, "sd://tenant/subproject/dataset"}; + AuthProviderMock auth; + CloudBlobClientNewMock *blob_client; + +}; + + +TEST_F(AzureStorageCurlTest, azstorage_blob_client_get) { + + // Before token refresh + const std::string mock_token = "mocktoken"; + ASSERT_TRUE(blob_client->sas_token == mock_token); + ASSERT_TRUE(get_sas_token() == mock_token); + + // After token refresh + auth.mock_gcstoken("https://mocksa.blob.core.windows.net/mockcontainer?mocktokennew"); + const std::string new_mock_token = "mocktokennew"; + update_blob_client(); + ASSERT_TRUE(blob_client->sas_token == new_mock_token); + ASSERT_TRUE(get_sas_token() == new_mock_token); +} + +TEST_F(AzureStorageCurlTest, azstorage_object_exist) { + + // Success: true + ON_CALL(*blob_client, exists).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + return true; + }); + + EXPECT_CALL(*blob_client, exists("b", "o", nullptr)); + ASSERT_TRUE(azstorage.objectExists("gs://b/o")); + + // Success: false + ON_CALL(*blob_client, exists).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + return false; + }); + + EXPECT_CALL(*blob_client, exists("b", "o", nullptr)); + ASSERT_FALSE(azstorage.objectExists("gs://b/o")); + + // Error + ON_CALL(*blob_client, exists).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + throw SDException("error"); + return false; + }); + + EXPECT_CALL(*blob_client, exists("b", "o", nullptr)); + EXPECT_THROW(azstorage.objectExists("gs://b/o"), SDException); +} + +TEST_F(AzureStorageCurlTest, azstorage_object_size) { + + // Success + ON_CALL(*blob_client, get_blob_size).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + return 1024; + }); + + EXPECT_CALL(*blob_client, get_blob_size("b", "o", nullptr)); + long long size = azstorage.objectSize("gs://b/o"); + ASSERT_TRUE(size == 1024); + + // Error + ON_CALL(*blob_client, get_blob_size).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + throw SDException("error"); + return 0; + }); + + EXPECT_CALL(*blob_client, get_blob_size("b", "o", nullptr)); + EXPECT_THROW(azstorage.objectSize("gs://b/o"), SDException); +} + +TEST_F(AzureStorageCurlTest, azstorage_object_upload) { + + // Prepare mock data + const std::string datastr{"Hello"}; + + // Success + ON_CALL(*blob_client, upload_from).WillByDefault([datastr](std::string /*container_name*/, std::string /*blob_name*/, const char *source, std::uint64_t length, const seismicdrive::GcsContext* /*gcsContext*/) { + std::string str(source, length); + ASSERT_TRUE(str == datastr); + }); + + EXPECT_CALL(*blob_client, upload_from("b", "o",datastr.c_str(), datastr.size(), nullptr)); + azstorage.uploadObject("gs://b/o", datastr.c_str(), datastr.size()); + + // Error + ON_CALL(*blob_client, upload_from).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const char */*source*/, std::uint64_t /*length*/, const seismicdrive::GcsContext* /*gcsContext*/) { + throw SDException("error"); + }); + + EXPECT_CALL(*blob_client, upload_from("b", "o", datastr.c_str(), datastr.size(), nullptr)); + EXPECT_THROW(azstorage.uploadObject("gs://b/o", datastr.c_str(), datastr.size()), SDException); +} + +TEST_F(AzureStorageCurlTest, azstorage_object_upload_from_file) { + + // Prepare mock data + const char *file_name = "test.txt"; + + std::ofstream ofile; + ofile.open(file_name); + ofile << "Hello"; + ofile.close(); + + const std::string datastr{"Hello"}; + + // Success + ON_CALL(*blob_client, upload_from_file).WillByDefault([datastr](std::string /*container_name*/, std::string /*blob_name*/, const std::string & file_name, const seismicdrive::GcsContext* /*gcsContext*/) { + std::string result; + std::ifstream ifile; + ifile.open(file_name); + ifile >> result; + ASSERT_TRUE(result == datastr); + }); + + EXPECT_CALL(*blob_client, upload_from_file("b", "o", file_name, nullptr)); + azstorage.uploadObject("gs://b/o", file_name); + + // Error + ON_CALL(*blob_client, upload_from_file).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const std::string & /*file_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + throw SDException("error"); + }); + + EXPECT_CALL(*blob_client, upload_from_file("b", "o", file_name, nullptr)); + EXPECT_THROW(azstorage.uploadObject("gs://b/o", file_name), SDException); + + // Clean up + remove(file_name); +} + +TEST_F(AzureStorageCurlTest, azstorage_object_download_range) { + + // Prepare mock data + const std::string datastr{"Hello"}; + const uint64_t data_offset = 1; + const uint64_t data_length = 3; + char out_buffer[1024] = { 0 }; + + // Success + ON_CALL(*blob_client, download_to_range).WillByDefault([datastr, data_offset, data_length](std::string /*container_name*/, std::string /*blob_name*/, char* target, uint64_t offset, uint64_t length, const seismicdrive::GcsContext* /*gcsContext*/) { + ASSERT_TRUE(offset == data_offset); + ASSERT_TRUE(length == data_length); + strcpy(target, datastr.c_str()); + }); + + EXPECT_CALL(*blob_client, download_to_range("b", "o", out_buffer, data_offset, data_length, nullptr)); + azstorage.downloadObject("gs://b/o", (void*)out_buffer, data_offset, data_length); + std::string bufferstr(out_buffer); + ASSERT_TRUE(bufferstr == datastr); + + // Error + ON_CALL(*blob_client, download_to_range).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, char* /*target*/, uint64_t /*offset*/, uint64_t /*length*/, const seismicdrive::GcsContext* /*gcsContext*/) { + throw SDException("error"); + }); + + EXPECT_CALL(*blob_client, download_to_range("b", "o", out_buffer, data_offset, data_length, nullptr)); + EXPECT_THROW(azstorage.downloadObject("gs://b/o", (void*)out_buffer, data_offset, data_length), SDException); + +} + +TEST_F(AzureStorageCurlTest, azstorage_object_download) { + + // Prepare mock data + const std::string datastr{"Hello"}; + uint64_t data_length = 5; + char out_buffer[1024] = { 0 }; + + // Success + ON_CALL(*blob_client, download_to).WillByDefault([datastr](std::string /*container_name*/, std::string /*blob_name*/, char *target, std::uint64_t length, const seismicdrive::GcsContext* /*gcsContext*/) { + strncpy(target, datastr.c_str(), length); + }); + + EXPECT_CALL(*blob_client, download_to("b", "o", out_buffer, data_length, nullptr)); + azstorage.downloadObject("gs://b/o", (void*)out_buffer, data_length); + std::string bufferstr(out_buffer); + ASSERT_TRUE(bufferstr == datastr); + + // Error + ON_CALL(*blob_client, download_to).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, char * /*target*/, std::uint64_t /*length*/, const seismicdrive::GcsContext* /*gcsContext*/){ + throw SDException("error"); + }); + + EXPECT_CALL(*blob_client, download_to("b", "o", out_buffer, data_length, nullptr)); + EXPECT_THROW(azstorage.downloadObject("gs://b/o", (void*)out_buffer, data_length), SDException); +} + +TEST_F(AzureStorageCurlTest, azstorage_object_delete) { + + bool blob_is_deleted = false; + + // Success + ON_CALL(*blob_client, delete_blob).WillByDefault([&blob_is_deleted](std::string /*container_name*/, std::string /*blob_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + blob_is_deleted = true; + }); + + EXPECT_CALL(*blob_client, delete_blob("b", "o", nullptr)); + azstorage.deleteObject("gs://b/o"); + ASSERT_TRUE(blob_is_deleted); + + // Error + ON_CALL(*blob_client, delete_blob).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + throw SDException("error"); + return 0; + }); + + EXPECT_CALL(*blob_client, delete_blob("b", "o", nullptr)); + EXPECT_THROW(azstorage.deleteObject("gs://b/o"), SDException); +} + +TEST_F(AzureStorageCurlTest, azstorage_objects_size) { + + const std::string name1 = "x1"; + const std::string name2 = "x2"; + const uint64_t size1 = 1024; + const uint64_t size2 = 2048; + + // Success + ON_CALL(*blob_client, get_blob_size).WillByDefault([name1, size1, size2](std::string /*container_name*/, std::string blob_name, const seismicdrive::GcsContext* /*gcsContext*/) { + return blob_name == name1 ? size1 : size2; + }); + + EXPECT_CALL(*blob_client, get_blob_size("b", name1, nullptr)); + EXPECT_CALL(*blob_client, get_blob_size("b", name2, nullptr)); + auto sizes = azstorage.objectsSize({"gs://b/"+name1, "gs://b/"+name2}); + ASSERT_FALSE(sizes[0]-size1); + ASSERT_FALSE(sizes[1]-size2); + + // Error + ON_CALL(*blob_client, get_blob_size).WillByDefault([](std::string /*container_name*/, std::string /*blob_name*/, const seismicdrive::GcsContext* /*gcsContext*/) { + throw SDException("error"); + return 0; + }); + + EXPECT_CALL(*blob_client, get_blob_size("b", "o", nullptr)); + EXPECT_THROW(azstorage.objectsSize({"gs://b/o"}), SDException); +} + +} } // namespace seismicdrive::curl diff --git a/version.txt b/version.txt deleted file mode 100644 index 10cc8e0e728c15fb64889294f2d9d497ace9aa8b..0000000000000000000000000000000000000000 --- a/version.txt +++ /dev/null @@ -1,3 +0,0 @@ -Seismic Store Client C++ Library (SDAPI) -Version: 3.6.17 -Repo SHA-1 hash ID: ab985baaa2687f9611f5b2d2b3542f355af75b15