Commit 3f9f56ea authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Include azure-storage-blobs-cpp in the build. This may speed up access to...

Include azure-storage-blobs-cpp in the build. This may speed up access to Azure Storage on the Linux builds of OpenZGY and ZGY-Cloud.
parent c60a6b3d
......@@ -13,16 +13,19 @@ RUN apt-get -y install cmake curl g++ libcurl3-dev libssl-dev zip python python-
RUN pip install egg colorama flask requests
# Azure prerequisites. Have patience.
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
find / -type f -print > /root/before.lst; true
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/before.md5; \
find / -type f -print | sort -o /root/before.lst; true
RUN mkdir -p /opt && \
cd /opt && \
git clone https://github.com/microsoft/vcpkg.git && \
cd vcpkg && \
./bootstrap-vcpkg.sh -disableMetrics
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
find / -type f -print > /root/after.lst; true
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/after.md5; \
find / -type f -print | sort -o /root/after.lst; true
# A regular user, not root, should be used for development.
RUN groupadd -r admin; useradd -G admin --shell /bin/bash -p '' me
......
......@@ -13,16 +13,19 @@ RUN apt-get -y install cmake curl g++ libcurl3-dev zip python python-pip git
RUN pip install egg colorama flask requests
# Azure prerequisites. Have patience.
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
find / -type f -print > /root/before.lst; true
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/before.md5; \
find / -type f -print | sort -o /root/before.lst; true
RUN mkdir -p /opt && \
cd /opt && \
git clone https://github.com/microsoft/vcpkg.git && \
cd vcpkg && \
./bootstrap-vcpkg.sh -disableMetrics
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
find / -type f -print > /root/after.lst; true
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/after.md5; \
find / -type f -print | sort -o /root/after.lst; true
# A regular user, not root, should be used for development.
RUN useradd -G sudo --shell /bin/bash -p '' me
......
......@@ -2,23 +2,29 @@ FROM centos:centos8
RUN dnf install -y \
make gcc-c++ libcurl-devel openssl-devel \
which git emacs sudo cmake3 patch python2 unzip zip diffutils
which git sudo cmake patch python2 unzip zip diffutils
RUN echo dnf -y install \
tigervnc-server openssh-server dbus-x11 \
xterm firefox emacs ImageMagick icewm
RUN pip2 install --upgrade pip && pip2 install egg colorama flask requests
# Azure prerequisites. Have patience.
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
find / -type f -print > /root/before.lst; true
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/before.md5; \
find / -type f -print | sort -o /root/before.lst; true
RUN mkdir -p /opt && \
cd /opt && \
git clone https://github.com/microsoft/vcpkg.git && \
cd vcpkg && \
./bootstrap-vcpkg.sh -disableMetrics
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
find / -type f -print > /root/after.lst; true
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/after.md5; \
find / -type f -print | sort -o /root/after.lst; true
# --- TEMPORARY KLUDGES -- #
RUN dnf -y install libarchive # CMake bug, deps to newer version are missing.
RUN useradd -G wheel -p '' me
......
......@@ -14,16 +14,19 @@ RUN yum install -y gcc-c++ make which git emacs sudo cmake3 libcurl-devel openss
RUN pip install --upgrade pip && pip install egg colorama flask requests
# Azure prerequisites. Have patience.
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
find / -type f -print > /root/before.lst; true
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/before.md5; \
find / -type f -print | sort -o /root/before.lst; true
RUN mkdir -p /opt && \
cd /opt && \
git clone https://github.com/microsoft/vcpkg.git && \
cd vcpkg && \
./bootstrap-vcpkg.sh -disableMetrics
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
find / -type f -print > /root/after.lst; true
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/after.md5; \
find / -type f -print | sort -o /root/after.lst; true
# A regular user, not root, should be used for development.
RUN useradd -G wheel -p '' me
......
......@@ -12,16 +12,19 @@ RUN apt-get update && \
python3 python3-pip git vim sudo pkg-config
# Azure prerequisites. Have patience.
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
find / -type f -print > /root/before.lst; true
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/before.md5; \
find / -type f -print | sort -o /root/before.lst; true
RUN mkdir -p /opt && \
cd /opt && \
git clone https://github.com/microsoft/vcpkg.git && \
cd vcpkg && \
./bootstrap-vcpkg.sh -disableMetrics
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
find / -type f -print > /root/after.lst; true
RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/after.md5; \
find / -type f -print | sort -o /root/after.lst; true
# A regular user, not root, should be used for development.
RUN groupadd -r admin; useradd -G admin --shell /bin/bash -p '' me
......
......@@ -37,16 +37,19 @@ RUN set -e; \
make install
# Azure prerequisites. Have patience.
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
find / -type f -print > /root/before.lst; true
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/before.md5; \
find / -type f -print | sort -o /root/before.lst; true
RUN mkdir -p /opt && \
cd /opt && \
git clone https://github.com/microsoft/vcpkg.git && \
cd vcpkg && \
scl enable devtoolset-7 './bootstrap-vcpkg.sh -disableMetrics'
RUN cd /opt/vcpkg; scl enable devtoolset-7 './vcpkg install azure-storage-cpp'
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
find / -type f -print > /root/after.lst; true
RUN cd /opt/vcpkg; scl enable devtoolset-7 './vcpkg install azure-storage-blobs-cpp'
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/after.md5; \
find / -type f -print | sort -o /root/after.lst; true
# If you are going to run "sdutil auth login", this needs a full graphical
# environment which you can connect to with vnc. This will pull in ~500
......
......@@ -14,16 +14,18 @@ RUN pip install egg colorama flask requests
# Azure prerequisites. Have patience.
# Currently fails with compiler errors using gcc 6.3.0
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
find / -type f -print > /root/before.lst; true
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/before.md5; \
find / -type f -print | sort -o /root/before.lst; true
#RUN mkdir -p /opt && \
# cd /opt && \
# git clone https://github.com/microsoft/vcpkg.git && \
# cd vcpkg && \
# ./bootstrap-vcpkg.sh -disableMetrics
#RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
find / -type f -print > /root/after.lst; true
RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
sort -z | xargs -r0 md5sum > /root/after.md5; \
find / -type f -print | sort -o /root/after.lst; true
# A regular user, not root, should be used for development.
RUN useradd -G sudo --shell /bin/bash -p '' me
......
......@@ -6,8 +6,8 @@ WORKAREA = $(abspath $(CWD)/../os-seismic-store-cpp-lib)
#WORKAREA2 = $(abspath $(CWD)/../seismic-store-sdutil)
WORKAREA3 = $(abspath $(CWD)/../seismic-service-bin)
DISTROS = centos xenial bionic focal stretch buster omega fedora centos8
DISTROS += omega-static omega-dynamic
DISTROS = centos stretch xenial bionic focal buster omega fedora centos8
#DISTROS += omega-static omega-dynamic
BUILD_ALL = $(addprefix build-,$(DISTROS))
RUN_ALL = $(addprefix run-,$(DISTROS))
......@@ -52,27 +52,27 @@ build.sh: build-template.sh
.PHONY: xterm
xterm:
xterm -geometry 200x24+50+100 -e docker start -ai sd-centos &
xterm -sl 256 -T 'centos (gcc 4.9.2)' -geometry 200x24+50+100 -e docker start -ai sd-centos &
@sleep 2
xterm -geometry 200x24+100+200 -e docker start -ai sd-xenial &
xterm -sl 256 -T 'Debian stretch (gcc 6.3.0)' -geometry 200x24+100+200 -e docker start -ai sd-stretch &
@sleep 2
xterm -geometry 200x24+150+300 -e docker start -ai sd-bionic &
xterm -sl 256 -T 'Ubuntu (gcc xenial 5.4.0)' -geometry 200x24+150+300 -e docker start -ai sd-xenial &
@sleep 2
xterm -geometry 200x24+200+400 -e docker start -ai sd-focal &
xterm -sl 256 -T 'Ubuntu bionic (gcc 7.5.0)' -geometry 200x24+200+400 -e docker start -ai sd-bionic &
@sleep 2
xterm -geometry 200x24+250+500 -e docker start -ai sd-stretch &
xterm -sl 256 -T 'Ubuntu focal (gcc 9.3.0)' -geometry 200x24+250+500 -e docker start -ai sd-focal &
@sleep 2
xterm -geometry 200x24+300+600 -e docker start -ai sd-buster &
xterm -sl 256 -T 'Debian buster (gcc 8.3.0)' -geometry 200x24+300+600 -e docker start -ai sd-buster &
@sleep 2
xterm -geometry 200x24+350+700 -e docker start -ai sd-omega &
xterm -sl 256 -T 'omega (gcc 7.3.1)' -geometry 200x24+350+700 -e docker start -ai sd-omega &
@sleep 2
xterm -geometry 200x24+400+800 -e docker start -ai sd-fedora &
xterm -sl 256 -T 'fedora (gcc 9.3.1)' -geometry 200x24+400+800 -e docker start -ai sd-fedora &
@sleep 2
xterm -geometry 200x24+450+900 -e docker start -ai sd-centos8 &
@sleep 2
xterm -geometry 200x24+500+1000 -e docker start -ai sd-omega-static &
@sleep 2
xterm -geometry 200x24+550+1100 -e docker start -ai sd-omega-dynamic &
xterm -sl 256 -T 'centos8 (gcc 8.4.1)' -geometry 200x24+450+900 -e docker start -ai sd-centos8 &
# @sleep 2
# xterm -sl 256 -T 'omega-static (gcc 7.3.1)' -geometry 200x24+500+1000 -e docker start -ai sd-omega-static &
# @sleep 2
# xterm -sl 256 -T 'omega-dynamic (gcc 7.3.1)' -geometry 200x24+550+1100 -e docker start -ai sd-omega-dynamic &
#Can no longer be built because devtoolset-4 is EOL.
#This is a problem if building somewhere the image isn't cached.
......@@ -146,4 +146,4 @@ clean:
$(RM) build.sh
clobber: clean
docker rm $(patsubst %,sd-%,$(DISTROS)) || true
docker rm --force $(patsubst %,sd-%,$(DISTROS)) || true
......@@ -44,15 +44,15 @@ AZUREDEF=
if [ -r /opt/vcpkg/scripts/buildsystems/vcpkg.cmake ]
then
AZUREMSG="(with Google and Azure support)"
AZUREDEF="-DCMAKE_TOOLCHAIN_FILE=/opt/vcpkg/scripts/buildsystems/vcpkg.cmake -DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure"
AZUREDEF="-DCMAKE_TOOLCHAIN_FILE=/opt/vcpkg/scripts/buildsystems/vcpkg.cmake -DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure;azure-curl"
elif [ -r /usr/local/lib/libazurestorage.so ]
then
AZUREMSG="(with Google and (dynamic?) Azure support)"
AZUREDEF="-DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure"
AZUREDEF="-DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure;azure-curl"
elif [ -r /usr/local/lib/libazurestorage.a ]
then
AZUREMSG="(with Google and (static?) Azure support)"
AZUREDEF="-DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure"
AZUREDEF="-DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure;azure-curl"
fi
#Version on commandline?
......@@ -83,12 +83,42 @@ $CMAKE -DCMAKE_BUILD_TYPE=${1:-Release} ${VERSIONDEF} ${AZUREDEF} ${SRC}
#make || true
#test -x crc32c/lib64 || ln -s lib crc32c/lib64
# This kludge is needed as of June 2021 but has to be applied outside
# the build container because inside the source is readonly.
# After some experimenting I found that this approach is the minimal
# change that makes all 6 config build cleanly when using vcpkg.
# I have not tested the same configs without vcpkg.
#sed -ie '/find_package.*[Ll]ib[Xx]ml2.*/s/\(find.*\)/set( LibXml2_FOUND true ) # \1/' src/CMakeLists.txt
# This kludge is for integrating the slb-only extensions to sdapi.
# As above, it needs to be run outside the container. The assumption
# is that the private repository folder is the parent of the osdu one.
# This REALLY ought to be done as a plug-in.
#cp -t src/src/core ../src/src/core/*
#cp -t src/src/lib/auth ../src/src/lib/auth/*
make
rm -rf tmp/include tmp/lib/linux64 sdapi_linux64.tar.gz
rm -rf tmp
mkdir -p tmp/include tmp/lib/linux64
# Capture the hash of the OSDU part of SDAPI, and the version of some
# packages installed by vcpkg.
echo "LINUX: ${LINUXDISTRO:-unknown}"
echo "SDAPI: $(git -C /home/me/sdapi rev-parse HEAD)" > tmp/version.txt
if [ -x /opt/vcpkg/vcpkg ]
then
echo "VCPKG: $(git -C /opt/vcpkg rev-parse HEAD)" >> tmp/version.txt || true
echo "" >> tmp/version.txt
echo "AZURE:" >> tmp/version.txt
sudo date > /dev/null 2>&1 # Skip the first-time lecture.
sudo /opt/vcpkg/vcpkg list >> tmp/version.txt || true
fi
echo "" >> tmp/version.txt
ldd libsdapi.so >> tmp/version.txt || true
cp -a -t tmp/include ${SRC}/src/core/*.h ${SRC}/src/lib/accessors/*.h
cp -a -t tmp/lib/linux64 libsdapi*
tar zcf ../sdapi_linux64.tar.gz -C tmp .
tar zcf ../sdapi_linux64.tar.gz -C tmp version.txt include lib
# Embed Linux distro and compiler version in the file name.
# (Python version gets embedded also, but only for the wheels).
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment