Include azure-storage-blobs-cpp in the build. This may speed up access to...

Include azure-storage-blobs-cpp in the build. This may speed up access to Azure Storage on the Linux builds of OpenZGY and ZGY-Cloud.

sd-env/Dockerfile-bionic

@@ -13,16 +13,19 @@ RUN apt-get -y install cmake curl g++ libcurl3-dev libssl-dev zip python python-
 RUN pip install egg colorama flask requests
 
 # Azure prerequisites. Have patience.
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
-    find / -type f -print > /root/before.lst; true
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/before.md5; \
+    find / -type f -print | sort -o /root/before.lst; true
 RUN mkdir -p /opt && \
     cd /opt && \
     git clone https://github.com/microsoft/vcpkg.git && \
     cd vcpkg && \
     ./bootstrap-vcpkg.sh -disableMetrics
 RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
-    find / -type f -print > /root/after.lst; true
+RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/after.md5; \
+    find / -type f -print | sort -o /root/after.lst; true
 
 # A regular user, not root, should be used for development.
 RUN groupadd -r admin; useradd -G admin --shell /bin/bash -p '' me

sd-env/Dockerfile-buster

@@ -13,16 +13,19 @@ RUN apt-get -y install cmake curl g++ libcurl3-dev zip python python-pip git
 RUN pip install egg colorama flask requests
 
 # Azure prerequisites. Have patience.
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
-    find / -type f -print > /root/before.lst; true
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/before.md5; \
+    find / -type f -print | sort -o /root/before.lst; true
 RUN mkdir -p /opt && \
     cd /opt && \
     git clone https://github.com/microsoft/vcpkg.git && \
     cd vcpkg && \
     ./bootstrap-vcpkg.sh -disableMetrics
 RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
-    find / -type f -print > /root/after.lst; true
+RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/after.md5; \
+    find / -type f -print | sort -o /root/after.lst; true
 
 # A regular user, not root, should be used for development.
 RUN useradd -G sudo --shell /bin/bash -p '' me

sd-env/Dockerfile-centos8

@@ -2,23 +2,29 @@ FROM centos:centos8
 
 RUN dnf install -y \
         make gcc-c++ libcurl-devel openssl-devel \
-        which git emacs sudo cmake3 patch python2 unzip zip diffutils
+        which git sudo cmake patch python2 unzip zip diffutils
 RUN echo dnf -y install \
         tigervnc-server openssh-server dbus-x11 \
         xterm firefox emacs ImageMagick icewm
 RUN pip2 install --upgrade pip && pip2 install egg colorama flask requests
 
 # Azure prerequisites. Have patience.
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
-    find / -type f -print > /root/before.lst; true
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/before.md5; \
+    find / -type f -print | sort -o /root/before.lst; true
 RUN mkdir -p /opt && \
     cd /opt && \
     git clone https://github.com/microsoft/vcpkg.git && \
     cd vcpkg && \
     ./bootstrap-vcpkg.sh -disableMetrics
 RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
-    find / -type f -print > /root/after.lst; true
+RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/after.md5; \
+    find / -type f -print | sort -o /root/after.lst; true
+
+# --- TEMPORARY KLUDGES -- #
+RUN dnf -y install libarchive # CMake bug, deps to newer version are missing.
 
 RUN useradd -G wheel -p '' me
 

sd-env/Dockerfile-fedora

@@ -14,16 +14,19 @@ RUN yum install -y gcc-c++ make which git emacs sudo cmake3 libcurl-devel openss
 RUN pip install --upgrade pip && pip install egg colorama flask requests
 
 # Azure prerequisites. Have patience.
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
-    find / -type f -print > /root/before.lst; true
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/before.md5; \
+    find / -type f -print | sort -o /root/before.lst; true
 RUN mkdir -p /opt && \
     cd /opt && \
     git clone https://github.com/microsoft/vcpkg.git && \
     cd vcpkg && \
     ./bootstrap-vcpkg.sh -disableMetrics
 RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
-    find / -type f -print > /root/after.lst; true
+RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/after.md5; \
+    find / -type f -print | sort -o /root/after.lst; true
 
 # A regular user, not root, should be used for development.
 RUN useradd -G wheel -p '' me

sd-env/Dockerfile-focal

@@ -12,16 +12,19 @@ RUN apt-get update && \
 	python3 python3-pip git vim sudo pkg-config
 
 # Azure prerequisites. Have patience.
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
-    find / -type f -print > /root/before.lst; true
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/before.md5; \
+    find / -type f -print | sort -o /root/before.lst; true
 RUN mkdir -p /opt && \
     cd /opt && \
     git clone https://github.com/microsoft/vcpkg.git && \
     cd vcpkg && \
     ./bootstrap-vcpkg.sh -disableMetrics
 RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
-    find / -type f -print > /root/after.lst; true
+RUN cd /opt/vcpkg; ./vcpkg install azure-storage-blobs-cpp
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/after.md5; \
+    find / -type f -print | sort -o /root/after.lst; true
 
 # A regular user, not root, should be used for development.
 RUN groupadd -r admin; useradd -G admin --shell /bin/bash -p '' me

sd-env/Dockerfile-omega

@@ -37,16 +37,19 @@ RUN set -e; \
     make install
 
 # Azure prerequisites. Have patience.
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
-    find / -type f -print > /root/before.lst; true
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/before.md5; \
+    find / -type f -print | sort -o /root/before.lst; true
 RUN mkdir -p /opt && \
     cd /opt && \
     git clone https://github.com/microsoft/vcpkg.git && \
     cd vcpkg && \
     scl enable devtoolset-7 './bootstrap-vcpkg.sh -disableMetrics'
 RUN cd /opt/vcpkg; scl enable devtoolset-7 './vcpkg install azure-storage-cpp'
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
-    find / -type f -print > /root/after.lst; true
+RUN cd /opt/vcpkg; scl enable devtoolset-7 './vcpkg install azure-storage-blobs-cpp'
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/after.md5; \
+    find / -type f -print | sort -o /root/after.lst; true
 
 # If you are going to run "sdutil auth login", this needs a full graphical
 # environment which you can connect to with vnc. This will pull in ~500

sd-env/Dockerfile-stretch

@@ -14,16 +14,18 @@ RUN pip install egg colorama flask requests
 
 # Azure prerequisites. Have patience.
 # Currently fails with compiler errors using gcc 6.3.0
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/before.md5;\
-    find / -type f -print > /root/before.lst; true
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/before.md5; \
+    find / -type f -print | sort -o /root/before.lst; true
 #RUN mkdir -p /opt && \
 #    cd /opt && \
 #    git clone https://github.com/microsoft/vcpkg.git && \
 #    cd vcpkg && \
 #    ./bootstrap-vcpkg.sh -disableMetrics
 #RUN cd /opt/vcpkg; ./vcpkg install azure-storage-cpp
-RUN find / -name \*.so\* -type f -print0|sort -z|xargs -r0 md5sum > /root/after.md5;\
-    find / -type f -print > /root/after.lst; true
+RUN find / \( -name \*.so\* -o -name \*.a \) -type f -print0 | \
+    sort -z | xargs -r0 md5sum > /root/after.md5; \
+    find / -type f -print | sort -o /root/after.lst; true
 
 # A regular user, not root, should be used for development.
 RUN useradd -G sudo --shell /bin/bash -p '' me

sd-env/Makefile

@@ -6,8 +6,8 @@ WORKAREA   = $(abspath $(CWD)/../os-seismic-store-cpp-lib)
 #WORKAREA2 = $(abspath $(CWD)/../seismic-store-sdutil)
 WORKAREA3  = $(abspath $(CWD)/../seismic-service-bin)
 
-DISTROS = centos xenial bionic focal stretch buster omega fedora centos8
-DISTROS += omega-static omega-dynamic
+DISTROS = centos stretch xenial bionic focal buster omega fedora centos8
+#DISTROS += omega-static omega-dynamic
 BUILD_ALL = $(addprefix build-,$(DISTROS))
 RUN_ALL = $(addprefix run-,$(DISTROS))
 
@@ -52,27 +52,27 @@ build.sh: build-template.sh
 
 .PHONY: xterm
 xterm:
-	xterm -geometry 200x24+50+100 -e docker start -ai sd-centos &
+	xterm -sl 256  -T 'centos (gcc 4.9.2)' -geometry 200x24+50+100 -e docker start -ai sd-centos &
 	@sleep 2
-	xterm -geometry 200x24+100+200 -e docker start -ai sd-xenial &
+	xterm -sl 256  -T 'Debian stretch (gcc 6.3.0)' -geometry 200x24+100+200 -e docker start -ai sd-stretch &
 	@sleep 2
-	xterm -geometry 200x24+150+300 -e docker start -ai sd-bionic &
+	xterm -sl 256  -T 'Ubuntu (gcc xenial 5.4.0)' -geometry 200x24+150+300 -e docker start -ai sd-xenial &
 	@sleep 2
-	xterm -geometry 200x24+200+400 -e docker start -ai sd-focal &
+	xterm -sl 256  -T 'Ubuntu bionic (gcc 7.5.0)' -geometry 200x24+200+400 -e docker start -ai sd-bionic &
 	@sleep 2
-	xterm -geometry 200x24+250+500 -e docker start -ai sd-stretch &
+	xterm -sl 256  -T 'Ubuntu focal (gcc 9.3.0)' -geometry 200x24+250+500 -e docker start -ai sd-focal &
 	@sleep 2
-	xterm -geometry 200x24+300+600 -e docker start -ai sd-buster &
+	xterm -sl 256  -T 'Debian buster (gcc 8.3.0)' -geometry 200x24+300+600 -e docker start -ai sd-buster &
 	@sleep 2
-	xterm -geometry 200x24+350+700 -e docker start -ai sd-omega &
+	xterm -sl 256  -T 'omega (gcc 7.3.1)' -geometry 200x24+350+700 -e docker start -ai sd-omega &
 	@sleep 2
-	xterm -geometry 200x24+400+800 -e docker start -ai sd-fedora &
+	xterm -sl 256  -T 'fedora (gcc 9.3.1)' -geometry 200x24+400+800 -e docker start -ai sd-fedora &
 	@sleep 2
-	xterm -geometry 200x24+450+900 -e docker start -ai sd-centos8 &
-	@sleep 2
-	xterm -geometry 200x24+500+1000 -e docker start -ai sd-omega-static &
-	@sleep 2
-	xterm -geometry 200x24+550+1100 -e docker start -ai sd-omega-dynamic &
+	xterm -sl 256  -T 'centos8 (gcc 8.4.1)' -geometry 200x24+450+900 -e docker start -ai sd-centos8 &
+#	@sleep 2
+#	xterm -sl 256  -T 'omega-static (gcc 7.3.1)' -geometry 200x24+500+1000 -e docker start -ai sd-omega-static &
+#	@sleep 2
+#	xterm -sl 256  -T 'omega-dynamic (gcc 7.3.1)' -geometry 200x24+550+1100 -e docker start -ai sd-omega-dynamic &
 
 #Can no longer be built because devtoolset-4 is EOL.
 #This is a problem if building somewhere the image isn't cached.
@@ -146,4 +146,4 @@ clean:
 	$(RM) build.sh
 
 clobber: clean
-	docker rm $(patsubst %,sd-%,$(DISTROS)) || true
+	docker rm --force $(patsubst %,sd-%,$(DISTROS)) || true

sd-env/build-template.sh

@@ -44,15 +44,15 @@ AZUREDEF=
 if [ -r /opt/vcpkg/scripts/buildsystems/vcpkg.cmake ]
 then
     AZUREMSG="(with Google and Azure support)"
-    AZUREDEF="-DCMAKE_TOOLCHAIN_FILE=/opt/vcpkg/scripts/buildsystems/vcpkg.cmake -DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure"
+    AZUREDEF="-DCMAKE_TOOLCHAIN_FILE=/opt/vcpkg/scripts/buildsystems/vcpkg.cmake -DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure;azure-curl"
 elif [ -r /usr/local/lib/libazurestorage.so ]
 then
     AZUREMSG="(with Google and (dynamic?) Azure support)"
-    AZUREDEF="-DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure"
+    AZUREDEF="-DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure;azure-curl"
 elif [ -r /usr/local/lib/libazurestorage.a ]
 then
     AZUREMSG="(with Google and (static?) Azure support)"
-    AZUREDEF="-DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure"
+    AZUREDEF="-DOPTIONAL_STORAGE_PROVIDERS_ENABLED:STRING=azure;azure-curl"
 fi
 
 #Version on commandline?
@@ -83,12 +83,42 @@ $CMAKE -DCMAKE_BUILD_TYPE=${1:-Release} ${VERSIONDEF} ${AZUREDEF} ${SRC}
 #make || true
 #test -x crc32c/lib64 || ln -s lib crc32c/lib64
 
+# This kludge is needed as of June 2021 but has to be applied outside
+# the build container because inside the source is readonly.
+# After some experimenting I found that this approach is the minimal
+# change that makes all 6 config build cleanly when using vcpkg.
+# I have not tested the same configs without vcpkg.
+#sed -ie '/find_package.*[Ll]ib[Xx]ml2.*/s/\(find.*\)/set( LibXml2_FOUND true ) # \1/' src/CMakeLists.txt
+
+# This kludge is for integrating the slb-only extensions to sdapi.
+# As above, it needs to be run outside the container. The assumption
+# is that the private repository folder is the parent of the osdu one.
+# This REALLY ought to be done as a plug-in.
+#cp -t src/src/core ../src/src/core/*
+#cp -t src/src/lib/auth ../src/src/lib/auth/*
+
 make
-rm -rf tmp/include tmp/lib/linux64 sdapi_linux64.tar.gz
+rm -rf tmp
 mkdir -p tmp/include tmp/lib/linux64
+
+# Capture the hash of the OSDU part of SDAPI, and the version of some
+# packages installed by vcpkg.
+echo "LINUX: ${LINUXDISTRO:-unknown}"
+echo "SDAPI: $(git -C /home/me/sdapi rev-parse HEAD)" > tmp/version.txt
+if [ -x /opt/vcpkg/vcpkg ]
+then
+    echo "VCPKG: $(git -C /opt/vcpkg rev-parse HEAD)" >> tmp/version.txt || true
+    echo "" >> tmp/version.txt
+    echo "AZURE:" >> tmp/version.txt
+    sudo date > /dev/null 2>&1 # Skip the first-time lecture.
+    sudo /opt/vcpkg/vcpkg list >> tmp/version.txt || true
+fi
+echo "" >> tmp/version.txt
+ldd libsdapi.so >> tmp/version.txt || true
+
 cp -a -t tmp/include ${SRC}/src/core/*.h ${SRC}/src/lib/accessors/*.h
 cp -a -t tmp/lib/linux64 libsdapi*
-tar zcf ../sdapi_linux64.tar.gz -C tmp .
+tar zcf ../sdapi_linux64.tar.gz -C tmp version.txt include lib
 
 # Embed Linux distro and compiler version in the file name.
 # (Python version gets embedded also, but only for the wheels).