Commit 3de20175 authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Major overhaul of build pipelines.

parent 7cdf94a7
Pipeline #56669 passed with stages
in 7 minutes and 42 seconds
.git
build/deploy
build/testdata
build/temp
# Test files created in the source tree. Should be fixed.
**/.dockerignore
**/.git*
/.git*
!/.git/HEAD
!/.git/refs/heads
!/seismic-store-cpp-lib/.git/HEAD
!/seismic-store-cpp-lib/.git/refs/heads
**/build
**/tmp-*.zgy
**/test-venv
**/__pycache__
**/*.egg-info
python/build
python/test-venv
wrapper/build
wrapper/test-venv
native/sdglue/build
native/sdglue/test-venv
**/.eggs
**/typescript*
**/*~
**/*#
misc/**
.eggs
azure/**
**/Dockerfile*
/sd-env/Makefile
......@@ -7,6 +7,7 @@ test-venv
*.egg-info
.eggs
.coverage
typescript
# These really shouldn't be in the source area.
/python/build
......
......@@ -137,8 +137,8 @@ docker-build:
rmdir --ignore-fail-on-non-empty seismic-service-bin private
docker rmi -f $(TAG):old $(TAG):test || true
$(RM) cid.txt
docker run --cidfile cid.txt $(TAG) make SDAPI_INTERNAL=$(SDAPI_INTERNAL) $(DOCKERTARGET)
docker cp $$(cat cid.txt):/home/build/oz/build/deploy - | gzip -9 > deploy.tgz
docker run --cidfile cid.txt $(TAG) make -j 8 SDAPI_INTERNAL=$(SDAPI_INTERNAL) $(DOCKERTARGET)
docker cp $$(cat cid.txt):/home/me/oz/build/deploy - | gzip -9 > deploy.tgz
docker rm $$(cat cid.txt)
$(RM) cid.txt
docker build -t $(TAG):test -f $(DOCKERFILE)-test .
......
......@@ -121,7 +121,41 @@ All writes and updates have the following *recommendations*:
## <span style="color:blue">Building and testing the core parts</span>
### Building Linux core
### Building on Linux using docker
This is the simplest approach.
```sh
git clone https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/open-zgy.git open-zgy
git clone --recursive https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib.git open-zgy/seismic-store-cpp-lib
make -C sd-env final-${distro}
make -C sd-env testonly-${distro}
```
This will build SDAPI, build OpenZGY, and run the OpenZGY unit tests.
Currently supported in the distro parameter is
- stretch
- xenial
- bionic
- focal
- buster
- omega
- fedora
- centos8
The resulting docker image can be used to experiment with both the
pure and the wrapped C++ Python versions. The image also contains a
tar file that can be copied out of the image for deploying OpenZGY.
You might want to use ```make -C sd-env PULL="" final-${distro}```
if you don't want to trigger a full rebuild, which can easily take
30 minutes, every time the base image is updated with some minor fix.
There can be such a thing as too much continous integration.
For an explanation of the more obscure targets you might want to build,
see the [detailed readme file](sd-env/README-details.md).
### Building Linux core locally
The code should build pretty much out of the box as long as Seismic
Store access is not enabled. On Linux there is a top level Makefile.
......@@ -130,7 +164,11 @@ In both cases both C++ and Python versions are built / packaged.
See [Output folders](#output-folders) for where to find the output.
### Building Windows core
If you want support for Seismic Store then using the docker build is
recommended. Otherwise you need to build SDAPI first and then copy the
binaries by hand into the OpenZGY source tree.
### Building Windows core locally
Prerequisites are Visual Studio 2019 with Platform Toolset v142.
If building with cloud support you might also need NuGet,
......@@ -195,10 +233,12 @@ The repository that is publicly accessible but may lag behind the internal versi
So when the instructions below mention downloading the cloud library you may download SDAPI from
[Azure](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/os-seismic-store-cpp-lib) or [OSDU](https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib.git).
### Building Linux cloud
### Building Linux cloud locally
This is somewhat tricky. Consider using the
[docker build](#Building-on Linux-using-docker) instead.
If you want to use Seismic Store to access the cloud then there are
some additional steps needed. You will need to download and build the
You will need to download and build the
Seismic Store SDK a.k.a. SDAPI. And then package the binaries so they
can be picked up by OpenZGY. The Linux build expects it to be
available as a tarball in the source tree. So if you want cloud
......
# name: 'OpenZGY Linux Ubuntu Bionic py36'
name: $(Build.BuildID)-bionic
trigger: none
schedules:
- cron: "0 3 * * 1"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'bionic'
tagtail: 'osdu'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/bionic-gcc75/libopenzgy.so.*.*
deploy/native/bionic-gcc75/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/bionic-gcc75/SdGlue-*-cp36-cp36m-linux_x86_64.whl
deploy/wrapper/bionic-gcc75/OpenZgyBindings-*-cp36-cp36m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc750/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux Ubuntu Bionic py36'
name: $(Build.BuildID)-bionic
trigger: none
schedules:
- cron: "0 4 * * 1"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'bionic'
tagtail: 'default'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/bionic-gcc75/libopenzgy.so.*.*
deploy/native/bionic-gcc75/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/bionic-gcc75/SdGlue-*-cp36-cp36m-linux_x86_64.whl
deploy/wrapper/bionic-gcc75/OpenZgyBindings-*-cp36-cp36m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc750/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux Debian Buster py37'
name: $(Build.BuildID)-buster
trigger: none
schedules:
- cron: "0 3 * * 2"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'buster'
tagtail: 'osdu'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/buster-gcc83/libopenzgy.so.*.*
deploy/native/buster-gcc83/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/buster-gcc83/SdGlue-*-cp37-cp37m-linux_x86_64.whl
deploy/wrapper/buster-gcc83/OpenZgyBindings-*-cp37-cp37m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc830/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux Debian Buster py37'
name: $(Build.BuildID)-buster
trigger: none
schedules:
- cron: "0 4 * * 2"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'buster'
tagtail: 'default'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/buster-gcc83/libopenzgy.so.*.*
deploy/native/buster-gcc83/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/buster-gcc83/SdGlue-*-cp37-cp37m-linux_x86_64.whl
deploy/wrapper/buster-gcc83/OpenZgyBindings-*-cp37-cp37m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc830/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux CentOS 7 py36'
name: $(Build.BuildID)-centos7
trigger: none
schedules:
- cron: "0 3 * * 3"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'centos7'
tagtail: 'osdu'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/centos7-gcc48/libopenzgy.so.*.*
deploy/native/centos7-gcc48/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/centos7-gcc48/SdGlue-*-cp36-cp36m-linux_x86_64.whl
deploy/wrapper/centos7-gcc48/OpenZgyBindings-*-cp36-cp36m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc485/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux CentOS 7 py36'
name: $(Build.BuildID)-centos7
trigger: none
schedules:
- cron: "0 4 * * 3"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'centos7'
tagtail: 'default'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/centos7-gcc48/libopenzgy.so.*.*
deploy/native/centos7-gcc48/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/centos7-gcc48/SdGlue-*-cp36-cp36m-linux_x86_64.whl
deploy/wrapper/centos7-gcc48/OpenZgyBindings-*-cp36-cp36m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc485/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux CentOS 8 py36'
name: $(Build.BuildID)-centos8-coverage
trigger: none
schedules:
- cron: "0 3 * * 0"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
variables:
- group: 'Salmon Variables'
jobs:
- job:
timeoutInMinutes: 60
workspace:
clean: all
steps:
- template: templates/combo-coverage-steps.yml
parameters:
linuxdistro: 'centos8'
tagtail: coverage
deployed: >
coverage.tgz
summary.txt
final.info
# name: 'OpenZGY Linux CentOS 8 py36'
name: $(Build.BuildID)-centos8
trigger: none
schedules:
- cron: "0 3 * * 4"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'centos8'
tagtail: 'osdu'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/centos8-gcc84/libopenzgy.so.*.*
deploy/native/centos8-gcc84/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/centos8-gcc84/SdGlue-*-cp36-cp36m-linux_x86_64.whl
deploy/wrapper/centos8-gcc84/OpenZgyBindings-*-cp36-cp36m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc841/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux CentOS 8 py36'
name: $(Build.BuildID)-centos8-vg
trigger: none
schedules:
- cron: "0 4 * * 0"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
variables:
- group: 'Salmon Variables'
jobs:
- job:
timeoutInMinutes: 120
workspace:
clean: all
steps:
- template: templates/combo-valgrind-steps.yml
parameters:
linuxdistro: 'centos8'
tagtail: valgrind
deployed: >
valgrind.txt
valgrind-1.txt
valgrind-2.txt
# name: 'OpenZGY Linux CentOS 8 py36'
name: $(Build.BuildID)-centos8
trigger:
- master
schedules:
- cron: "0 4 * * 4"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'centos8'
tagtail: 'default'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/centos8-gcc84/libopenzgy.so.*.*
deploy/native/centos8-gcc84/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/centos8-gcc84/SdGlue-*-cp36-cp36m-linux_x86_64.whl
deploy/wrapper/centos8-gcc84/OpenZgyBindings-*-cp36-cp36m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc841/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux Ubuntu Focal py38'
name: $(Build.BuildID)-focal
trigger: none
schedules:
- cron: "0 3 * * 5"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'focal'
tagtail: 'osdu'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/focal-gcc93/libopenzgy.so.*.*
deploy/native/focal-gcc93/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/focal-gcc93/SdGlue-*-cp38-cp38-linux_x86_64.whl
deploy/wrapper/focal-gcc93/OpenZgyBindings-*-cp38-cp38-linux_x86_64.whl
deploy/sdapi/Lin64_gcc930/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux Ubuntu Focal py38'
name: $(Build.BuildID)-focal
trigger: none
schedules:
- cron: "0 4 * * 5"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'focal'
tagtail: 'default'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/focal-gcc93/libopenzgy.so.*.*
deploy/native/focal-gcc93/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/focal-gcc93/SdGlue-*-cp38-cp38-linux_x86_64.whl
deploy/wrapper/focal-gcc93/OpenZgyBindings-*-cp38-cp38-linux_x86_64.whl
deploy/sdapi/Lin64_gcc930/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux omega py36'
name: $(Build.BuildID)-omega
trigger: none
schedules:
- cron: "0 3 * * 6"
displayName: Nightly build
branches:
include:
- master
pool:
name: Paal-Kvamme-Test
workspace:
clean: all
variables:
- group: 'Salmon Variables'
steps:
- template: templates/combo-build-steps.yml
parameters:
linuxdistro: 'omega'
tagtail: 'osdu'
deployed: >
deploy.tgz
deploy/pure/OpenZGY-*-py3-none-any.whl
deploy/pure/OpenZGY-*.tar.gz
deploy/native/omega-gcc73/libopenzgy.so.*.*
deploy/native/omega-gcc73/libsdapi.so.*.*.*
deploy/native/include/openzgy/api.h
deploy/native/include/openzgy/exception.h
deploy/native/include/openzgy/iocontext.h
deploy/sdglue/omega-gcc73/SdGlue-*-cp36-cp36m-linux_x86_64.whl
deploy/wrapper/omega-gcc73/OpenZgyBindings-*-cp36-cp36m-linux_x86_64.whl
deploy/sdapi/Lin64_gcc731/sdapi_linux64.tar.gz
# name: 'OpenZGY Linux omega py36'
name: $(Build.BuildID)-omega
trigger: none
schedules:
- cron: "0 4 * * 6"
displayName: Nightly build
branches:
include:
- master