Commit 0cefe14b authored by Zakhar Shkonda's avatar Zakhar Shkonda Committed by Ivan Medeiros Monteiro
Browse files

Build with SDAPI

parent 64084e3e
include:
- template: Auto-DevOps.gitlab-ci.yml
- local: gitlab/templates.yml
stages:
- prepare
- build
......@@ -36,6 +37,7 @@ variables:
CODE_QUALITY_DISABLED: ""
BUILD_DISABLED: "1"
build:
rules: #introducing BUILD_DISABLED: overriding original rule to be able to ignore the original job
- if: '$BUILD_DISABLED'
......@@ -46,25 +48,6 @@ build_artifact:
rules:
- if: '$BUILD_DISABLED'
when: never
centos8-container:
stage: prepare
image: docker:19.03
tags:
- docker-runner
script:
- mkdir -p seismic-service-bin private
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- grep -v '^COPY' scripts/Dockerfile-centos8 | docker build --pull -t $SHA_IMAGE_TAG-centos8
-f - .
- docker push $SHA_IMAGE_TAG-centos8
only:
- master
- merge_requests
- kvamme62/citest
- kvamme62/cicd
- skorozsi/sast
doc-container:
stage: prepare
image: docker:19.03
......@@ -82,27 +65,184 @@ doc-container:
- kvamme62/citest
- kvamme62/cicd
- skorozsi/sast
# A ready-made image prepared by the seismic store team, is used for compilation.
# The image contains all the necessary components for SDAPI compilation.
# It makes building OpenZGY with SDAPI much faster and easier.
# Thus, there is a number of features:
# * SDAPI dependencies are frozen in the base image.
# * SDAPI itself gets built from the latest sources.
# * OpenZGY-only dependencies get the latest version available.
# * OpenZGY gets built from the latest sources.
# * Cloning the SDAPI source is done inside the container, in .compile-job.
# * Cloning the OpenZGY source is done by gitlab and mounted inside the container.
#
# TODO
# Prepare ready-made images in our registry for all supported platforms.
# Compile the documentation based on the ready-made image.
# Available cloud providers:
# --azure-provider
# --azure-curl-provider
# --aws-provider
# --ibm-provider
# --gcp-provider --does not added yet
# Ubuntu focal compile
focal-compile-azure-curl:
extends: .focal_compile_job
variables:
LINUXDISTRO: focal
CLOUD_PROVIDERS: "--azure-curl-provider"
focal-compile-aws:
extends: .focal_compile_job
variables:
LINUXDISTRO: focal
CLOUD_PROVIDERS: "--aws-provider"
focal-compile-ibm:
extends: .focal_compile_job
variables:
LINUXDISTRO: focal
CLOUD_PROVIDERS: "--ibm-provider"
focal-compile-polycloud:
extends: .focal_compile_job
variables:
LINUXDISTRO: focal
CLOUD_PROVIDERS: "--azure-curl-provider --aws-provider --ibm-provider"
# Ubuntu focal tests
focal-test-azure-curl:
extends: .test-focal
dependencies:
- focal-compile-azure-curl
focal-test-aws:
extends: .test-focal
dependencies:
- focal-compile-aws
focal-test-ibm:
extends: .test-focal
dependencies:
- focal-compile-ibm
focal-test-polycloud:
extends: .test-focal
dependencies:
- focal-compile-polycloud
#Ubuntu focal deploy
focal-deploy-azure-curl:
extends: .deploy_job
variables:
LINUXDISTRO: focal
dependencies:
- focal-compile-azure-curl
focal-deploy-aws:
extends: .deploy_job
variables:
LINUXDISTRO: focal
dependencies:
- focal-compile-aws
focal-deploy-ibm:
extends: .deploy_job
variables:
LINUXDISTRO: focal
dependencies:
- focal-compile-ibm
focal-deploy-polycloud:
extends: .deploy_job
variables:
LINUXDISTRO: focal
dependencies:
- focal-compile-polycloud
#Centos7 compile
#centos7-compile-azure-curl:
# extends: .centos7_compile_job
# variables:
# LINUXDISTRO: centos7
# CLOUD_PROVIDERS: "--azure-curl-provider"
#centos7-compile-azure:
# extends: .centos7_compile_job
# variables:
# LINUXDISTRO: centos7
# CLOUD_PROVIDERS: "--azure-provider"
#centos7-compile-aws:
# extends: .centos7_compile_job
# variables:
# LINUXDISTRO: centos7
# CLOUD_PROVIDERS: "--aws-provider"
#centos7-compile-ibm:
# extends: .centos7_compile_job
# variables:
# LINUXDISTRO: centos7
# CLOUD_PROVIDERS: "--ibm-provider"
#centos7-compile-polycloud:
# extends: .centos7_compile_job
# variables:
# LINUXDISTRO: centos7
# CLOUD_PROVIDERS: "--azure-curl-provider --azure-provider --aws-provider --ibm-provider"
#Centos7 deploy
#centos7-deploy-azure-curl:
# extends: .deploy_job
# variables:
# LINUXDISTRO: centos7
# dependencies:
# - centos7-compile-azure-curl
# - test-wrapper-focal
#centos7-deploy-azure:
# extends: .deploy_job
# variables:
# LINUXDISTRO: centos7
# dependencies:
# - centos7-compile-azure
# - test-wrapper-focal
#centos7-deploy-aws:
# extends: .deploy_job
# variables:
# LINUXDISTRO: centos7
# dependencies:
# - centos7-compile-aws
# - test-wrapper-focal
#centos7-deploy-ibm:
# extends: .deploy_job
# variables:
# LINUXDISTRO: centos7
# dependencies:
# - centos7-compile-ibm
# - test-wrapper-focal
#centos7-deploy-polycloud:
# extends: .deploy_job
# variables:
# LINUXDISTRO: centos7
# dependencies:
# - centos7-compile-polycloud
# - test-wrapper-focal
centos8-compile:
stage: build
image: "$SHA_IMAGE_TAG-centos8"
tags:
- docker-runner
artifacts:
name: OpenZGY-centos8
expire_in: 1 hr
paths:
- build/deploy
exclude:
- build/**.timestamp
script:
- make build testscripts
only:
- master
- merge_requests
- kvamme62/citest
- kvamme62/cicd
- skorozsi/sast
doc-compile:
stage: build
......@@ -122,74 +262,8 @@ doc-compile:
- kvamme62/citest
- kvamme62/cicd
- skorozsi/sast
test-native:
stage: test
image: "$SHA_IMAGE_TAG-centos8"
script:
- mkdir /tmp/run
- tar cf build.tar build
- tar xvf build.tar -C /tmp/run
- tar xvf build/deploy/testscripts.tar -C /tmp/run
- cd /tmp/run
- "env OPENZGY_TESTDATA= /bin/bash native/src/runtests.sh"
only:
- master
- merge_requests
- kvamme62/citest
- kvamme62/cicd
- skorozsi/sast
".test-sdglue":
stage: test
image: "$SHA_IMAGE_TAG-centos8"
script:
- mkdir /tmp/run
- tar cf build.tar build
- tar xvf build.tar -C /tmp/run
- tar xvf build/deploy/testscripts.tar -C /tmp/run
- cd /tmp/run
- "/bin/bash native/sdglue/runtests.sh"
only:
- master
- merge_requests
- kvamme62/citest
- kvamme62/cicd
- skorozsi/sast
test-wrapper:
stage: test
image: "$SHA_IMAGE_TAG-centos8"
script:
- mkdir /tmp/run
- tar cf build.tar build
- tar xvf build.tar -C /tmp/run
- tar xvf build/deploy/testscripts.tar -C /tmp/run
- cd /tmp/run
- "/bin/bash wrapper/runtests.sh"
only:
- master
- merge_requests
- kvamme62/citest
- kvamme62/cicd
- skorozsi/sast
test-pure:
stage: test
image: "$SHA_IMAGE_TAG-centos8"
script:
- mkdir /tmp/run
- tar cf build.tar build
- tar xvf build.tar -C /tmp/run
- tar xvf build/deploy/testscripts.tar -C /tmp/run
- cd /tmp/run
- "/bin/bash python/runtests.sh"
only:
- master
- merge_requests
- kvamme62/citest
- kvamme62/cicd
- skorozsi/sast
windows-build-x64-release:
stage: build
......@@ -283,3 +357,4 @@ code_quality:
- if: '$CODE_QUALITY_DISABLED'
when: never
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
# Compile job templates
.compile_job:
stage: build
image: # OVERRIDE-IN-CHILD
tags:
- osdu-medium
artifacts:
name: OpenZGY-$LINUXDISTRO
expire_in: 1 hr
paths:
- build
exclude:
- build/**.timestamp
before_script:
- |
set -e -x
git clone --recursive https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib.git osdu-sdapi-cpp
echo "All files in $(pwd)"; ls -FC .. . * || true
# Patch version for OpenZGY
export AZURE_BUILDID="${CI_PIPELINE_IID}"
# Build SDAPI
cd osdu-sdapi-cpp
chmod +x devops/scripts/gen_version.sh
SDAPI_VERSION=$(devops/scripts/gen_version.sh)
echo ${SDAPI_VERSION}
devops/scripts/build-linux64.sh --build-mode=Release \
--lib-version=${SDAPI_VERSION} \
--build-utest \
${CLOUD_PROVIDERS}
#Pack SDAPI without root directory (required for OpenZGY)
tar --directory=./dist/Release/sdapi-${SDAPI_VERSION}/ -cf - . | gzip > ./dist/sdapi_linux64_osdu.tar.gz
#Copy SDAPI to ./seismic-service-bin/Lin64_gcc$GCCVERSION
cd ..
GCCVERSION=$(gcc -dumpfullversion | sed -e 's/\.//g')
mkdir -p seismic-service-bin/Lin64_gcc$GCCVERSION
cp ./osdu-sdapi-cpp/dist/sdapi_linux64_osdu.tar.gz ./seismic-service-bin/Lin64_gcc$GCCVERSION/sdapi_linux64_osdu.tar.gz
script: #Implement
after_script:
- |
PROVIDERS=$(echo ${CLOUD_PROVIDERS} | sed 's/--//g' | sed 's/-provider//g' | sed 's/ /./g')
PKGINFOPATH=$(find ./build/deploy/native -name PKG-INFO)
echo $PKGINFOPATH
OPENZGYVERSION=$(grep '^Version:' $PKGINFOPATH | cut -d ' ' -f 2)
echo $OPENZGYVERSION
tar --directory=./build/deploy/ -cf - . | gzip > ./build/OpenZGY.${LINUXDISTRO}.sdms-bundle-${PROVIDERS}.${OPENZGYVERSION}.tar.gz
.focal_compile_job:
extends: .compile_job
image: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/sdapi-osdu-ubuntu-static-deps:latest
artifacts:
name: OpenZGY-focal
script: |
#Install python for OpenZGY
DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \
python3-dev python3-pip python3-setuptools python3-tk
pip3 install wheel virtualenv numpy cython
#Build OpenZGY
make build testscripts
.centos7_compile_job:
extends: .compile_job
image: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/sdapi-osdu-centos7-static-deps:latest
artifacts:
name: OpenZGY-centos7
script: |
#Install componets for OpenZGY build
yum -y install \
bzip2 \
pip3 install wheel virtualenv numpy cython \
python2 python3 python3-devel python3-tkinter
python3.6 -m ensurepip; \
python3.6 -m pip install --upgrade pip; \
python3.6 -m pip install wheel virtualenv numpy cython
/bin/rm -f /usr/bin/pip3; \
test -r /usr/bin/python3 || ln -s python3.6 /usr/bin/python3; \
test -r /usr/bin/pip3 || ln -s /usr/local/bin/pip3 /usr/bin/pip3
#Build OpenZGY
make build testscripts
# Deploy job templates
.deploy_job:
stage: deploy
image: docker:19.03
script: |
set -e -x
apk add curl
ls -laR
echo "${CI_API_V4_URL}"
echo "${CI_PROJECT_ID}"
PKGINFOPATH=$(find ./build/ -name PKG-INFO)
OPENZGYVERSION=$(grep '^Version:' $PKGINFOPATH | cut -d ' ' -f 2)
OPENZGYPATH=$(find "$(pwd -P)" -name OpenZGY.${LINUXDISTRO}*.tar.gz)
OPENZGYFILE=$(basename ${OPENZGYPATH})
echo ${OPENZGYVERSION}
echo ${OPENZGYPATH}
echo ${OPENZGYFILE}
curl -v --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${OPENZGYPATH} "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/OpenZGY/$OPENZGYVERSION/${OPENZGYFILE}"
# Test templates. Can be used if we put tests in a separate job.
.test:
stage: test
image: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/sdapi-osdu-ubuntu-static-deps:latest
before_script:
- |
set -e -x
DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \
strace valgrind \
uuid-dev \
python3-dev python3-pip python3-setuptools python3-tk
pip3 install wheel virtualenv numpy cython
ls -laR
python3 -m pip install build/deploy/pure/OpenZGY-*.whl
python3 -m pip install build/deploy/sdglue/*/SdGlue-*.whl || true
python3 -m pip install build/deploy/wrapper/*/OpenZgyBindings-*.whl
du -skc build/deploy/native/* && ls -l build/deploy
export CLIENT_ID=""
export CLIENT_SECRET=""
export OPENZGY_SDURL=""
export OPENZGY_SDAPIKEY=""
export OPENZGY_TOKEN=""
export OPENZGY_TESTDATA="../testdata"
export OPENZGY_SDTESTDATA="sd://sntc/testdata"
export OPENZGY_SDTESTSINK="sd://sntc/testsink/d"
.test-focal:
extends: .test
image: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/sdapi-osdu-ubuntu-static-deps:latest
script:
- |
echo "RUN native/src/runtests.sh"
native/src/runtests.sh
echo "RUN wrapper/runtests.sh"
wrapper/runtests.sh
echo "RUN python/runtests.sh"
python/runtests.sh
dependencies:
- focal-compile-azure-curl
......@@ -69,7 +69,11 @@ ifneq ($(strip $(HAVE_SD)),)
$(SD_SENTINEL) $(SD_LIBRARY): $(SD_ZIPFILE)
$(RM) -rf $(SD_BUILDDIR)
/bin/mkdir -p $(SD_BUILDDIR)
tar zxf $(SD_ZIPFILE) -C $(SD_BUILDDIR) --wildcards '*/libsdapi.so*' 'include' 'version.txt'
tar zxf $(SD_ZIPFILE) -C $(SD_BUILDDIR) --wildcards '*/libsdapi.so*' '*include' '*version.txt'
rm -f $(SD_BUILDDIR)/lib/linux64/libsdapi.so
ls -laR $(SD_BUILDDIR)
find $(SD_BUILDDIR)/lib/linux64/ -name libsdapi.so.* -type f -exec cp {} $(SD_BUILDDIR)/lib/linux64/libsdapi.so \;
# find $(SD_BUILDDIR)/linux64/libsdapi.so.* -type f -exec cp {} $(SD_BUILDDIR)/lib/linux64/libsdapi.so \;
touch $(SD_SENTINEL)
else
$(SD_SENTINEL):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment