Skip to content
Snippets Groups Projects

GONRG-5441: improve collect version airflow composer script

+ 38
28
@@ -7,8 +7,12 @@ packages[osdu-airflow]="https://community.opengroup.org/api/v4/projects/668/pack
packages[osdu-api]="https://community.opengroup.org/api/v4/projects/148/packages"
packages[osdu-ingestion]="https://community.opengroup.org/api/v4/projects/823/packages"
# Ex. 0.14
PATTERN_VERSION=$(echo "${TAG:1}" | cut -d "." -f 1,2)
# Check that TAG variable exists
if [[ -z ${TAG} ]]; then echo "TAG variable is not set on the pipeline"; exit 1; fi
# Ex. 0.14 consists of PATTERN_PART1=0 and PATTERN_PART2=14
PATTERN_PART1="$(echo "${TAG:1}" | cut -d "." -f 1)"
PATTERN_PART2="$(echo "${TAG:1}" | cut -d "." -f 2)"
release_versions() {
# Ex. osdu-airflow
@@ -16,41 +20,47 @@ release_versions() {
# Ex. https://community.opengroup.org/api/v4/projects/668/packages
PACKAGE_URL=$2
for i in {1..100}
# Initial values for variables
PACKAGES_TOTAL=1
PAGE_NUMBER=1
# Collect versions
while [[ $PACKAGES_TOTAL -ne 0 ]]
do
# Number of pypi packages per page. More info https://docs.gitlab.com/ee/api/index.html#pagination
PACKAGES_TOTAL=$(curl -s -q -X GET "${PACKAGE_URL}?package_type=pypi&per_page=100&page=${i}" | jq length)
printf "\n"
echo "--------------------------------------------------------------------------------------------"
echo "Checking ${PAGE_NUMBER} API page..."
echo "--------------------------------------------------------------------------------------------"
set +e
curl -s -q -X GET "${PACKAGE_URL}?package_type=pypi&per_page=100&page=${i}" | jq -r ".[].version" | grep ^"${PATTERN_VERSION}" >> /dev/null 2>&1
PAGE_OUTPUT=$(curl -s -q -X GET "${PACKAGE_URL}?package_type=pypi&per_page=100&page=${PAGE_NUMBER}")
echo "${PAGE_OUTPUT}" | jq -r ".[].version" | grep -P "^${PATTERN_PART1}\.${PATTERN_PART2}\.[0-9]{1,2}\$" >> /dev/null 2>&1
EXIT_CODE=$?
set -e
if [ ${EXIT_CODE} -ne 0 ] || [ -z "${TAG}" ]
if [ ${EXIT_CODE} -eq 0 ]
then
printf "\n"
echo "--------------------------------------------------------------------------------------------"
echo "Required pypi package version or tag does not exist!"
echo "--------------------------------------------------------------------------------------------"
exit 1
else
# Output all pypi package versions to the file
curl -s -q -X GET "${PACKAGE_URL}?package_type=pypi&per_page=100&page=${i}" | jq -r ".[].version" | grep ^"${PATTERN_VERSION}" >> /tmp/"${PACKAGE_NAME}".versions
fi
# Check if we collected versions for all pypi packages
if [[ ${PACKAGES_TOTAL} -ne 100 ]]
then
printf "\n"
echo "--------------------------------------------------------------------------------------------"
echo "All required versions for ${PACKAGE_NAME} pypi packages have been collected!"
echo "--------------------------------------------------------------------------------------------"
echo "Continue..."
echo "--------------------------------------------------------------------------------------------"
break
echo "${PAGE_OUTPUT}" | jq -r ".[].version" | grep -P "^${PATTERN_PART1}\.${PATTERN_PART2}\.[0-9]{1,2}\$" >> /tmp/"${PACKAGE_NAME}".versions
fi
# Number of pypi packages per page. More info https://docs.gitlab.com/ee/api/index.html#pagination
PACKAGES_TOTAL=$(echo "${PAGE_OUTPUT}" | jq length)
((PAGE_NUMBER++))
done
# Check if we have collected any versions
if [[ -e /tmp/"${PACKAGE_NAME}".versions ]]
then
printf "\n"
echo "--------------------------------------------------------------------------------------------"
echo "Checking next API page..."
echo "All required versions for ${PACKAGE_NAME} pypi packages have been collected!"
echo "--------------------------------------------------------------------------------------------"
done
echo "Continue..."
echo "--------------------------------------------------------------------------------------------"
else
printf "\n"
echo "--------------------------------------------------------------------------------------------"
echo "Required pypi package version or tag does not exist!"
echo "--------------------------------------------------------------------------------------------"
exit 1
fi
MAX="-1"
@@ -65,7 +75,7 @@ release_versions() {
done < /tmp/"${PACKAGE_NAME}".versions
# Max pypi package version. Ex. 0.14.1
VERSION_MAX="${PATTERN_VERSION}.${MAX}"
VERSION_MAX="${PATTERN_PART1}.${PATTERN_PART2}.${MAX}"
printf "\n"
echo "--------------------------------------------------------------------------------------------"
Loading