Commit 069790ae authored by David Diederich's avatar David Diederich
Browse files

Merge branch 'python-support' into 'master'

Python support

See merge request !104
parents b5db3129 ce975c7d
Pipeline #16811 passed with stages
in 23 seconds
# PIP_REQUIREMENTS:
# If there is more than one requirements file, or it is named something other than requirements.txt,
# set the variable PIP_REQUIREMENTS to be a space delimited list of the requirements file. There is no
# way to escape an embedded space within the filename -- you need to rename the file to have no whitespace.
# PIP_CMD, VENV_CMD:
# Normally, you should not need to configure these. If you override the image (presumably to use a different version of python),
# then you may need to alter these commands as well. In particular, old versions of python did not include the 'venv' module, so
# you may wish to set VENV_CMD to 'virtualenv'.
.python:
image: python:3.7-buster
tags: ['osdu-medium']
cache:
key: $CI_JOB_IMAGE
paths:
- venv
before_script:
- test -z "$PIP_REQUIREMENTS" && export PIP_REQUIREMENTS="requirements.txt"; echo "$PIP_REQUIREMENTS"
- test -z "$PIP_CMD" && export PIP_CMD="pip"; echo "$PIP_CMD"
- test -z "$VENV_CMD" && export VENV_CMD="python -m venv"; echo "$VENV_CMD"
# Establish a virtual environment for the packages, which is cached between runs
- $VENV_CMD venv
- source venv/bin/activate
# In case the requirements files don't mention these, we need them for the compile-and-unit-test step
- $PIP_CMD install --upgrade pip
- $PIP_CMD install wheel pytest pytest-cov
# Install all the requirements
- |
for req in $PIP_REQUIREMENTS; do
echo "--------------------------------------------------------------------------------"
echo -e "Installing $req\n"
$PIP_CMD install -r $req
done
# If the unit tests are stored in a directory other than ./tests/unit, define it using PYTEST_DIR
# Also, you can configure the test coverage module using PYTEST_COV, it defaults to 'app'
compile-and-unit-test:
extends:
- .python
- .skipForTriggeringMergeRequests
stage: build
script:
- test -z "$PYTEST_DIR" && export PYTEST_DIR="tests/unit"; echo "$PYTEST_DIR"
- test -z "$PYTEST_COV" && export PYTEST_COV="app"; echo "$PYTEST_COV"
# This is used by the fossa-analyze job later (and is theoretically available for any other job that might want it)
- $PIP_CMD freeze > all-requirements.txt
# This runs the unit tests
- pytest --junit-xml=unit_tests_report.xml --cov="$PYTEST_COV" --cov-report=xml:coverage.xml --cov-report=term "$PYTEST_DIR"
coverage: '/^(?i)(TOTAL).*\s+(\d+\%)$/'
artifacts:
when: always
paths:
- all-requirements.txt
reports:
junit: unit_tests_report.xml
cobertura: coverage.xml
# This checks to see if the NOTICE file generated based on the current code
# matches the one that is committed in the repository. Because FOSSA can
# occassionally order the lines different between executions, we use a python
# program to parse the file into sections and sort each section. Then we compare
# and expect the same number of sections, names of sections, and contents
# (without consider order).
fossa-check-notice:
image: $CI_REGISTRY/divido/fossa-cli/fossa-cli:v4.1
stage: attribution
needs: ['fossa-analyze']
tags: ['osdu-small']
only:
variables:
- $FOSSA_API_KEY
artifacts:
when: on_failure
paths:
- public
script:
# fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results
# When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined
# then things will fail and we'll have to make this smarter
- test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG"
- mkdir -p public
- fossa report licenses --project "${CI_PROJECT_TITLE}" --branch "${CI_COMMIT_BRANCH}" > public/NOTICE
- mkdir committed-NOTICE generated-NOTICE
- |
python <<EOF
import re
import os
regexSeparator = re.compile(r"^=+$")
regexSubtitle = re.compile(r"^The following software have components provided under the terms of this license:$")
# ----------------------------------------
def writeComponents(sectionFile, components):
for component in sorted(components):
sectionFile.write(component)
sectionFile.close()
# ----------------------------------------
def matchSectionHeader(lines, idx):
if (idx <= 3): return None
sep1 = regexSeparator.match(lines[idx - 4].rstrip())
sep2 = regexSeparator.match(lines[idx - 2].rstrip())
sub = regexSubtitle.match(lines[idx - 1].rstrip())
blank = (lines[idx].rstrip() == "")
if sep1 and sep2 and sub and blank:
return lines[idx - 3].rstrip()
return None
# ----------------------------------------
def split(licenseFile, outputDir):
lines = []
with open(licenseFile, 'r') as f:
lines = f.readlines()
# --------------------
sectionFile = None
components = []
for idx in range(0, len(lines)):
line = lines[idx]
if (line.startswith("-")):
components.append(line)
newHeader = matchSectionHeader(lines, idx)
if newHeader:
if sectionFile is not None: writeComponents(sectionFile, components)
sectionFile = open(os.path.join(outputDir, newHeader), 'w')
components = []
# Write the header to the new file
for writeIdx in range(idx - 4, idx + 1):
sectionFile.write(lines[writeIdx])
# ----------------------------------------
split("NOTICE", "committed-NOTICE")
split("public/NOTICE", "generated-NOTICE")
EOF
- |
if ! diff -r committed-NOTICE generated-NOTICE; then
echo "================================================================================"
echo "There are differences between the NOTICE file in the repository and the one generated just now by this command"
echo "The most likely cause of this is changes to the project dependencies -- the NOTICE file is providing attribution"
echo "to the projects being used, so it must be updated whenever that changes. This includes projects used indirectly as"
echo "dependencies of dependencies."
echo "----------------------------------------"
echo "You can download the NOTICE file from this job's artifacts and directly commit it to the repository to resolve"
echo "this. Before doing so, review the differences to make sure that they make sense given the changes that you have"
echo "made. If they do not, reach out to a maintainer to help diagnose the issue."
exit 1
fi
# --------------------------------------------------------------------------------
fossa-report:
image: $CI_REGISTRY/divido/fossa-cli/fossa-cli:v4.1
stage: attribution
needs: ['fossa-analyze']
tags: ['osdu-small']
only:
variables:
# To get the FOSSA_ACCESS, log into the project's main page and generate a "Live Project Report" URL from the Summary tab
# The access token will be shown in the URL (the last path component)
- $FOSSA_API_KEY && $FOSSA_ACCESS
variables:
FOSSA_REPORT_URL: https://app.fossa.com/api/revisions/custom%2B12773%2F${CI_PROJECT_NAME}%24${CI_COMMIT_SHA}/attribution?access=${FOSSA_ACCESS}&includeProjectLicense=true&includeLicenseScan=true&includeDirectDependencies=true&includeDeepDependencies=true&includeLicenseHeaders=&includeLicenseList=true&format=HTML
artifacts:
paths:
- public
script:
- mkdir -p public
- wget $FOSSA_REPORT_URL -O public/fossa-notice.html
include:
- local: 'scanners/fossa-global.yml'
# --------------------------------------------------------------------------------
fossa-analyze:
extends: .maven
image: $CI_REGISTRY/divido/fossa-cli-utilities/fossa-cli-utilities:v3.1
stage: scan
needs: ['compile-and-unit-test']
tags: ['osdu-medium']
only:
variables:
- $FOSSA_API_KEY
script:
# fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results
# When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined
# then things will fail and we'll have to make this smarter
- test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG"
- $MAVEN install -DskipTests=true
- cp "$CI_PROJECT_DIR/.mvn/community-maven.settings.xml" ~/.m2/settings.xml
- fossa analyze --project "${CI_PROJECT_TITLE}" --project-url "${CI_PROJECT_URL}" --branch "${CI_COMMIT_BRANCH}"
- fossa-check-for-licensing-issues
include:
- local: 'scanners/fossa-global.yml'
# --------------------------------------------------------------------------------
fossa-analyze:
image: $CI_REGISTRY/divido/fossa-cli-utilities/fossa-cli-utilities:v4.0
stage: scan
needs: ['compile-and-unit-test']
tags: ['osdu-medium']
only:
variables:
- $FOSSA_API_KEY
script:
# fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results
# When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined
# then things will fail and we'll have to make this smarter
- test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG"
- |
if [ -e all-requirements.txt ]; then
# FOSSA needs all requirements in a single file, which has to be named "requirements.txt"
mv all-requirements.txt requirements.txt
else
echo "I was expecting a file named 'all-requirements.txt' to have been generated by compile-and-unit-test"
echo "However, that file doesn't seem to exist"
echo "----------------------------------------"
echo "That file should have been the output of a 'pip freeze', so that I knew what the full list of deep"
echo "dependencies were. I can't reasonably generate that in this job, because I don't know what python image"
echo "is appropriate. If this structure has been changed in the build/python.yml, you may need to update this"
echo "logic as well (in scanners/fossa-python.yml)"
exit 1
fi
- fossa analyze --project "${CI_PROJECT_TITLE}" --project-url "${CI_PROJECT_URL}" --branch "${CI_COMMIT_BRANCH}"
- fossa-check-for-licensing-issues
include:
- local: 'scanners/fossa-maven.yml'
fossa-analyze:
extends: .maven
image: $CI_REGISTRY/divido/fossa-cli-utilities/fossa-cli-utilities:v3.1
stage: scan
needs: ['compile-and-unit-test']
tags: ['osdu-medium']
only:
variables:
- $FOSSA_API_KEY
script:
# fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results
# When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined
# then things will fail and we'll have to make this smarter
- test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG"
- $MAVEN install -DskipTests=true
- cp "$CI_PROJECT_DIR/.mvn/community-maven.settings.xml" ~/.m2/settings.xml
- fossa analyze --project "${CI_PROJECT_TITLE}" --project-url "${CI_PROJECT_URL}" --branch "${CI_COMMIT_BRANCH}"
- fossa-check-for-licensing-issues
# --------------------------------------------------------------------------------
# This checks to see if the NOTICE file generated based on the current code
# matches the one that is committed in the repository. Because FOSSA can
# occassionally order the lines different between executions, we use a python
# program to parse the file into sections and sort each section. Then we compare
# and expect the same number of sections, names of sections, and contents
# (without consider order).
fossa-check-notice:
image: $CI_REGISTRY/divido/fossa-cli/fossa-cli:v4.1
stage: attribution
needs: ['fossa-analyze']
tags: ['osdu-small']
only:
variables:
- $FOSSA_API_KEY
artifacts:
when: on_failure
paths:
- public
script:
# See comments about this line from fossa-analyze
- test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG"
- mkdir -p public
- fossa report licenses --project "${CI_PROJECT_TITLE}" --branch "${CI_COMMIT_BRANCH}" > public/NOTICE
- mkdir committed-NOTICE generated-NOTICE
- |
python <<EOF
import re
import os
regexSeparator = re.compile(r"^=+$")
regexSubtitle = re.compile(r"^The following software have components provided under the terms of this license:$")
# ----------------------------------------
def writeComponents(sectionFile, components):
for component in sorted(components):
sectionFile.write(component)
sectionFile.close()
# ----------------------------------------
def matchSectionHeader(lines, idx):
if (idx <= 3): return None
sep1 = regexSeparator.match(lines[idx - 4].rstrip())
sep2 = regexSeparator.match(lines[idx - 2].rstrip())
sub = regexSubtitle.match(lines[idx - 1].rstrip())
blank = (lines[idx].rstrip() == "")
if sep1 and sep2 and sub and blank:
return lines[idx - 3].rstrip()
return None
# ----------------------------------------
def split(licenseFile, outputDir):
lines = []
with open(licenseFile, 'r') as f:
lines = f.readlines()
# --------------------
sectionFile = None
components = []
for idx in range(0, len(lines)):
line = lines[idx]
if (line.startswith("-")):
components.append(line)
newHeader = matchSectionHeader(lines, idx)
if newHeader:
if sectionFile is not None: writeComponents(sectionFile, components)
sectionFile = open(os.path.join(outputDir, newHeader), 'w')
components = []
# Write the header to the new file
for writeIdx in range(idx - 4, idx + 1):
sectionFile.write(lines[writeIdx])
# ----------------------------------------
split("NOTICE", "committed-NOTICE")
split("public/NOTICE", "generated-NOTICE")
EOF
- |
if ! diff -r committed-NOTICE generated-NOTICE; then
echo "================================================================================"
echo "There are differences between the NOTICE file in the repository and the one generated just now by this command"
echo "The most likely cause of this is changes to the project dependencies -- the NOTICE file is providing attribution"
echo "to the projects being used, so it must be updated whenever that changes. This includes projects used indirectly as"
echo "dependencies of dependencies."
echo "----------------------------------------"
echo "You can download the NOTICE file from this job's artifacts and directly commit it to the repository to resolve"
echo "this. Before doing so, review the differences to make sure that they make sense given the changes that you have"
echo "made. If they do not, reach out to a maintainer to help diagnose the issue."
exit 1
fi
# --------------------------------------------------------------------------------
fossa-report:
image: $CI_REGISTRY/divido/fossa-cli/fossa-cli:v4.1
stage: attribution
needs: ['fossa-analyze']
tags: ['osdu-small']
only:
variables:
# To get the FOSSA_ACCESS, log into the project's main page and generate a "Live Project Report" URL from the Summary tab
# The access token will be shown in the URL (the last path component)
- $FOSSA_API_KEY && $FOSSA_ACCESS
deprecated-pipeline-include:
stage: .pre
variables:
FOSSA_REPORT_URL: https://app.fossa.com/api/revisions/custom%2B12773%2F${CI_PROJECT_NAME}%24${CI_COMMIT_SHA}/attribution?access=${FOSSA_ACCESS}&includeProjectLicense=true&includeLicenseScan=true&includeDirectDependencies=true&includeDeepDependencies=true&includeLicenseHeaders=&includeLicenseList=true&format=HTML
artifacts:
paths:
- public
DEPRECATED_MSG: "The FOSSA scanner now supports different build environments, which should be specified directly. Consider using fossa-maven.yml instead."
allow_failure: true
script:
- mkdir -p public
- wget $FOSSA_REPORT_URL -O public/fossa-notice.html
- echo "$DEPRECATED_MSG"
- /bin/false
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment