Commit 62055c53 authored by David Diederich's avatar David Diederich
Browse files

Merge branch 'restore-fossa-ci' into 'master'

Remove experimental FOSSA logic, now that it is merged in the main CI-CD branches

See merge request !377
parents 6c16e609 6e901f7f
Pipeline #87453 failed with stages
in 53 minutes and 48 seconds
......@@ -46,6 +46,9 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "build/python.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/fossa-python.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
......@@ -225,133 +228,3 @@ osdu-gcp-dev2-test:
# Allow failure on private development deployments
ibm-deploy-devpri:
allow_failure: true
# --------------------------------------------------------------------------------
# Experimental FOSSA jobs. These will be promoted to the standard ci-cd-pipelines after
# they've had some testing in a real project
fossa-analyze:
image: $CI_REGISTRY/divido/fossa-with-cache/incremental:latest
stage: scan
needs: ['compile-and-unit-test']
rules:
- if: $FOSSA_API_KEY
variables:
FOSSA_OUTPUT_DIR: fossa-output
artifacts:
paths:
- fossa-output
when: always
expire_in: 2 days
script:
# fossa-with-cache needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results
# When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined
# then things will fail and we'll have to make this smarter
- test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG"
- |
if [ ! -e all-requirements.txt ]; then
echo "I was expecting a file named 'all-requirements.txt' to have been generated by compile-and-unit-test"
echo "However, that file doesn't seem to exist"
echo "----------------------------------------"
echo "That file should have been the output of a 'pip freeze', so that I knew what the full list of deep"
echo "dependencies were. I can't reasonably generate that in this job, because I don't know what python image"
echo "is appropriate. If this structure has been changed in the build/python.yml, you may need to update this"
echo "logic as well (in scanners/fossa-python.yml)"
exit 1
fi
# This variable is used by the python build environment to refer to the set of requirements that need to
# be compiled down into the single 'all-requirements.txt'. Here, we override it to supply fossa-with-cache
# with a direct answer.
- PIP_REQUIREMENTS=all-requirements.txt fossa-with-cache
fossa-check-notice:
image: $CI_REGISTRY/divido/fossa-with-cache/incremental:latest
stage: scan
needs: ['fossa-analyze']
tags: ['osdu-small']
rules:
- if: $FOSSA_API_KEY
artifacts:
when: on_failure
paths:
- fossa-output/cached-NOTICE
- fossa-output/generated-clean-NOTICE
expire_in: 2 days
script:
# Check to see if a newer commit exists for the pipeline's branch, and if it does, use that NOTICE instead of this one's
- |
if [ "$CI_COMMIT_BRANCH" != "" ]; then
colorCmd="\e[32;1m"
colorReset="\e[0m"
function echoCmd() {
echo -e "${colorCmd}>" "$@" "${colorReset}"
}
echoCmd git fetch
git fetch
echoCmd git diff --name-only HEAD origin/$CI_COMMIT_BRANCH
branchDiffs="$(git diff --name-only HEAD origin/$CI_COMMIT_BRANCH)"
echo $branchDiffs
echo "--------------------"
if [ "$branchDiffs" == "NOTICE" ]; then
echo "The branch associated with this pipeline ($CI_COMMIT_BRANCH) has been changed, but the only changes are the NOTICE file"
echo "I will use the NOTICE file from origin/$CI_COMMIT_BRANCH ($(git rev-parse --short origin/$CI_COMMIT_BRANCH)) as the basis for comparison"
echoCmd git checkout origin/$CI_COMMIT_BRANCH -- NOTICE
git checkout origin/$CI_COMMIT_BRANCH -- NOTICE
elif [ "$branchDiffs" == "" ]; then
echo "The branch associated with this pipeline ($CI_COMMIT_BRANCH) has not been changed since the commit that spawned this pipeline"
echo "I will use the NOTICE file from the pipeline's commit ($CI_COMMIT_SHORT_SHA) as the basis for comparison"
else
echo "The branch associated with this pipeline ($CI_COMMIT_BRANCH) has been changed, but the changes include more than just the NOTICE file"
echo "I will use the NOTICE file from the pipeline's commit ($CI_COMMIT_SHORT_SHA) as the basis for comparison"
fi
fi
# Use a cached NOTICE if available, otherwise use a generated one
- |
if [ -e fossa-output/cached-NOTICE ]; then
fossaGeneratedNotice=fossa-output/cached-NOTICE;
elif [ -e fossa-output/generated-clean-NOTICE ]; then
fossaGeneratedNotice=fossa-output/generated-clean-NOTICE
else
echo "Couldn't find either a cached-NOTICE or generated-clean-NOTICE in the fossa-output/ directory"
echo
echo "At least one of these should have been generated by a previous job stage (fossa-analyze) and stored"
echo "as an artifact. Something must be wrong in the CI setup"
exit 1
fi
echo "Comparing with $fossaGeneratedNotice"
# If the comparison finds differences, let the user know what to do next
- |
if ! fossa-compare-notices NOTICE $fossaGeneratedNotice; then
echo --------------------------------------------------------------------------------
echo "There are differences in the NOTICE file"
echo "Please review these differences, and if they look appropriate based on your"
echo "changes, update the committed NOTICE file"
echo "--------------------"
echo "If you make changes to the NOTICE file (and only the NOTICE file), you can"
echo "re-run this single stage of the pipeline alone rather than the whole pipeline"
echo "One way to achieve this:"
echo "$ wget -O NOTICE '${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/${fossaGeneratedNotice}?inline=false'"
echo "$ git add NOTICE"
echo "$ git commit -m 'Updating NOTICE'"
echo "$ git push -o ci.skip"
echo "Then retry this job"
exit 1
fi
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment