Merge branch 'restore-fossa-ci' into 'master'

Remove experimental FOSSA logic, now that it is merged in the main CI-CD branches

See merge request !377

.gitlab-ci.yml

@@ -46,6 +46,9 @@ include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "build/python.yml"
 
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "scanners/fossa-python.yml"
+
   - project: "osdu/platform/ci-cd-pipelines"
     file: "scanners/gitlab-ultimate.yml"
 
@@ -225,133 +228,3 @@ osdu-gcp-dev2-test:
 # Allow failure on private development deployments
 ibm-deploy-devpri:
   allow_failure: true
-
-# --------------------------------------------------------------------------------
-# Experimental FOSSA jobs. These will be promoted to the standard ci-cd-pipelines after
-# they've had some testing in a real project
-
-fossa-analyze:
-  image: $CI_REGISTRY/divido/fossa-with-cache/incremental:latest
-
-  stage: scan
-  needs: ['compile-and-unit-test']
-  rules:
-    - if: $FOSSA_API_KEY
-
-  variables:
-    FOSSA_OUTPUT_DIR: fossa-output
-
-  artifacts:
-    paths:
-      - fossa-output
-    when: always
-    expire_in: 2 days
-
-  script:
-    # fossa-with-cache needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results
-    # When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined
-    # then things will fail and we'll have to make this smarter
-    - test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG"
-
-    - |
-      if [ ! -e all-requirements.txt ]; then
-        echo "I was expecting a file named 'all-requirements.txt' to have been generated by compile-and-unit-test"
-        echo "However, that file doesn't seem to exist"
-        echo "----------------------------------------"
-        echo "That file should have been the output of a 'pip freeze', so that I knew what the full list of deep"
-        echo "dependencies were. I can't reasonably generate that in this job, because I don't know what python image"
-        echo "is appropriate. If this structure has been changed in the build/python.yml, you may need to update this"
-        echo "logic as well (in scanners/fossa-python.yml)"
-        exit 1
-      fi
-
-    # This variable is used by the python build environment to refer to the set of requirements that need to
-    # be compiled down into the single 'all-requirements.txt'. Here, we override it to supply fossa-with-cache
-    # with a direct answer.
-    - PIP_REQUIREMENTS=all-requirements.txt fossa-with-cache
-
-fossa-check-notice:
-  image: $CI_REGISTRY/divido/fossa-with-cache/incremental:latest
-  stage: scan
-  needs: ['fossa-analyze']
-  tags: ['osdu-small']
-  rules:
-    - if: $FOSSA_API_KEY
-
-  artifacts:
-    when: on_failure
-    paths:
-      - fossa-output/cached-NOTICE
-      - fossa-output/generated-clean-NOTICE
-    expire_in: 2 days
-
-  script:
-    # Check to see if a newer commit exists for the pipeline's branch, and if it does, use that NOTICE instead of this one's
-    - |
-      if [ "$CI_COMMIT_BRANCH" != "" ]; then
-        colorCmd="\e[32;1m"
-        colorReset="\e[0m"
-        function echoCmd() {
-          echo -e "${colorCmd}>" "$@" "${colorReset}"
-        }
-
-        echoCmd git fetch
-        git fetch
-
-        echoCmd git diff --name-only HEAD origin/$CI_COMMIT_BRANCH
-        branchDiffs="$(git diff --name-only HEAD origin/$CI_COMMIT_BRANCH)"
-        echo $branchDiffs
-        echo "--------------------"
-
-        if [ "$branchDiffs" == "NOTICE" ]; then
-          echo "The branch associated with this pipeline ($CI_COMMIT_BRANCH) has been changed, but the only changes are the NOTICE file"
-          echo "I will use the NOTICE file from origin/$CI_COMMIT_BRANCH ($(git rev-parse --short origin/$CI_COMMIT_BRANCH)) as the basis for comparison"
-          echoCmd git checkout origin/$CI_COMMIT_BRANCH -- NOTICE
-          git checkout origin/$CI_COMMIT_BRANCH -- NOTICE
-
-        elif [ "$branchDiffs" == "" ]; then
-          echo "The branch associated with this pipeline ($CI_COMMIT_BRANCH) has not been changed since the commit that spawned this pipeline"
-          echo "I will use the NOTICE file from the pipeline's commit ($CI_COMMIT_SHORT_SHA) as the basis for comparison"
-
-        else
-          echo "The branch associated with this pipeline ($CI_COMMIT_BRANCH) has been changed, but the changes include more than just the NOTICE file"
-          echo "I will use the NOTICE file from the pipeline's commit ($CI_COMMIT_SHORT_SHA) as the basis for comparison"
-        fi
-      fi
-
-    # Use a cached NOTICE if available, otherwise use a generated one
-    - |
-      if [ -e fossa-output/cached-NOTICE ]; then
-        fossaGeneratedNotice=fossa-output/cached-NOTICE;
-
-      elif [ -e fossa-output/generated-clean-NOTICE ]; then
-        fossaGeneratedNotice=fossa-output/generated-clean-NOTICE
-
-      else
-        echo "Couldn't find either a cached-NOTICE or generated-clean-NOTICE in the fossa-output/ directory"
-        echo
-        echo "At least one of these should have been generated by a previous job stage (fossa-analyze) and stored"
-        echo "as an artifact. Something must be wrong in the CI setup"
-        exit 1
-      fi
-
-      echo "Comparing with $fossaGeneratedNotice"
-
-    # If the comparison finds differences, let the user know what to do next
-    - |
-      if ! fossa-compare-notices NOTICE $fossaGeneratedNotice; then
-        echo --------------------------------------------------------------------------------
-        echo "There are differences in the NOTICE file"
-        echo "Please review these differences, and if they look appropriate based on your"
-        echo "changes, update the committed NOTICE file"
-        echo "--------------------"
-        echo "If you make changes to the NOTICE file (and only the NOTICE file), you can"
-        echo "re-run this single stage of the pipeline alone rather than the whole pipeline"
-        echo "One way to achieve this:"
-        echo "$ wget -O NOTICE '${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/${fossaGeneratedNotice}?inline=false'"
-        echo "$ git add NOTICE"
-        echo "$ git commit -m 'Updating NOTICE'"
-        echo "$ git push -o ci.skip"
-        echo "Then retry this job"
-        exit 1
-      fi