From 352a3db891f7bd65a357ab57c4f5936cd7fd32c7 Mon Sep 17 00:00:00 2001 From: Seismic DMS Bot <sdms_bot@slb.com> Date: Fri, 29 Jul 2022 10:06:59 -0500 Subject: [PATCH] ci: adding fossa scan --- .devcontainer/Dockerfile | 8 +++---- .devcontainer/README.md | 4 ++-- .devcontainer/sync_deps.py | 22 +++++++++---------- README.md | 10 ++++----- devops/docker/build.ubuntu.dockerfile | 12 +++++----- .../providers/aws/build-aws/buildspec.yaml | 4 ++-- ...{requirements.txt => all-requirements.txt} | 0 test/aws-test/build-aws/run-tests.sh | 4 ++-- 8 files changed, 32 insertions(+), 32 deletions(-) rename test/{requirements.txt => all-requirements.txt} (100%) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 36d8804..c9183a8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,12 +7,12 @@ USER vscode WORKDIR /tmp # Install required dependencies -COPY requirements.txt ./requirements.txt -RUN pip3 install -r requirements.txt +COPY all-requirements.txt ./all-requirements.txt +RUN pip3 install -r all-requirements.txt # Install tests required dependencies -COPY test/requirements.txt ./requirements.txt -RUN pip3 install -r requirements.txt +COPY test/all-requirements.txt ./all-requirements.txt +RUN pip3 install -r all-requirements.txt WORKDIR / diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 968c463..b27eeb2 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -24,7 +24,7 @@ This current Devcontainer is scoped to a repo folder. Currently the Docker image #### Requirements -Dockerfile currently looks to the `requirements.txt` for the remaining python tools and libraries. +Dockerfile currently looks to the `all-requirements.txt` for the remaining python tools and libraries. #### Extensions The image also has the following VS Code extension installed. See devcontainer.json for more details. @@ -55,7 +55,7 @@ You will be running the SDUtil application with login credentials or an Oauth2 t #### DevContainer Sessions and Package Management -Current implementation of devcontainers bakes requirements.txt packages into the image. However, devcontainers have the concept **user session** in which devs interacting with source code from within the container can exit the container and return to the running container without losing their session state and without rebuilding the container. Therefore, new changes made to the requirements.txt outside the user session are by default not detected upon resuming the session. To activate detection of [requirements.txt](../requirements.txt) changes and package syncronization, take advantage of devcontainer's [**poststartcommand**](https://code.visualstudio.com/docs/remote/devcontainerjson-reference) feature with the following command as a value. +Current implementation of devcontainers bakes all-requirements.txt packages into the image. However, devcontainers have the concept **user session** in which devs interacting with source code from within the container can exit the container and return to the running container without losing their session state and without rebuilding the container. Therefore, new changes made to the all-requirements.txt outside the user session are by default not detected upon resuming the session. To activate detection of [all-requirements.txt](../all-requirements.txt) changes and package syncronization, take advantage of devcontainer's [**poststartcommand**](https://code.visualstudio.com/docs/remote/devcontainerjson-reference) feature with the following command as a value. > Note: Orphaned packages leftover as a result of uninstalling python packages are not removed between devcontainer **user session**. A clean package environment requires a docker rebuild of the devcontainer. Docker rebuilds are recommended when experiencing issues with python packages during a devcontainer **user session**. diff --git a/.devcontainer/sync_deps.py b/.devcontainer/sync_deps.py index 78f88de..0e799a5 100644 --- a/.devcontainer/sync_deps.py +++ b/.devcontainer/sync_deps.py @@ -4,11 +4,11 @@ import sys import difflib import subprocess -orig_deps_file = './../session/requirements.txt' -new_deps_file = './requirements.txt' +orig_deps_file = './../session/all-requirements.txt' +new_deps_file = './all-requirements.txt' ''' -read package dependencies listed in requirements.txt files +read package dependencies listed in all-requirements.txt files ''' def read_dependencies(): orig_deps = "" @@ -17,18 +17,18 @@ def read_dependencies(): orig_deps = open(orig_deps_file, 'r') new_deps = open(new_deps_file, 'r') else: - raise Exception("A requirements.txt file was not found") + raise Exception("A all-requirements.txt file was not found") return orig_deps, new_deps ''' -report the diff between two requirements.txt files +report the diff between two all-requirements.txt files ''' def report_changes(orig_deps, new_deps): change_report = difflib.ndiff(orig_deps.readlines(), new_deps.readlines()) return change_report ''' -sync changes by detecting changes in requirements.txt to pip un-/install +sync changes by detecting changes in all-requirements.txt to pip un-/install ''' def sync_changes(change_report): changes_detected = False @@ -36,7 +36,7 @@ def sync_changes(change_report): if change.startswith('- '): dependency = change[2:].replace('\n','') auto_uninstall = "-y" - print("Detected requirements.txt change for [{dep}] \n".format(dep=dependency)) + print("Detected all-requirements.txt change for [{dep}] \n".format(dep=dependency)) sys.stdout.flush() try: # Note: Uninstall will not remove site-packages/dist-info directories or resultant dangling dependencies @@ -46,7 +46,7 @@ def sync_changes(change_report): changes_detected = True if change.startswith('+ '): dependency = change[2:].replace('\n','') - print("Detected requirements.txt change for [{dep}] \n".format(dep=dependency)) + print("Detected all-requirements.txt change for [{dep}] \n".format(dep=dependency)) sys.stdout.flush() os.system("pip --no-cache-dir install {dep}".format(dep=dependency)) changes_detected = True @@ -63,18 +63,18 @@ def replace_session_deps(changes_detected): if os.path.exists(orig_deps_file) and os.path.exists(new_deps_file): os.remove(orig_deps_file ) - print("Updating session with latest requirements.txt \n") + print("Updating session with latest all-requirements.txt \n") sys.stdout.flush() with open(new_deps_file) as new_deps: with open(orig_deps_file, "w") as orig_deps: for line in new_deps: orig_deps.write(line) else: - raise Exception("A requirements.txt file was not found \n") + raise Exception("A all-requirements.txt file was not found \n") def main(): try: - print("main module: syncing requirements.txt deps for devcontainer session in [os:{os}]...\n".format(os=sys.platform)) + print("main module: syncing all-requirements.txt deps for devcontainer session in [os:{os}]...\n".format(os=sys.platform)) sys.stdout.flush() orig_deps, new_deps = read_dependencies() change_report = report_changes(orig_deps, new_deps) diff --git a/README.md b/README.md index bf8735c..3d18949 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ unzip the utility to a folder unzip sdutil-1.0.0 ``` -The utility requires additional modules noted in [requirements.txt](requirements.txt). You could either install the modules as is or install them in virtualenv to keep your host clean from package conflicts. if you don't want to install them in a virtual environment please jump directly to the step 3. +The utility requires additional modules noted in [all-requirements.txt](all-requirements.txt). You could either install the modules as is or install them in virtualenv to keep your host clean from package conflicts. if you don't want to install them in a virtual environment please jump directly to the step 3. ```sh # check if virtualenv is already installed @@ -67,7 +67,7 @@ install required dependencies: ```bash # run it from the extracted sdutil folder -pip install -r requirements.txt +pip install -r all-requirements.txt ``` ## Usage @@ -218,7 +218,7 @@ Requirements ```bash # install required dependencies: - pip install -r test/requirements.txt + pip install -r test/all-requirements.txt ``` Integral/Unit tests @@ -284,7 +284,7 @@ python3 -m venv sdutilenv source sdutilenv/bin/Activate #install python package for sdutil -pip install -r requirements.txt +pip install -r all-requirements.txt ``` @@ -386,7 +386,7 @@ virtualenv sdutilenv source ./sdutilenv/bin/activate #install python package for sdutil -pip install -r requirements.txt +pip install -r all-requirements.txt ``` diff --git a/devops/docker/build.ubuntu.dockerfile b/devops/docker/build.ubuntu.dockerfile index e1d50cb..fe947cf 100644 --- a/devops/docker/build.ubuntu.dockerfile +++ b/devops/docker/build.ubuntu.dockerfile @@ -9,14 +9,14 @@ RUN apt-get -y update && apt-get -y install python3-pip WORKDIR /tmp # install required dependencies -COPY requirements.txt ./requirements.txt -RUN pip3 install -r requirements.txt -RUN rm -f requirements.txt +COPY all-requirements.txt ./all-requirements.txt +RUN pip3 install -r all-requirements.txt +RUN rm -f all-requirements.txt # install tests required dependencies -COPY test/requirements.txt ./requirements.txt -RUN pip3 install -r requirements.txt -RUN rm -f requirements.txt +COPY test/all-requirements.txt ./all-requirements.txt +RUN pip3 install -r all-requirements.txt +RUN rm -f all-requirements.txt # finalize WORKDIR / diff --git a/sdlib/api/providers/aws/build-aws/buildspec.yaml b/sdlib/api/providers/aws/build-aws/buildspec.yaml index 6f8f8cd..3024f52 100644 --- a/sdlib/api/providers/aws/build-aws/buildspec.yaml +++ b/sdlib/api/providers/aws/build-aws/buildspec.yaml @@ -26,8 +26,8 @@ phases: commands: - export REPO_NAME=${PWD##*/} - export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'` - - echo "Installing requirements.txt" - - pip install -r ./requirements.txt + - echo "Installing all-requirements.txt" + - pip install -r ./all-requirements.txt - pip install pylint - pylint ./sdlib/api/providers/aws/*.py --disable=F0001 --errors-only - echo "Building integration testing assemblies and gathering artifacts..." diff --git a/test/requirements.txt b/test/all-requirements.txt similarity index 100% rename from test/requirements.txt rename to test/all-requirements.txt diff --git a/test/aws-test/build-aws/run-tests.sh b/test/aws-test/build-aws/run-tests.sh index f72559b..940fc7d 100755 --- a/test/aws-test/build-aws/run-tests.sh +++ b/test/aws-test/build-aws/run-tests.sh @@ -42,11 +42,11 @@ pushd "$SCRIPT_SOURCE_DIR"/../../ #going up to ../bin echo $(pwd) python3 -m venv sdutilenv source sdutilenv/bin/activate -pip install -r aws-test/build-aws/requirements.txt +pip install -r aws-test/build-aws/all-requirements.txt echo 'Generating token...' -pip3 install -r aws-test/build-aws/requirements.txt +pip3 install -r aws-test/build-aws/all-requirements.txt token=$(python3 aws-test/build-aws/aws_jwt_client.py) echo 'Registering a subproject for testing...' -- GitLab