diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 36d88049d4fd5141f714252bbaed383eb86c7d14..c9183a84c99c4a11f6e124ccc71d11f28e088db1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,12 +7,12 @@ USER vscode WORKDIR /tmp # Install required dependencies -COPY requirements.txt ./requirements.txt -RUN pip3 install -r requirements.txt +COPY all-requirements.txt ./all-requirements.txt +RUN pip3 install -r all-requirements.txt # Install tests required dependencies -COPY test/requirements.txt ./requirements.txt -RUN pip3 install -r requirements.txt +COPY test/all-requirements.txt ./all-requirements.txt +RUN pip3 install -r all-requirements.txt WORKDIR / diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 968c4637c40ccf8a2dd448ca124b0444fc3392b2..b27eeb256ed4d702d579f071daf692ab8ae330b5 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -24,7 +24,7 @@ This current Devcontainer is scoped to a repo folder. Currently the Docker image #### Requirements -Dockerfile currently looks to the `requirements.txt` for the remaining python tools and libraries. +Dockerfile currently looks to the `all-requirements.txt` for the remaining python tools and libraries. #### Extensions The image also has the following VS Code extension installed. See devcontainer.json for more details. @@ -55,7 +55,7 @@ You will be running the SDUtil application with login credentials or an Oauth2 t #### DevContainer Sessions and Package Management -Current implementation of devcontainers bakes requirements.txt packages into the image. However, devcontainers have the concept **user session** in which devs interacting with source code from within the container can exit the container and return to the running container without losing their session state and without rebuilding the container. Therefore, new changes made to the requirements.txt outside the user session are by default not detected upon resuming the session. To activate detection of [requirements.txt](../requirements.txt) changes and package syncronization, take advantage of devcontainer's [**poststartcommand**](https://code.visualstudio.com/docs/remote/devcontainerjson-reference) feature with the following command as a value. +Current implementation of devcontainers bakes all-requirements.txt packages into the image. However, devcontainers have the concept **user session** in which devs interacting with source code from within the container can exit the container and return to the running container without losing their session state and without rebuilding the container. Therefore, new changes made to the all-requirements.txt outside the user session are by default not detected upon resuming the session. To activate detection of [all-requirements.txt](../all-requirements.txt) changes and package syncronization, take advantage of devcontainer's [**poststartcommand**](https://code.visualstudio.com/docs/remote/devcontainerjson-reference) feature with the following command as a value. > Note: Orphaned packages leftover as a result of uninstalling python packages are not removed between devcontainer **user session**. A clean package environment requires a docker rebuild of the devcontainer. Docker rebuilds are recommended when experiencing issues with python packages during a devcontainer **user session**. diff --git a/.devcontainer/sync_deps.py b/.devcontainer/sync_deps.py index 78f88de0a03bccb949282548d52d10ba9529b978..0e799a56f4718f542e1028dbc51e17d6b605d707 100644 --- a/.devcontainer/sync_deps.py +++ b/.devcontainer/sync_deps.py @@ -4,11 +4,11 @@ import sys import difflib import subprocess -orig_deps_file = './../session/requirements.txt' -new_deps_file = './requirements.txt' +orig_deps_file = './../session/all-requirements.txt' +new_deps_file = './all-requirements.txt' ''' -read package dependencies listed in requirements.txt files +read package dependencies listed in all-requirements.txt files ''' def read_dependencies(): orig_deps = "" @@ -17,18 +17,18 @@ def read_dependencies(): orig_deps = open(orig_deps_file, 'r') new_deps = open(new_deps_file, 'r') else: - raise Exception("A requirements.txt file was not found") + raise Exception("A all-requirements.txt file was not found") return orig_deps, new_deps ''' -report the diff between two requirements.txt files +report the diff between two all-requirements.txt files ''' def report_changes(orig_deps, new_deps): change_report = difflib.ndiff(orig_deps.readlines(), new_deps.readlines()) return change_report ''' -sync changes by detecting changes in requirements.txt to pip un-/install +sync changes by detecting changes in all-requirements.txt to pip un-/install ''' def sync_changes(change_report): changes_detected = False @@ -36,7 +36,7 @@ def sync_changes(change_report): if change.startswith('- '): dependency = change[2:].replace('\n','') auto_uninstall = "-y" - print("Detected requirements.txt change for [{dep}] \n".format(dep=dependency)) + print("Detected all-requirements.txt change for [{dep}] \n".format(dep=dependency)) sys.stdout.flush() try: # Note: Uninstall will not remove site-packages/dist-info directories or resultant dangling dependencies @@ -46,7 +46,7 @@ def sync_changes(change_report): changes_detected = True if change.startswith('+ '): dependency = change[2:].replace('\n','') - print("Detected requirements.txt change for [{dep}] \n".format(dep=dependency)) + print("Detected all-requirements.txt change for [{dep}] \n".format(dep=dependency)) sys.stdout.flush() os.system("pip --no-cache-dir install {dep}".format(dep=dependency)) changes_detected = True @@ -63,18 +63,18 @@ def replace_session_deps(changes_detected): if os.path.exists(orig_deps_file) and os.path.exists(new_deps_file): os.remove(orig_deps_file ) - print("Updating session with latest requirements.txt \n") + print("Updating session with latest all-requirements.txt \n") sys.stdout.flush() with open(new_deps_file) as new_deps: with open(orig_deps_file, "w") as orig_deps: for line in new_deps: orig_deps.write(line) else: - raise Exception("A requirements.txt file was not found \n") + raise Exception("A all-requirements.txt file was not found \n") def main(): try: - print("main module: syncing requirements.txt deps for devcontainer session in [os:{os}]...\n".format(os=sys.platform)) + print("main module: syncing all-requirements.txt deps for devcontainer session in [os:{os}]...\n".format(os=sys.platform)) sys.stdout.flush() orig_deps, new_deps = read_dependencies() change_report = report_changes(orig_deps, new_deps) diff --git a/README.md b/README.md index bf8735c99debfec7e7b392312aae2601b2cab8cf..3d18949f3bd61dd756015d4c9c7222aa1cea8add 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ unzip the utility to a folder unzip sdutil-1.0.0 ``` -The utility requires additional modules noted in [requirements.txt](requirements.txt). You could either install the modules as is or install them in virtualenv to keep your host clean from package conflicts. if you don't want to install them in a virtual environment please jump directly to the step 3. +The utility requires additional modules noted in [all-requirements.txt](all-requirements.txt). You could either install the modules as is or install them in virtualenv to keep your host clean from package conflicts. if you don't want to install them in a virtual environment please jump directly to the step 3. ```sh # check if virtualenv is already installed @@ -67,7 +67,7 @@ install required dependencies: ```bash # run it from the extracted sdutil folder -pip install -r requirements.txt +pip install -r all-requirements.txt ``` ## Usage @@ -218,7 +218,7 @@ Requirements ```bash # install required dependencies: - pip install -r test/requirements.txt + pip install -r test/all-requirements.txt ``` Integral/Unit tests @@ -284,7 +284,7 @@ python3 -m venv sdutilenv source sdutilenv/bin/Activate #install python package for sdutil -pip install -r requirements.txt +pip install -r all-requirements.txt ``` @@ -386,7 +386,7 @@ virtualenv sdutilenv source ./sdutilenv/bin/activate #install python package for sdutil -pip install -r requirements.txt +pip install -r all-requirements.txt ``` diff --git a/devops/docker/build.ubuntu.dockerfile b/devops/docker/build.ubuntu.dockerfile index e1d50cb0884c0c7a9c7fea5dbc41cab17fead63b..fe947cf0536cc75a9124940fd23c377870e41ee1 100644 --- a/devops/docker/build.ubuntu.dockerfile +++ b/devops/docker/build.ubuntu.dockerfile @@ -9,14 +9,14 @@ RUN apt-get -y update && apt-get -y install python3-pip WORKDIR /tmp # install required dependencies -COPY requirements.txt ./requirements.txt -RUN pip3 install -r requirements.txt -RUN rm -f requirements.txt +COPY all-requirements.txt ./all-requirements.txt +RUN pip3 install -r all-requirements.txt +RUN rm -f all-requirements.txt # install tests required dependencies -COPY test/requirements.txt ./requirements.txt -RUN pip3 install -r requirements.txt -RUN rm -f requirements.txt +COPY test/all-requirements.txt ./all-requirements.txt +RUN pip3 install -r all-requirements.txt +RUN rm -f all-requirements.txt # finalize WORKDIR / diff --git a/sdlib/api/providers/aws/build-aws/buildspec.yaml b/sdlib/api/providers/aws/build-aws/buildspec.yaml index 6f8f8cdb7b0df5e702089f0b8c6692beaef84bc1..3024f520b812be414c0c713c231e041511ea0a62 100644 --- a/sdlib/api/providers/aws/build-aws/buildspec.yaml +++ b/sdlib/api/providers/aws/build-aws/buildspec.yaml @@ -26,8 +26,8 @@ phases: commands: - export REPO_NAME=${PWD##*/} - export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'` - - echo "Installing requirements.txt" - - pip install -r ./requirements.txt + - echo "Installing all-requirements.txt" + - pip install -r ./all-requirements.txt - pip install pylint - pylint ./sdlib/api/providers/aws/*.py --disable=F0001 --errors-only - echo "Building integration testing assemblies and gathering artifacts..." diff --git a/test/requirements.txt b/test/all-requirements.txt similarity index 100% rename from test/requirements.txt rename to test/all-requirements.txt diff --git a/test/aws-test/build-aws/run-tests.sh b/test/aws-test/build-aws/run-tests.sh index f72559b17a94ac2f4295a90fca94fa59a63d0db8..940fc7da0628825baf4eab03dc867f921f046966 100755 --- a/test/aws-test/build-aws/run-tests.sh +++ b/test/aws-test/build-aws/run-tests.sh @@ -42,11 +42,11 @@ pushd "$SCRIPT_SOURCE_DIR"/../../ #going up to ../bin echo $(pwd) python3 -m venv sdutilenv source sdutilenv/bin/activate -pip install -r aws-test/build-aws/requirements.txt +pip install -r aws-test/build-aws/all-requirements.txt echo 'Generating token...' -pip3 install -r aws-test/build-aws/requirements.txt +pip3 install -r aws-test/build-aws/all-requirements.txt token=$(python3 aws-test/build-aws/aws_jwt_client.py) echo 'Registering a subproject for testing...'