Commit 8ab06c2f authored by Alexandre Vincent's avatar Alexandre Vincent
Browse files

Merge branch 'avincent/pip-tools-dependency-management' into 'master'

pip tools dependency management

See merge request !200
parents f484d749 3dfb4ed0
Pipeline #57189 passed with stages
in 14 minutes and 26 seconds
......@@ -14,7 +14,7 @@
# limitations under the License.
variables:
PIP_REQUIREMENTS: "frozenrequirements_dev.txt requirements.txt requirements_dev.txt"
PIP_REQUIREMENTS: "requirements.txt requirements_dev.txt"
AZURE_SERVICE: wellbore-ddms
AZURE_DOCKER_SUBDIR: build/Dockerfile
......@@ -172,8 +172,7 @@ osdu-gcp-test-python:
- source env/bin/activate
- pip install --upgrade pip
- pip install wheel pytest pytest-cov
- pip install -r requirements.txt
- pip install -r requirements_dev.txt
- pip install -r $PIP_REQUIREMENTS
- cd tests/integration
- echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > file.json
- gcloud auth activate-service-account --key-file file.json
......
......@@ -99,6 +99,12 @@ Note: container memory is not entirely dedicated to Dask workers, fastapi servic
pip install -r requirements.txt
```
Or, for a developer setup, this will install tools to help you work with the code.
```bash
pip install -r requirements.txt -r requirements_dev.txt
```
6. Run the service
```bash
......@@ -329,7 +335,7 @@ docker build -t=$IMAGE_TAG --rm . -f ./build/dockerfile --build-arg PIP_WHEEL_DI
```bash
# Install test dependencies
pip install -r requirements_dev.txt
pip install -r requirements.txt -r requirements_dev.txt
python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit
```
......@@ -358,6 +364,34 @@ pytest ./functional --environment="./generated/postman_environment.json" --filte
For more information see the [integration tests README](tests/integration/README.md)
### Manage package dependencies
Anytime, you may want to ensure your virtual environment is in sync with your requirements specification.
For this you can use:
```bash
pip-sync
```
If you want to work with other requirements file, you can specify them
```bash
pip-sync requirements.txt requirements_dev.txt
```
If you want to update `requirements.txt` to retrieve the most recent version, respecting bounds set in `requirements.in`, you can use:
```bash
pip-compile
```
If you want to update the version of only one dependency, for instance fastapi:
```bash
pip-compile --upgrade-package fastapi
```
For more information: https://github.com/jazzband/pip-tools/
### Debugging:
#### Port Forward from Kubernetes
......
......@@ -14,9 +14,8 @@
FROM python:3.7-slim-buster
COPY requirements.txt frozenrequirements.txt ./
COPY requirements.txt ./
RUN pip install -r frozenrequirements.txt
RUN pip install -r requirements.txt
COPY ./app /app
......
adal==1.2.7
adlfs==0.7.7
aiohttp==3.7.4.post0
aioredis==1.3.1
anyio==3.2.1
asgiref==3.4.1
async-timeout==3.0.1
attrs==21.2.0
azure-common==1.1.27
azure-core==1.16.0
azure-datalake-store==0.0.52
azure-identity==1.6.0
azure-keyvault==4.1.0
azure-keyvault-certificates==4.3.0
azure-keyvault-keys==4.4.0
azure-keyvault-secrets==4.3.0
azure-storage-blob==12.8.1
backoff==1.11.1
boto3==1.18.1
botocore==1.21.1
cachetools==4.2.2
certifi==2021.5.30
cffi==1.14.6
chardet==4.0.0
charset-normalizer==2.0.3
click==8.0.1
cloudpickle==1.6.0
colorama==0.4.4
cryptography==3.4.7
dask==2021.7.2
decorator==5.0.9
distributed==2021.7.2
fastapi==0.66.0
fsspec==2021.7.0
gcsfs==2021.7.0
google-api-core==1.31.0
google-auth==1.33.0
google-auth-oauthlib==0.4.4
google-cloud-core==1.7.1
google-cloud-monitoring==0.36.0
google-cloud-trace==0.24.0
googleapis-common-protos==1.53.0
grpcio==1.38.1
h11==0.12.0
HeapDict==1.0.1
hiredis==2.0.0
httpcore==0.13.6
httpx==0.18.2
idna==3.2
isodate==0.6.0
jmespath==0.10.0
JSONBender==0.9.3
jsonpath-ng==1.5.3
locket==0.2.1
msal==1.12.0
msal-extensions==0.3.0
msgpack==1.0.2
msrest==0.6.21
multidict==5.1.0
numpy==1.21.1
oauthlib==3.1.1
opencensus==0.7.13
opencensus-context==0.1.2
opencensus-ext-azure==1.0.8
opencensus-ext-logging==0.1.0
opencensus-ext-ocagent==0.7.1
opencensus-ext-stackdriver==0.7.4
opencensus-proto==0.1.0
packaging==21.0
pandas==1.2.4
partd==1.2.0
ply==3.11
portalocker==1.7.1
protobuf==3.17.3
psutil==5.8.0
pyarrow==4.0.1
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycparser==2.20
pydantic==1.8.2
PyJWT==2.1.0
pyparsing==2.4.7
python-dateutil==2.8.2
python-multipart==0.0.5
python-rapidjson==1.4
python-ulid==1.0.3
pytz==2021.1
PyYAML==5.4.1
requests==2.26.0
requests-oauthlib==1.3.0
rfc3986==1.5.0
rsa==4.7.2
s3transfer==0.5.0
six==1.16.0
sniffio==1.2.0
sortedcontainers==2.4.0
starlette==0.14.2
structlog==21.1.0
tblib==1.7.0
toolz==0.11.1
tornado==6.1
typing-extensions==3.10.0.0
urllib3==1.26.6
uvicorn==0.14.0
xmltodict==0.11.0
yarl==1.6.3
zict==2.0.0
adal==1.2.7
adlfs==0.7.7
aiohttp==3.7.4.post0
aioredis==1.3.1
anyio==3.2.1
asgiref==3.4.1
async-timeout==3.0.1
atomicwrites==1.4.0
attrs==21.2.0
azure-common==1.1.27
azure-core==1.16.0
azure-datalake-store==0.0.52
azure-identity==1.6.0
azure-keyvault==4.1.0
azure-keyvault-certificates==4.3.0
azure-keyvault-keys==4.4.0
azure-keyvault-secrets==4.3.0
azure-storage-blob==12.8.1
backoff==1.11.1
boto3==1.18.1
botocore==1.21.1
cachetools==4.2.2
certifi==2021.5.30
cffi==1.14.6
chardet==4.0.0
charset-normalizer==2.0.3
click==8.0.1
cloudpickle==1.6.0
colorama==0.4.4
coverage==5.5
cryptography==3.4.7
dask==2021.7.2
decorator==5.0.9
distributed==2021.7.2
fastapi==0.66.0
fsspec==2021.7.0
gcsfs==2021.7.0
google-api-core==1.31.0
google-auth==1.33.0
google-auth-oauthlib==0.4.4
google-cloud-core==1.7.1
google-cloud-monitoring==0.36.0
google-cloud-trace==0.24.0
googleapis-common-protos==1.53.0
grpcio==1.38.1
h11==0.12.0
HeapDict==1.0.1
hiredis==2.0.0
httpcore==0.13.6
httpx==0.18.2
idna==3.2
iniconfig==1.1.1
isodate==0.6.0
jmespath==0.10.0
JSONBender==0.9.3
jsonpath-ng==1.5.3
jsonschema==3.2.0
locket==0.2.1
mock==4.0.3
msal==1.12.0
msal-extensions==0.3.0
msgpack==1.0.2
msrest==0.6.21
multidict==5.1.0
munch==2.5.0
numpy==1.21.1
oauthlib==3.1.1
openapi-schema-validator==0.1.5
openapi-spec-validator==0.3.1
opencensus==0.7.13
opencensus-context==0.1.2
opencensus-ext-azure==1.0.8
opencensus-ext-logging==0.1.0
opencensus-ext-ocagent==0.7.1
opencensus-ext-stackdriver==0.7.4
opencensus-proto==0.1.0
packaging==21.0
pandas==1.2.4
partd==1.2.0
pluggy==0.13.1
ply==3.11
portalocker==1.7.1
protobuf==3.17.3
psutil==5.8.0
py==1.10.0
pyarrow==4.0.1
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycparser==2.20
pydantic==1.8.2
PyJWT==2.1.0
pyparsing==2.4.7
pyrsistent==0.18.0
pytest==6.2.4
pytest-asyncio==0.15.1
pytest-cov==2.12.1
pytest-dependency==0.5.1
pytest-httpx==0.12.0
pytest-mock==3.6.1
python-dateutil==2.8.2
python-multipart==0.0.5
python-rapidjson==1.4
python-ulid==1.0.3
pytz==2021.1
PyYAML==5.4.1
requests==2.26.0
requests-oauthlib==1.3.0
rfc3986==1.5.0
rsa==4.7.2
s3transfer==0.5.0
six==1.16.0
sniffio==1.2.0
sortedcontainers==2.4.0
starlette==0.14.2
structlog==21.1.0
tblib==1.7.0
toml==0.10.2
toolz==0.11.1
tornado==6.1
typing-extensions==3.10.0.0
urllib3==1.26.6
uvicorn==0.14.0
xmltodict==0.11.0
yarl==1.6.3
zict==2.0.0
......@@ -14,11 +14,9 @@
FROM python:3.7-slim-buster
COPY requirements.txt requirements_dev.txt frozenrequirements.txt ./
COPY requirements.txt ./
RUN pip install -r frozenrequirements.txt
RUN pip install -r requirements.txt
RUN pip install -r requirements_dev.txt
COPY ./app /app
COPY provider/os-wellbore-ddms-aws/build-aws/entrypoint.sh .
......
fastapi==0.65.1
aiohttp==3.7.4.post0
cryptography>=2.7
pyarrow>=4.0.1, <5.0
pandas==1.2.4
uvicorn
Click
backoff
structlog
python-rapidjson
python-multipart
jsonpath-ng # TODO: remove when bulk_path no longer supported
jsonbender==0.9.3
natsort
opencensus
opencensus-ext-stackdriver
opencensus-ext-azure
opencensus-ext-ocagent
opencensus-ext-logging
# for chunking feature
dask[distributed]==2021.7.2
fsspec
python-ulid
--extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
osdu-log-recognition-lib>=0.0.9
osdu-data-ecosystem-storage~=1.2.0
osdu-data-ecosystem-search>=0.3.2, <0.4
osdu-core-lib-python-ibm~=1.1.0
osdu-core-lib-python-gcp~=1.1.0
osdu-core-lib-python-azure~=1.2.1
osdu-core-lib-python-aws>=0.0.1, <0.1
osdu-core-lib-python~=1.1.0
fastapi>=0.65.1
aiohttp==3.7.4.post0
cryptography>=2.7
pyarrow>=0.15.0
pandas==1.2.4
uvicorn
Click
backoff
structlog
python-rapidjson
python-multipart
jsonpath-ng # TODO: remove when bulk_path no longer supported
jsonbender==0.9.3
natsort
#
# This file is autogenerated by pip-compile with python 3.7
# To update, run:
#
# pip-compile requirements.in
#
--extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
opencensus
opencensus-ext-stackdriver
opencensus-ext-azure
opencensus-ext-ocagent
opencensus-ext-logging
# for chunking feature
adal==1.2.7
# via azure-datalake-store
adlfs==0.7.7
# via osdu-core-lib-python-azure
aiohttp==3.7.4.post0
# via
# -r requirements.in
# adlfs
# gcsfs
# osdu-core-lib-python
# osdu-core-lib-python-gcp
aioredis==1.3.1
# via osdu-core-lib-python
anyio==3.2.1
# via httpcore
asgiref==3.4.1
# via uvicorn
async-timeout==3.0.1
# via
# aiohttp
# aioredis
attrs==21.2.0
# via
# aiohttp
# osdu-core-lib-python
azure-common==1.1.27
# via
# azure-keyvault-certificates
# azure-keyvault-keys
# azure-keyvault-secrets
azure-core==1.16.0
# via
# adlfs
# azure-identity
# azure-keyvault-certificates
# azure-keyvault-keys
# azure-keyvault-secrets
# azure-storage-blob
azure-datalake-store==0.0.52
# via adlfs
azure-identity==1.6.0
# via
# adlfs
# osdu-core-lib-python-azure
azure-keyvault==4.1.0
# via osdu-core-lib-python-azure
azure-keyvault-certificates==4.3.0
# via azure-keyvault
azure-keyvault-keys==4.4.0
# via azure-keyvault
azure-keyvault-secrets==4.3.0
# via azure-keyvault
azure-storage-blob==12.8.1
# via
# adlfs
# osdu-core-lib-python-azure
backoff==1.11.1
# via -r requirements.in
boto3==1.18.1
# via osdu-core-lib-python-ibm
botocore==1.21.1
# via
# boto3
# osdu-core-lib-python-ibm
# s3transfer
cachetools==4.2.2
# via
# google-auth
# osdu-core-lib-python-azure
certifi==2021.5.30
# via
# httpx
# msrest
# requests
cffi==1.14.6
# via
# azure-datalake-store
# cryptography
chardet==4.0.0
# via aiohttp
charset-normalizer==2.0.3
# via requests
click==8.0.1
# via
# -r requirements.in
# distributed
# uvicorn
cloudpickle==1.6.0
# via
# dask
# distributed
cryptography==3.4.7
# via
# -r requirements.in
# adal
# azure-identity
# azure-keyvault-keys
# azure-storage-blob
# msal
# osdu-core-lib-python
# osdu-core-lib-python-gcp
# pyjwt
dask[distributed]==2021.7.2
fsspec
python-ulid
--extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
osdu-log-recognition-lib>=0.0.9
osdu-data-ecosystem-storage~=1.2.0
osdu-data-ecosystem-search>=0.3.2, <0.4
# via
# -r requirements.in
# distributed
decorator==5.0.9
# via
# gcsfs
# jsonpath-ng
distributed==2021.07.2
# via dask
fastapi==0.65.1
# via
# -r requirements.in
# osdu-data-ecosystem-search
# osdu-data-ecosystem-storage
fsspec==2021.7.0
# via
# -r requirements.in
# adlfs
# dask
# gcsfs
# osdu-core-lib-python-azure
gcsfs==2021.7.0
# via osdu-core-lib-python-gcp
google-api-core[grpc]==1.31.0
# via
# google-cloud-core
# google-cloud-monitoring
# google-cloud-trace
# opencensus
google-auth==1.33.0
# via
# gcsfs
# google-api-core
# google-auth-oauthlib
# google-cloud-core
google-auth-oauthlib==0.4.4
# via gcsfs
google-cloud-core==1.7.1
# via google-cloud-trace
google-cloud-monitoring==0.36.0
# via opencensus-ext-stackdriver
google-cloud-trace==0.24.0
# via opencensus-ext-stackdriver
googleapis-common-protos==1.53.0
# via google-api-core
grpcio==1.38.1
# via
# google-api-core
# opencensus-ext-ocagent
# opencensus-proto
h11==0.12.0
# via
# httpcore
# uvicorn
heapdict==1.0.1
# via zict
hiredis==2.0.0
# via aioredis
httpcore==0.13.6
# via httpx
httpx==0.18.2
# via
# osdu-data-ecosystem-search
# osdu-data-ecosystem-storage
idna==3.2
# via
# anyio
# requests
# rfc3986
# yarl
importlib-metadata==4.6.3
# via click
isodate==0.6.0
# via msrest
jmespath==0.10.0
# via
# boto3
# botocore
jsonbender==0.9.3
# via -r requirements.in
jsonpath-ng==1.5.3
# via -r requirements.in
locket==0.2.1
# via partd
msal==1.12.0
# via
# azure-identity
# msal-extensions
msal-extensions==0.3.0
# via azure-identity
msgpack==1.0.2
# via distributed
msrest==0.6.21
# via
# azure-keyvault-certificates
# azure-keyvault-keys
# azure-keyvault-secrets
# azure-storage-blob
multidict==5.1.0
# via
# aiohttp
# yarl
natsort==7.1.1
# via -r requirements.in
numpy==1.21.1
# via
# pandas
# pyarrow
oauthlib==3.1.1
# via requests-oauthlib
opencensus==0.7.13
# via
# -r requirements.in
# opencensus-ext-azure
# opencensus-ext-logging