From f59675964bbad0550064f829b111d88c8a961682 Mon Sep 17 00:00:00 2001 From: jeremie Date: Mon, 28 Jun 2021 14:14:43 +0200 Subject: [PATCH 1/7] local fs storage enhancements --- .../dask/dask_bulk_storage.py | 10 +-- app/clients/storage_service_blob_storage.py | 74 +++++++++++++------ frozenrequirements.txt | 2 +- requirements.txt | 2 +- requirements_dev.txt | 1 + .../routers/ddms_v2/common_ddms_v2_test.py | 2 +- tests/unit/routers/ddms_v2/log_data_test.py | 2 +- .../routers/ddms_v3/common_ddms_v3_test.py | 2 +- .../trajectory/trajectory_ddms_v2_test.py | 2 +- 9 files changed, 62 insertions(+), 35 deletions(-) diff --git a/app/bulk_persistence/dask/dask_bulk_storage.py b/app/bulk_persistence/dask/dask_bulk_storage.py index 86448b68..9564aab6 100644 --- a/app/bulk_persistence/dask/dask_bulk_storage.py +++ b/app/bulk_persistence/dask/dask_bulk_storage.py @@ -13,7 +13,6 @@ # limitations under the License. import asyncio -import base64 import hashlib import json import time @@ -73,6 +72,7 @@ class DefaultWorkerPlugin(WorkerPlugin): exc = self.worker.exceptions[key] getLogger().exception("Task '%s' has failed with exception: %s" % (key, str(exc))) + class DaskBulkStorage: client = None """ Dask client """ @@ -124,10 +124,8 @@ class DaskBulkStorage: await DaskBulkStorage.client.close() # or shutdown DaskBulkStorage.client = None - @staticmethod - def _encode_record_id(record_id: str) -> str: - record_id_b64 = base64.b64encode(record_id.encode()).decode() - return record_id_b64.rstrip('=') # remove padding chars ('=') + def _encode_record_id(self, record_id: str) -> str: + return hashlib.sha1(record_id.encode()).hexdigest() def _get_base_directory(self, protocol=True): return f'{self.protocol}://{self.base_directory}' if protocol else self.base_directory @@ -230,7 +228,7 @@ class DaskBulkStorage: if isinstance(pdf.index, pd.DatetimeIndex): first_idx, last_idx = pdf.index[0].value, pdf.index[-1].value idx_range = f'{first_idx}_{last_idx}' - shape = hashlib.sha256('_'.join(map(str, pdf)).encode()).hexdigest() + shape = hashlib.sha1('_'.join(map(str, pdf)).encode()).hexdigest() t = round(time.time() * 1000) filename = f'{idx_range}_{t}.{shape}' diff --git a/app/clients/storage_service_blob_storage.py b/app/clients/storage_service_blob_storage.py index d8c05242..ddf4f52c 100644 --- a/app/clients/storage_service_blob_storage.py +++ b/app/clients/storage_service_blob_storage.py @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from asyncio import iscoroutinefunction, gather import uuid -from fastapi import FastAPI, HTTPException, status -from osdu.core.api.storage.tenant import Tenant +from asyncio import gather, iscoroutinefunction +from app.model import model_utils +from fastapi import FastAPI, HTTPException, status from odes_storage.models import * from osdu.core.api.storage.blob_storage_base import BlobStorageBase from osdu.core.api.storage.exceptions import ResourceNotFoundException - -from app.model import model_utils +from osdu.core.api.storage.tenant import Tenant +from ulid import ULID async def no_check_appkey_token(appkey, token): @@ -60,12 +60,36 @@ class StorageRecordServiceBlobStorage: self._container: str = container self._auth_check = auth_check_coro - def _build_record_path(self, id: str, data_partition: str): - return f'{data_partition or "global"}_r_{id.replace(":", "_")}' + # def _build_record_path(self, id: str, data_partition: str): + # return f'{data_partition or "global"}_r_{id.replace(":", "_")}' + + @staticmethod + def _get_record_folder(id: str, data_partition: str): + #id_b64 = base64.b64encode(id.encode()).decode().rstrip('=') + id_b64 = hash(id) + folder = f'{data_partition or "global"}_r_{id_b64}' + return folder + + async def _get_all_version_object(self, id: str, data_partition: str): + folder = self._get_record_folder(id, data_partition) + tenant = Tenant(project_id=self._project, bucket_name=self._container, data_partition_id=data_partition) + return sorted(await self._storage.list_objects(tenant=tenant, prefix=folder)) + + async def _build_record_path(self, id: str, data_partition: str, version=None): + folder = self._get_record_folder(id, data_partition) + if version: + return f'{folder}/{version}' + objects = await self._get_all_version_object(id, data_partition) + return objects[-1] if objects else None async def _check_auth(self, appkey=None, token=None): await self._auth_check(appkey, token) + @staticmethod + def _get_new_id_for_record(record: Record): + kind, _version = record.kind.rsplit(':', 1) + return kind + ':' + str(uuid.uuid4()) + async def create_or_update_records(self, record: List[Record] = None, data_partition_id: str = None, @@ -77,12 +101,12 @@ class StorageRecordServiceBlobStorage: # insert id if new record for rec in record_list: if rec.id is None: - rec.id = str(uuid.uuid4()) - + rec.id = self._get_new_id_for_record(rec)# str(uuid.uuid4()) + rec.version = int(ULID()) # generate new version -> ulid is sorted that helps us to know the latest version await gather(*[ self._storage.upload( Tenant(project_id=self._project, bucket_name=self._container, data_partition_id=data_partition_id), - self._build_record_path(record.id, data_partition_id), + await self._build_record_path(record.id, data_partition_id, version=rec.version), model_utils.record_to_json(record), content_type='application/json') for record in record_list @@ -93,14 +117,17 @@ class StorageRecordServiceBlobStorage: recordIds=[record.id for record in record_list], skipped_record_ids=[]) - async def get_record(self, + async def get_record_version(self, id: str, + version: int, data_partition_id: str = None, appkey: str = None, token: str = None) -> Record: await self._check_auth(appkey, token) - object_name = self._build_record_path(id, data_partition_id) try: + object_name = await self._build_record_path(id, data_partition_id, version=version) + if object_name is None: + raise ResourceNotFoundException("Item not found") bin_data = await self._storage.download( Tenant(project_id=self._project, bucket_name=self._container, data_partition_id=data_partition_id), object_name) @@ -114,17 +141,18 @@ class StorageRecordServiceBlobStorage: appkey: str = None, token: str = None) -> RecordVersions: # only one version /latest is supported - return RecordVersions(recordId=id, versions=[0]) + objects = await self._get_all_version_object(id, data_partition_id) + versions = [o.split('/')[-1] for o in objects] + return RecordVersions(recordId=id, versions=versions) - async def get_record_version(self, - id: str, - version: int, - data_partition_id: str = None, - attribute: List[str] = None, - appkey: str = None, - token: str = None) -> Record: - # always return the latest - return await self.get_record(id, data_partition_id, appkey, token) + async def get_record(self, + id: str, + data_partition_id: str = None, + attribute: List[str] = None, + appkey: str = None, + token: str = None) -> Record: + # return the latest + return await self.get_record_version(id, None, data_partition_id, appkey, token) async def delete_record(self, id: str, @@ -132,7 +160,7 @@ class StorageRecordServiceBlobStorage: appkey: str = None, token: str = None) -> None: await self._check_auth(appkey, token) - object_name = self._build_record_path(id, data_partition_id) + object_name = await self._build_record_path(id, data_partition_id) # todo delete all versions try: await self._storage.delete( Tenant(project_id=self._project, bucket_name=self._container, data_partition_id=data_partition_id), diff --git a/frozenrequirements.txt b/frozenrequirements.txt index a4ff90f8..99727313 100644 --- a/frozenrequirements.txt +++ b/frozenrequirements.txt @@ -27,7 +27,7 @@ cloudpickle==1.6.0 colorama==0.4.4 coverage==5.5 cryptography==3.4.7 -dask==2021.4.1 +dask[distributed]==2021.6.2 decorator==5.0.9 distributed==2021.4.1 fastapi==0.65.1 diff --git a/requirements.txt b/requirements.txt index 50c79a5a..b5c3cfb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ opencensus-ext-ocagent opencensus-ext-logging # for chunking feature -dask[distributed]==2021.4.1 +dask[distributed]==2021.6.2 fsspec --extra-index-url \ diff --git a/requirements_dev.txt b/requirements_dev.txt index 2336da87..71ab834d 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -7,6 +7,7 @@ httpx numpy pandas pyarrow +python-ulid # Note since 3.8 includes Mock 4.0+. mock>=4.0 diff --git a/tests/unit/routers/ddms_v2/common_ddms_v2_test.py b/tests/unit/routers/ddms_v2/common_ddms_v2_test.py index 168be58d..dac7539e 100644 --- a/tests/unit/routers/ddms_v2/common_ddms_v2_test.py +++ b/tests/unit/routers/ddms_v2/common_ddms_v2_test.py @@ -414,7 +414,7 @@ def test_post_records_successful(client, base_url, record_obj): with mock.patch.object(StorageRecordServiceClientMock, 'create_or_update_records', moc_create_or_update_records): # when - response = client.post(base_url, data=json.dumps(record_dict_list)) + response = client.post(base_url, data=json.dumps(record_dict_list), headers={'content-type': 'application/json'}) # then assert response.status_code == status.HTTP_200_OK diff --git a/tests/unit/routers/ddms_v2/log_data_test.py b/tests/unit/routers/ddms_v2/log_data_test.py index 929f158d..fc650667 100644 --- a/tests/unit/routers/ddms_v2/log_data_test.py +++ b/tests/unit/routers/ddms_v2/log_data_test.py @@ -119,7 +119,7 @@ def client_with_log(client): response = client.get(f"/ddms/v2/logs/{log_id}/versions", headers=headers) assert response.status_code == 200, "GET log data failed" - version_id = response.json()["versions"][0] + version_id = response.json()["versions"][1] yield client, log_id, version_id diff --git a/tests/unit/routers/ddms_v3/common_ddms_v3_test.py b/tests/unit/routers/ddms_v3/common_ddms_v3_test.py index 7f6b6638..a51008a9 100644 --- a/tests/unit/routers/ddms_v3/common_ddms_v3_test.py +++ b/tests/unit/routers/ddms_v3/common_ddms_v3_test.py @@ -130,7 +130,7 @@ def test_post_records_successful(client): test_Wellbores = json.load(f) Wellbore.parse_obj(test_Wellbores[0]) # when - response = client.post(base_url, data=json.dumps(test_Wellbores)) + response = client.post(base_url, data=json.dumps(test_Wellbores), headers={'content-type' : 'application/json'}) # then assert response.status_code == status.HTTP_200_OK diff --git a/tests/unit/routers/trajectory/trajectory_ddms_v2_test.py b/tests/unit/routers/trajectory/trajectory_ddms_v2_test.py index f33d7538..a1f6f310 100644 --- a/tests/unit/routers/trajectory/trajectory_ddms_v2_test.py +++ b/tests/unit/routers/trajectory/trajectory_ddms_v2_test.py @@ -124,7 +124,7 @@ def client_with_log(client): response = client.get(f"/ddms/v2/trajectories/{trajectory_id}/versions", headers=headers) assert response.status_code == 200, "GET log data failed" - version_id = response.json()["versions"][0] + version_id = response.json()["versions"][1] yield client, log_id, version_id -- GitLab From 6874e39c4c0e2e520b71e31dd1f4dd3dcc13b968 Mon Sep 17 00:00:00 2001 From: jeremie Date: Mon, 28 Jun 2021 14:59:12 +0200 Subject: [PATCH 2/7] fix record id generation --- app/clients/storage_service_blob_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/clients/storage_service_blob_storage.py b/app/clients/storage_service_blob_storage.py index ddf4f52c..8019ef85 100644 --- a/app/clients/storage_service_blob_storage.py +++ b/app/clients/storage_service_blob_storage.py @@ -87,8 +87,8 @@ class StorageRecordServiceBlobStorage: @staticmethod def _get_new_id_for_record(record: Record): - kind, _version = record.kind.rsplit(':', 1) - return kind + ':' + str(uuid.uuid4()) + kind = record.kind.split(':') + return ':'.join((kind[0], kind[2], uuid.uuid4().hex)) async def create_or_update_records(self, record: List[Record] = None, -- GitLab From 84aa67bcbdc4b81f7235d0335b7f7f9d344182a5 Mon Sep 17 00:00:00 2001 From: jeremie Date: Mon, 28 Jun 2021 15:03:29 +0200 Subject: [PATCH 3/7] update frozen requirements --- frozenrequirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/frozenrequirements.txt b/frozenrequirements.txt index 99727313..910860f3 100644 --- a/frozenrequirements.txt +++ b/frozenrequirements.txt @@ -29,7 +29,6 @@ coverage==5.5 cryptography==3.4.7 dask[distributed]==2021.6.2 decorator==5.0.9 -distributed==2021.4.1 fastapi==0.65.1 fsspec==2021.6.0 gcsfs==2021.6.0 -- GitLab From 49bc94ae12059e57dfd80b9039ce2ca93f1b710e Mon Sep 17 00:00:00 2001 From: jeremie Date: Tue, 29 Jun 2021 10:38:48 +0200 Subject: [PATCH 4/7] fix delete record --- app/clients/storage_service_blob_storage.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/app/clients/storage_service_blob_storage.py b/app/clients/storage_service_blob_storage.py index 8019ef85..60764448 100644 --- a/app/clients/storage_service_blob_storage.py +++ b/app/clients/storage_service_blob_storage.py @@ -60,12 +60,9 @@ class StorageRecordServiceBlobStorage: self._container: str = container self._auth_check = auth_check_coro - # def _build_record_path(self, id: str, data_partition: str): - # return f'{data_partition or "global"}_r_{id.replace(":", "_")}' @staticmethod def _get_record_folder(id: str, data_partition: str): - #id_b64 = base64.b64encode(id.encode()).decode().rstrip('=') id_b64 = hash(id) folder = f'{data_partition or "global"}_r_{id_b64}' return folder @@ -160,13 +157,13 @@ class StorageRecordServiceBlobStorage: appkey: str = None, token: str = None) -> None: await self._check_auth(appkey, token) - object_name = await self._build_record_path(id, data_partition_id) # todo delete all versions - try: - await self._storage.delete( - Tenant(project_id=self._project, bucket_name=self._container, data_partition_id=data_partition_id), - object_name) - except FileNotFoundError: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Item not found") + for object_name in await self._get_all_version_object(id, data_partition_id): + try: + await self._storage.delete( + Tenant(project_id=self._project, bucket_name=self._container, data_partition_id=data_partition_id), + object_name) + except FileNotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Item not found") async def get_schema(self, kind, data_partition_id=None, appkey=None, token=None, *args, **kwargs): raise NotImplementedError('StorageServiceBlobStorage.get_schema') -- GitLab From 401ed390e4b5df772aec0961940ad0ddc1158d32 Mon Sep 17 00:00:00 2001 From: jeremie Date: Tue, 29 Jun 2021 10:46:00 +0200 Subject: [PATCH 5/7] rename variable --- app/bulk_persistence/dask/dask_bulk_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/bulk_persistence/dask/dask_bulk_storage.py b/app/bulk_persistence/dask/dask_bulk_storage.py index 9564aab6..f5f99243 100644 --- a/app/bulk_persistence/dask/dask_bulk_storage.py +++ b/app/bulk_persistence/dask/dask_bulk_storage.py @@ -132,13 +132,13 @@ class DaskBulkStorage: def _get_blob_path(self, record_id: str, bulk_id: str, with_protocol=True) -> str: """Return the bulk path from the bulk_id.""" - record_id_b64 = self._encode_record_id(record_id) - return f'{self._get_base_directory(with_protocol)}/{record_id_b64}/bulk/{bulk_id}/data' + encoded_id = self._encode_record_id(record_id) + return f'{self._get_base_directory(with_protocol)}/{encoded_id}/bulk/{bulk_id}/data' def _build_path_from_session(self, session: Session, with_protocol=True) -> str: """Return the session path.""" - record_id_b64 = self._encode_record_id(session.recordId) - return f'{self._get_base_directory(with_protocol)}/{record_id_b64}/session/{session.id}/data' + encoded_id = self._encode_record_id(session.recordId) + return f'{self._get_base_directory(with_protocol)}/{encoded_id}/session/{session.id}/data' def _load(self, path, **kwargs) -> dd.DataFrame: """Read a Parquet file into a Dask DataFrame -- GitLab From a68297d2b97029b01b5acfa0041c1d5855923563 Mon Sep 17 00:00:00 2001 From: jeremie Date: Tue, 29 Jun 2021 10:48:10 +0200 Subject: [PATCH 6/7] rename variable --- app/clients/storage_service_blob_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/clients/storage_service_blob_storage.py b/app/clients/storage_service_blob_storage.py index 60764448..bc3967e0 100644 --- a/app/clients/storage_service_blob_storage.py +++ b/app/clients/storage_service_blob_storage.py @@ -63,8 +63,8 @@ class StorageRecordServiceBlobStorage: @staticmethod def _get_record_folder(id: str, data_partition: str): - id_b64 = hash(id) - folder = f'{data_partition or "global"}_r_{id_b64}' + encoded_id = hash(id) + folder = f'{data_partition or "global"}_r_{encoded_id}' return folder async def _get_all_version_object(self, id: str, data_partition: str): -- GitLab From c3915d72dbfead5ca95f99624d1c0cc01e87f14b Mon Sep 17 00:00:00 2001 From: jeremie Date: Wed, 30 Jun 2021 11:40:15 +0200 Subject: [PATCH 7/7] update notice --- NOTICE | 85 +++++++++++++++++----------------------------------------- 1 file changed, 24 insertions(+), 61 deletions(-) diff --git a/NOTICE b/NOTICE index 546b5569..e89b9569 100644 --- a/NOTICE +++ b/NOTICE @@ -12,7 +12,6 @@ The following software have components provided under the terms of this license: - boto3 (from https://github.com/boto/boto3) - botocore (from https://github.com/boto/botocore) - coverage (from https://coverage.readthedocs.io) -- cryptography (from https://github.com/pyca/cryptography) - google-api-core (from https://github.com/GoogleCloudPlatform/google-cloud-python) - google-auth (from https://github.com/GoogleCloudPlatform/google-auth-library-python) - google-auth-oauthlib (from https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) @@ -56,14 +55,14 @@ BSD-2-Clause ======================================================================== The following software have components provided under the terms of this license: +- decorator (from https://github.com/micheles/decorator) - grpcio (from http://www.grpc.io) - locket (from http://github.com/mwilliamson/locket.py) - mock (from https://github.com/testing-cabal/mock) - numpy (from http://www.numpy.org) -- ply (from http://www.dabeaz.com/ply/) +- packaging (from https://github.com/pypa/packaging) - pyasn1 (from http://sourceforge.net/projects/pyasn1/) - pyasn1-modules (from http://sourceforge.net/projects/pyasn1/) -- pycparser (from https://github.com/eliben/pycparser) - tblib (from https://github.com/ionelmc/python-tblib) ======================================================================== @@ -77,7 +76,6 @@ The following software have components provided under the terms of this license: - click (from http://github.com/mitsuhiko/click) - cloudpickle (from https://github.com/cloudpipe/cloudpickle) - colorama (from https://github.com/tartley/colorama) -- cryptography (from https://github.com/pyca/cryptography) - dask (from http://github.com/dask/dask/) - decorator (from https://github.com/micheles/decorator) - distributed (from https://distributed.readthedocs.io/en/latest/) @@ -89,26 +87,20 @@ The following software have components provided under the terms of this license: - httpx (from https://github.com/encode/httpx) - idna (from https://github.com/kjd/idna) - isodate (from http://cheeseshop.python.org/pypi/isodate) -- locket (from http://github.com/mwilliamson/locket.py) -- mock (from https://github.com/testing-cabal/mock) - numpy (from http://www.numpy.org) - oauthlib (from https://github.com/idan/oauthlib) -- packaging (from https://github.com/pypa/packaging) - pandas (from http://pandas.pydata.org) - partd (from http://github.com/dask/partd/) - ply (from http://www.dabeaz.com/ply/) - protobuf (from https://developers.google.com/protocol-buffers/) - psutil (from https://github.com/giampaolo/psutil) - pyarrow (from https://arrow.apache.org/) -- pyasn1 (from http://sourceforge.net/projects/pyasn1/) -- pyasn1-modules (from http://sourceforge.net/projects/pyasn1/) - pycparser (from https://github.com/eliben/pycparser) +- pyparsing (from http://pyparsing.wikispaces.com/) - pyrsistent (from http://github.com/tobgu/pyrsistent/) - python-dateutil (from https://dateutil.readthedocs.org) - python-rapidjson (from https://github.com/python-rapidjson/python-rapidjson) -- requests-oauthlib (from https://github.com/requests/requests-oauthlib) - starlette (from https://github.com/encode/starlette) -- tblib (from https://github.com/ionelmc/python-tblib) - toolz (from http://github.com/pytoolz/toolz/) - uvicorn (from https://github.com/tomchristie/uvicorn) - zict (from http://github.com/dask/zict/) @@ -119,35 +111,33 @@ CC-BY-4.0 The following software have components provided under the terms of this license: - adlfs (from https://github.com/hayesgb/adlfs/) -- dask (from http://github.com/dask/dask/) -- distributed (from https://distributed.readthedocs.io/en/latest/) - fsspec (from http://github.com/intake/filesystem_spec) - gcsfs (from https://github.com/dask/gcsfs) - pandas (from http://pandas.pydata.org) -- partd (from http://github.com/dask/partd/) -- toolz (from http://github.com/pytoolz/toolz/) ======================================================================== -CC-BY-SA-3.0 +CC0-1.0 ======================================================================== The following software have components provided under the terms of this license: +- dask (from http://github.com/dask/dask/) +- distributed (from https://distributed.readthedocs.io/en/latest/) - numpy (from http://www.numpy.org) ======================================================================== -CNRI-Python +DOC ======================================================================== The following software have components provided under the terms of this license: -- isodate (from http://cheeseshop.python.org/pypi/isodate) -- ply (from http://www.dabeaz.com/ply/) +- dask (from http://github.com/dask/dask/) +- distributed (from https://distributed.readthedocs.io/en/latest/) +- numpy (from http://www.numpy.org) ======================================================================== GPL-2.0-only ======================================================================== The following software have components provided under the terms of this license: -- coverage (from https://coverage.readthedocs.io) - grpcio (from http://www.grpc.io) ======================================================================== @@ -156,22 +146,28 @@ GPL-2.0-or-later The following software have components provided under the terms of this license: - grpcio (from http://www.grpc.io) +- pyparsing (from http://pyparsing.wikispaces.com/) ======================================================================== GPL-3.0-only ======================================================================== The following software have components provided under the terms of this license: -- coverage (from https://coverage.readthedocs.io) - grpcio (from http://www.grpc.io) + +======================================================================== +GPL-3.0-or-later +======================================================================== +The following software have components provided under the terms of this license: + - pyparsing (from http://pyparsing.wikispaces.com/) -- rfc3986 (from https://rfc3986.readthedocs.org) ======================================================================== ISC ======================================================================== The following software have components provided under the terms of this license: +- click (from http://github.com/mitsuhiko/click) - grpcio (from http://www.grpc.io) - requests-oauthlib (from https://github.com/requests/requests-oauthlib) @@ -196,21 +192,6 @@ The following software have components provided under the terms of this license: - chardet (from https://github.com/chardet/chardet) -======================================================================== -LGPL-2.1-or-later -======================================================================== -The following software have components provided under the terms of this license: - -- chardet (from https://github.com/chardet/chardet) - -======================================================================== -LGPL-3.0-only -======================================================================== -The following software have components provided under the terms of this license: - -- chardet (from https://github.com/chardet/chardet) -- pycparser (from https://github.com/eliben/pycparser) - ======================================================================== MIT ======================================================================== @@ -237,6 +218,7 @@ The following software have components provided under the terms of this license: - cachetools (from https://github.com/tkem/cachetools) - cffi (from http://cffi.readthedocs.org) - coverage (from https://coverage.readthedocs.io) +- distributed (from https://distributed.readthedocs.io/en/latest/) - fastapi (from https://github.com/tiangolo/fastapi) - grpcio (from http://www.grpc.io) - h11 (from https://github.com/python-hyper/h11) @@ -244,7 +226,6 @@ The following software have components provided under the terms of this license: - jmespath (from https://github.com/jmespath/jmespath.py) - jsonschema (from http://github.com/Julian/jsonschema) - msal (from https://github.com/AzureAD/microsoft-authentication-library-for-python) -- msal-extensions (from https://pypi.org/project/msal-extensions/0.1.3/) - msrest (from https://github.com/Azure/msrest-for-python) - munch (from http://github.com/Infinidat/munch) - numpy (from http://www.numpy.org) @@ -262,7 +243,6 @@ The following software have components provided under the terms of this license: - python-rapidjson (from https://github.com/python-rapidjson/python-rapidjson) - python-ulid (from https://github.com/mdomke/python-ulid) - pytz (from http://pythonhosted.org/pytz) -- requests-oauthlib (from https://github.com/requests/requests-oauthlib) - six (from http://pypi.python.org/pypi/six/) - sniffio (from https://github.com/python-trio/sniffio) - structlog (from http://www.structlog.org/) @@ -272,21 +252,21 @@ The following software have components provided under the terms of this license: - zipp (from https://github.com/jaraco/zipp) ======================================================================== -MPL-2.0 +MIT-CMU ======================================================================== The following software have components provided under the terms of this license: -- certifi (from http://certifi.io/) +- pyparsing (from http://pyparsing.wikispaces.com/) ======================================================================== -NCSA +MPL-2.0 ======================================================================== The following software have components provided under the terms of this license: -- numpy (from http://www.numpy.org) +- certifi (from http://certifi.io/) ======================================================================== -OPL-1.0 +NCSA ======================================================================== The following software have components provided under the terms of this license: @@ -305,16 +285,9 @@ Python-2.0 The following software have components provided under the terms of this license: - async-timeout (from https://github.com/aio-libs/async_timeout/) -- distributed (from https://distributed.readthedocs.io/en/latest/) - google-auth (from https://github.com/GoogleCloudPlatform/google-auth-library-python) -- google-auth-oauthlib (from https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) -- numpy (from http://www.numpy.org) - pandas (from http://pandas.pydata.org) - portalocker (from https://github.com/WoLpH/portalocker) -- python-dateutil (from https://dateutil.readthedocs.org) -- pytz (from http://pythonhosted.org/pytz) -- rsa (from https://stuvel.eu/rsa) -- sniffio (from https://github.com/python-trio/sniffio) - typing-extensions (from https://github.com/python/typing) - urllib3 (from https://urllib3.readthedocs.io/) @@ -339,20 +312,12 @@ The following software have components provided under the terms of this license: - jsonpath-ng (from https://github.com/h2non/jsonpath-ng) -======================================================================== -ZPL-2.1 -======================================================================== -The following software have components provided under the terms of this license: - -- pytz (from http://pythonhosted.org/pytz) - ======================================================================== Zlib ======================================================================== The following software have components provided under the terms of this license: - grpcio (from http://www.grpc.io) -- numpy (from http://www.numpy.org) ======================================================================== public-domain @@ -361,9 +326,7 @@ The following software have components provided under the terms of this license: - botocore (from https://github.com/boto/botocore) - grpcio (from http://www.grpc.io) -- numpy (from http://www.numpy.org) - pandas (from http://pandas.pydata.org) -- py (from http://pylib.readthedocs.org/) - pytz (from http://pythonhosted.org/pytz) -- GitLab