Commit 3ced63c4 authored by Anuj Gupta's avatar Anuj Gupta
Browse files

Update storage_service.py

parent f45b5349
Pipeline #44141 passed with stage
in 15 seconds
......@@ -22,7 +22,27 @@ from botocore.exceptions import ClientError
from sdlib.api.dataset import Dataset
from sdlib.api.storage_service import StorageFactory, StorageService
from boto3.s3.transfer import TransferConfig
import threading
import sys
class ProgressPercentage(object):
def __init__(self, filename):
self._filename = filename
self._size = float(os.path.getsize(filename))
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify we'll assume this is hooked up
# to a single filename.
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / self._size) * 100
sys.stdout.write(
"\r%s %s / %s (%.2f%%)" % (
self._filename, self._seen_so_far, self._size,
percentage))
sys.stdout.flush()
@StorageFactory.register(provider="ibm")
class IbmStorageService(StorageService):
......@@ -70,19 +90,22 @@ class IbmStorageService(StorageService):
except ClientError as err:
print(err)
config = TransferConfig(multipart_threshold=1024 * 2,
config = TransferConfig(multipart_threshold=1024 * 50,
max_concurrency=10,
multipart_chunksize=1024 * 2,
multipart_chunksize=1024 * 50,
use_threads=True)
transfer = S3Transfer(self._s3_client, config)
# transfer = S3Transfer(self._s3_client, config)
reponse = self._s3_client.upload_file(Filename=file_name, Bucket=bucket_name,
Key=object_name, Config=config,
Callback=ProgressPercentage(file_name))
bar_format = '- Uploading Data [ {percentage:3.0f}% |{bar}| {n_fmt}/{total_fmt} - {elapsed}|{remaining} - {rate_fmt}{postfix} ]'
# bar_format = '- Uploading Data [ {percentage:3.0f}% |{bar}| {n_fmt}/{total_fmt} - {elapsed}|{remaining} - {rate_fmt}{postfix} ]'
# Upload the file
with tqdm.tqdm(total=os.path.getsize(file_name), bar_format=bar_format, unit='B', unit_scale=True,
unit_divisor=1024) as pbar:
transfer.upload_file(file_name, bucket_name, object_name, callback=IbmStorageService._progress_hook(pbar))
# with tqdm.tqdm(total=os.path.getsize(file_name), bar_format=bar_format, unit='B', unit_scale=True,
# unit_divisor=1024) as pbar:
# transfer.upload_file(file_name, bucket_name, object_name, callback=IbmStorageService._progress_hook(pbar))
return True
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment