Skip to content

Commit

Permalink
Updating pyuda get_file to download files as 'opaque' type as to down…
Browse files Browse the repository at this point in the history
…load in multiple chunks with a progress bar.
  • Loading branch information
jholloc committed Dec 16, 2024
1 parent 46e236f commit c1d4efc
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 9 deletions.
2 changes: 1 addition & 1 deletion source/wrappers/python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name = "uda"
dynamic = ["version"]
readme = {file = 'README.md', content-type = "text/markdown"}
license = {text = "Apache-2.0 license"}
dependencies = ["numpy>1.7, <2", "six"]
dependencies = ["numpy>1.7, <2", "six", "progress"]
requires-python = ">= 3.5"

classifiers = [
Expand Down
36 changes: 28 additions & 8 deletions source/wrappers/python/pyuda/_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@

from six import with_metaclass
import logging
from collections import namedtuple
import math
from progress.bar import Bar
from collections.abc import Iterable
import sys
try:
Expand Down Expand Up @@ -85,18 +86,37 @@ def __init__(self, debug_level=logging.ERROR):
except ImportError:
pass

def get_file(self, source_file, output_file=None):
def get_file(self, source_file, output_file=None, chunk_size=1):
"""
Retrieve file using bytes plugin and write to file
:param source_file: the full path to the file
:param output_file: the name of the output file
:param str source_file: the full path to the file
:param str|None output_file: the name of the output file
:param int chunk_size: download chunk size in MB, set to 0 to download the file in one chunk
:return:
"""
if chunk_size < 0:
raise ValueError("chunk_size must not be negative")

if chunk_size:
result = cpyuda.get_data(f"bytes::size(path={source_file})", "")
size = result.data()
chunk_size = int(chunk_size * 1024 * 1024)
count = 0
steps = math.ceil(size / chunk_size)
bar = Bar('Downloading', max=steps, suffix='%(percent)d%%')
with open(output_file, 'wb') as f_out:
while count < size:
result = cpyuda.get_data(f"bytes::read(path={source_file}, max_bytes={chunk_size}, offset={count}, /opaque)", "")
data = result.data()
count += data.size
data.tofile(f_out)
bar.next()
print(flush=True)
else:
result = cpyuda.get_data(f"bytes::read(path={source_file}, /opaque)", "")

result = cpyuda.get_data("bytes::read(path=%s, /opaque)" % source_file, "")

with open(output_file, 'wb') as f_out:
result.data().tofile(f_out)
with open(output_file, 'wb') as f_out:
result.data().tofile(f_out)

return

Expand Down

0 comments on commit c1d4efc

Please sign in to comment.