import math
import os.path
import re
+import time
from typing import Optional, List
+from hbutils.string import plural_word
from huggingface_hub import CommitOperationAdd, CommitOperationDelete
from .base import RepoTypeTyping, get_hf_client, list_files_in_repository, _IGNORE_PATTERN_UNSET
from ..archive import get_archive_type, archive_pack
-from ..utils import walk_files, TemporaryDirectory
+from ..utils import walk_files, TemporaryDirectory, tqdm
[docs]def upload_file_to_file(local_file, repo_id: str, file_in_repo: str,
@@ -176,11 +178,14 @@ Source code for hfutils.operate.upload
_PATH_SEP = re.compile(r'[/\\]+')
-[docs]def upload_directory_as_directory(local_directory, repo_id: str, path_in_repo: str,
- repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
- message: Optional[str] = None, time_suffix: bool = True,
- clear: bool = False, ignore_patterns: List[str] = _IGNORE_PATTERN_UNSET,
- hf_token: Optional[str] = None, operation_chunk_size: Optional[int] = None):
+[docs]def upload_directory_as_directory(
+ local_directory, repo_id: str, path_in_repo: str,
+ repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+ message: Optional[str] = None, time_suffix: bool = True,
+ clear: bool = False, ignore_patterns: List[str] = _IGNORE_PATTERN_UNSET,
+ hf_token: Optional[str] = None, operation_chunk_size: Optional[int] = None,
+ upload_timespan: float = 5.0,
+):
"""
Upload a local directory and its files to a specified path in a Hugging Face repository.
@@ -207,6 +212,8 @@ Source code for hfutils.operate.upload
:param operation_chunk_size: Chunk size of the operations. All the operations will be
seperated into multiple commits when this is set.
:type operation_chunk_size: Optional[int]
+ :param upload_timespan: Upload minimal time interval when chunked uploading enabled.
+ :type upload_timespan: float
.. note::
When `operation_chunk_size` is set, multiple commits will be created. When some commits failed,
@@ -215,6 +222,11 @@ Source code for hfutils.operate.upload
.. warning::
When `operation_chunk_size` is set, multiple commits will be created. But HuggingFace's repository
api cannot guarantee the atomic feature of your data. So **this function is not thread-safe**.
+
+ .. note::
+ The rate limit of HuggingFace repository commit creation is approximately 120 commits / hour.
+ So if you really have large number of chunks to create, please set the `upload_timespan` to a value
+ no less than `30.0` to make sure your uploading will not be rate-limited.
"""
hf_client = get_hf_client(hf_token)
if clear:
@@ -252,9 +264,21 @@ Source code for hfutils.operate.upload
revision=revision
)[0].commit_id
+ last_upload_at = None
try:
- for chunk_id in range(int(math.ceil(len(operations) / operation_chunk_size))):
+ for chunk_id in tqdm(range(int(math.ceil(len(operations) / operation_chunk_size))),
+ desc='Chunked Commits'):
operation_chunk = operations[chunk_id * operation_chunk_size:(chunk_id + 1) * operation_chunk_size]
+
+ # sleep for the given time
+ if last_upload_at:
+ sleep_time = last_upload_at + upload_timespan - time.time()
+ if sleep_time > 0:
+ logging.info(f'Sleep for {sleep_time:.1f}s due to the timespan limitation ...')
+ time.sleep(sleep_time)
+
+ last_upload_at = time.time()
+ logging.info(f'Uploading chunk #{chunk_id}, with {plural_word(len(operation_chunk), "operation")} ...')
hf_client.create_commit(
repo_id=repo_id,
repo_type=repo_type,
@@ -262,6 +286,8 @@ Source code for hfutils.operate.upload
operations=operation_chunk,
commit_message=f'[Chunk #{chunk_id}] {commit_message}',
)
+
+
except Exception:
from ..repository import hf_hub_rollback
diff --git a/dev/chunkedcommit/_sources/information/environment.result.ipynb.txt b/dev/chunkedcommit/_sources/information/environment.result.ipynb.txt
index 074c31d909..ab58fd2e77 100644
--- a/dev/chunkedcommit/_sources/information/environment.result.ipynb.txt
+++ b/dev/chunkedcommit/_sources/information/environment.result.ipynb.txt
@@ -29,10 +29,10 @@
"execution_count": 1,
"metadata": {
"execution": {
- "iopub.execute_input": "2024-07-21T06:12:48.438134Z",
- "iopub.status.busy": "2024-07-21T06:12:48.437738Z",
- "iopub.status.idle": "2024-07-21T06:12:49.608988Z",
- "shell.execute_reply": "2024-07-21T06:12:49.608237Z"
+ "iopub.execute_input": "2024-07-21T06:31:39.044488Z",
+ "iopub.status.busy": "2024-07-21T06:31:39.043994Z",
+ "iopub.status.idle": "2024-07-21T06:31:40.217926Z",
+ "shell.execute_reply": "2024-07-21T06:31:40.217154Z"
},
"pycharm": {
"name": "#%%\n"
@@ -53,7 +53,7 @@
"text": [
"CPU Brand: AMD EPYC 7763 64-Core Processor\n",
"CPU Count: 4\n",
- "CPU Freq: 2936.629 MHz\n",
+ "CPU Freq: 2921.47025 MHz\n",
"Memory Size: 15.606 GiB\n",
"Has CUDA: No\n"
]
diff --git a/dev/chunkedcommit/api_doc/operate/upload.html b/dev/chunkedcommit/api_doc/operate/upload.html
index 3a857c8ced..38a4d916bc 100644
--- a/dev/chunkedcommit/api_doc/operate/upload.html
+++ b/dev/chunkedcommit/api_doc/operate/upload.html
@@ -169,7 +169,7 @@ upload_directory_as_archive
-
-hfutils.operate.upload.upload_directory_as_directory(local_directory, repo_id: str, path_in_repo: str, repo_type: ~typing.Literal['dataset', 'model', 'space'] = 'dataset', revision: str = 'main', message: str | None = None, time_suffix: bool = True, clear: bool = False, ignore_patterns: ~typing.List[str] = <object object>, hf_token: str | None = None, operation_chunk_size: int | None = None)[source]
+hfutils.operate.upload.upload_directory_as_directory(local_directory, repo_id: str, path_in_repo: str, repo_type: ~typing.Literal['dataset', 'model', 'space'] = 'dataset', revision: str = 'main', message: str | None = None, time_suffix: bool = True, clear: bool = False, ignore_patterns: ~typing.List[str] = <object object>, hf_token: str | None = None, operation_chunk_size: int | None = None, upload_timespan: float = 5.0)[source]
Upload a local directory and its files to a specified path in a Hugging Face repository.
- Parameters:
@@ -186,6 +186,7 @@ upload_directory_as_directoryHF_TOKEN variable if not assigned.
operation_chunk_size (Optional[int]) – Chunk size of the operations. All the operations will be
seperated into multiple commits when this is set.
upload_timespan (float) – Upload minimal time interval when chunked uploading enabled.
@@ -199,6 +200,12 @@ upload_directory_as_directory
+Note
+The rate limit of HuggingFace repository commit creation is approximately 120 commits / hour.
+So if you really have large number of chunks to create, please set the upload_timespan to a value
+no less than 30.0 to make sure your uploading will not be rate-limited.
+
diff --git a/dev/chunkedcommit/information/environment.html b/dev/chunkedcommit/information/environment.html
index 7062987c1a..2f039c1a08 100644
--- a/dev/chunkedcommit/information/environment.html
+++ b/dev/chunkedcommit/information/environment.html
@@ -132,7 +132,7 @@ Run Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment InformationRun Environment Information