Skip to content

Commit

Permalink
Add comments
Browse files Browse the repository at this point in the history
  • Loading branch information
hanouticelina committed Jan 3, 2025
1 parent 0738730 commit aa493eb
Showing 1 changed file with 5 additions and 0 deletions.
5 changes: 5 additions & 0 deletions src/huggingface_hub/_commit_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,7 @@ def _fetch_files_to_copy(
files_to_copy: Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]] = {}
# Store (path, revision) -> oid mapping
oid_info: Dict[Tuple[str, Optional[str]], Optional[str]] = {}
# 1. Fetch OIDs for destination paths in batches.
dest_paths = [op.path_in_repo for op in copies]
for offset in range(0, len(dest_paths), FETCH_LFS_BATCH_SIZE):
dest_repo_files = hf_api.get_paths_info(
Expand All @@ -615,6 +616,7 @@ def _fetch_files_to_copy(
if not isinstance(file, RepoFolder):
oid_info[(file.path, None)] = file.blob_id

# 2. Group by source revision and fetch source file info in batches.
for src_revision, operations in groupby(copies, key=lambda op: op.src_revision):
operations = list(operations) # type: ignore
src_paths = [op.src_path_in_repo for op in operations]
Expand All @@ -630,6 +632,7 @@ def _fetch_files_to_copy(
if isinstance(src_repo_file, RepoFolder):
raise NotImplementedError("Copying a folder is not implemented.")
oid_info[(src_repo_file.path, src_revision)] = src_repo_file.blob_id
# If it's an LFS file, store the RepoFile object. Otherwise, download raw bytes.
if src_repo_file.lfs:
files_to_copy[(src_repo_file.path, src_revision)] = src_repo_file
else:
Expand All @@ -644,6 +647,8 @@ def _fetch_files_to_copy(
response = get_session().get(url, headers=headers)
hf_raise_for_status(response)
files_to_copy[(src_repo_file.path, src_revision)] = response.content
# 3. Ensure all operations found a corresponding file in the Hub
# and track src/dest OIDs for each operation.
for operation in operations:
if (operation.src_path_in_repo, src_revision) not in files_to_copy:
raise EntryNotFoundError(
Expand Down

0 comments on commit aa493eb

Please sign in to comment.