dev(narugo): add docs for tar indexing

docs/source/api_doc/index/fetch.rst

@@ -0,0 +1,51 @@
+hfutils.index.fetch
+================================
+
+.. currentmodule:: hfutils.index.fetch
+
+.. automodule:: hfutils.index.fetch
+
+
+ArchiveStandaloneFileIncompleteDownload
+----------------------------------------------
+
+.. autoclass:: ArchiveStandaloneFileIncompleteDownload
+
+
+
+ArchiveStandaloneFileHashNotMatch
+----------------------------------------------
+
+.. autoclass:: ArchiveStandaloneFileHashNotMatch
+
+
+
+hf_tar_list_files
+----------------------------------------------
+
+.. autofunction:: hf_tar_list_files
+
+
+
+hf_tar_file_download
+----------------------------------------------
+
+.. autofunction:: hf_tar_file_download
+
+
+
+hf_tar_get_index
+----------------------------------------------
+
+.. autofunction:: hf_tar_get_index
+
+
+
+hf_tar_file_exists
+----------------------------------------------
+
+.. autofunction:: hf_tar_file_exists
+
+
+
+

docs/source/api_doc/index/index.rst

@@ -0,0 +1,14 @@
+hfutils.index
+================================
+
+.. currentmodule:: hfutils.index
+
+.. automodule:: hfutils.index
+
+
+.. toctree::
+    :maxdepth: 3
+
+    fetch
+    make
+

docs/source/api_doc/index/make.rst

@@ -0,0 +1,36 @@
+hfutils.index.make
+================================
+
+.. currentmodule:: hfutils.index.make
+
+.. automodule:: hfutils.index.make
+
+
+tar_create_index
+----------------------------------------------
+
+.. autofunction:: tar_create_index
+
+
+
+hf_tar_create_index
+----------------------------------------------
+
+.. autofunction:: hf_tar_create_index
+
+
+
+tar_get_index_info
+----------------------------------------------
+
+.. autofunction:: tar_get_index_info
+
+
+
+hf_tar_create_from_directory
+----------------------------------------------
+
+.. autofunction:: hf_tar_create_from_directory
+
+
+

docs/source/index.rst

@@ -30,6 +30,7 @@ Overview
     api_doc/archive/index
     api_doc/config/index
     api_doc/entry/index
+    api_doc/index/index
     api_doc/operate/index
     api_doc/utils/index
 

hfutils/index/fetch.py

@@ -10,18 +10,46 @@
 
 
 class ArchiveStandaloneFileIncompleteDownload(Exception):
-    pass
+    """
+    Exception raised when a standalone file in an archive is incompletely downloaded.
+    """
 
 
 class ArchiveStandaloneFileHashNotMatch(Exception):
-    pass
+    """
+    Exception raised when the hash of a standalone file in an archive does not match.
+    """
 
 
 def hf_tar_get_index(repo_id: str, archive_in_repo: str,
                      repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
                      idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
                      idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
                      hf_token: Optional[str] = None):
+    """
+    Get the index of a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: The index of the tar archive file.
+    :rtype: Dict
+    """
     hf_client = get_hf_client(hf_token)
     body, _ = os.path.splitext(archive_in_repo)
     default_index_file = f'{body}.json'
@@ -39,6 +67,30 @@ def hf_tar_list_files(repo_id: str, archive_in_repo: str,
                       idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
                       idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
                       hf_token: Optional[str] = None):
+    """
+    List files inside a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: The list of files inside the tar archive.
+    :rtype: List[str]
+    """
     index_data = hf_tar_get_index(
         repo_id=repo_id,
         archive_in_repo=archive_in_repo,
@@ -61,6 +113,32 @@ def hf_tar_file_exists(repo_id: str, archive_in_repo: str, file_in_archive: str,
                        idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
                        idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
                        hf_token: Optional[str] = None):
+    """
+    Check if a file exists inside a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param file_in_archive: The path to the file inside the archive.
+    :type file_in_archive: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: True if the file exists, False otherwise.
+    :rtype: bool
+    """
     index = hf_tar_get_index(
         repo_id=repo_id,
         archive_in_repo=archive_in_repo,
@@ -79,10 +157,26 @@ def hf_tar_file_exists(repo_id: str, archive_in_repo: str, file_in_archive: str,
 
 
 def _n_path(path):
+    """
+    Normalize a file path.
+
+    :param path: The file path to normalize.
+    :type path: str
+    :return: The normalized file path.
+    :rtype: str
+    """
     return os.path.normpath(os.path.join('/', path))
 
 
 def _hf_files_process(files: Dict[str, dict]):
+    """
+    Normalize file paths in a dictionary of files.
+
+    :param files: The dictionary of files.
+    :type files: Dict[str, dict]
+    :return: The dictionary of files with normalized paths.
+    :rtype: Dict[str, dict]
+    """
     return {_n_path(key): value for key, value in files.items()}
 
 
@@ -93,6 +187,40 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
                          proxies: Optional[Dict] = None, user_agent: Union[Dict, str, None] = None,
                          headers: Optional[Dict[str, str]] = None, endpoint: Optional[str] = None,
                          hf_token: Optional[str] = None):
+    """
+    Download a file from a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param file_in_archive: The path to the file inside the archive.
+    :type file_in_archive: str
+    :param local_file: The path to save the downloaded file locally.
+    :type local_file: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param proxies: The proxies to be used for the HTTP request.
+    :type proxies: Dict, optional
+    :param user_agent: The user agent for the HTTP request.
+    :type user_agent: Union[Dict, str, None], optional
+    :param headers: The additional headers for the HTTP request.
+    :type headers: Dict[str, str], optional
+    :param endpoint: The Hugging Face API endpoint.
+    :type endpoint: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    """
     index = hf_tar_get_index(
         repo_id=repo_id,
         archive_in_repo=archive_in_repo,

hfutils/index/hash.py

@@ -2,6 +2,16 @@
 
 
 def _f_sha256(file, chunk_for_hash: int = 1 << 20):
+    """
+    Calculate the SHA-256 hash of a file.
+
+    :param file: The path to the file.
+    :type file: str
+    :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).
+    :type chunk_for_hash: int, optional
+    :return: The SHA-256 hash of the file.
+    :rtype: str
+    """
     file_sha = sha256()
     with open(file, 'rb') as f:
         while True: