dev(narugo): add download cli

hfutils/entry/cli.py

@@ -1,7 +1,8 @@
 from .dispatch import hfutilcli
+from .download import _add_download_subcommand
 
 _DECORATORS = [
-
+    _add_download_subcommand
 ]
 
 cli = hfutilcli

hfutils/entry/download.py

@@ -0,0 +1,80 @@
+import warnings
+from typing import Optional
+
+import click
+
+from .base import CONTEXT_SETTINGS, command_wrap, ClickErrorException
+from ..operate import download_file_to_file, download_archive_as_directory, download_directory_as_directory
+from ..operate.base import REPO_TYPES, RepoTypeTyping
+
+
+class NoRemotePathAssigned(ClickErrorException):
+    exit_code = 0x11
+
+
+def _add_download_subcommand(cli: click.Group) -> click.Group:
+    @cli.command('download', help='Download data from HuggingFace.\n\n'
+                                  'Set environment $HF_TOKEN to use your own access token.',
+                 context_settings=CONTEXT_SETTINGS)
+    @click.option('-r', '--repository', 'repo_id', type=str, required=True,
+                  help='Repository to download from.')
+    @click.option('-t', '--type', 'repo_type', type=click.Choice(REPO_TYPES), default='dataset',
+                  help='Type of the HuggingFace repository.', show_default=True)
+    @click.option('-f', '--filename', 'file_in_repo', type=str, default=None,
+                  help='File in repository to download.')
+    @click.option('-a', '--archive', 'archive_in_repo', type=str, default=None,
+                  help='Archive file in repository to download and extract from')
+    @click.option('-d', '--directory', 'dir_in_repo', type=str, default=None,
+                  help='Directory in repository to download the full directory tree.')
+    @click.option('-o', '--output', 'output_path', type=str, required=True,
+                  help='Output path for download.')
+    @click.option('-R', '--revision', 'revision', type=str, default='main',
+                  help='Revision of repository.', show_default=True)
+    @command_wrap()
+    def download(repo_id: str, repo_type: RepoTypeTyping,
+                 file_in_repo: Optional[str], archive_in_repo: Optional[str], dir_in_repo: Optional[str],
+                 output_path: str, revision: str):
+        if not file_in_repo and not archive_in_repo and not dir_in_repo:
+            raise NoRemotePathAssigned('No remote path in repository assigned.\n'
+                                       'One of the -f, -a or -d option is required.')
+
+        if file_in_repo:
+            if archive_in_repo:
+                warnings.warn('File in repository assigned, value of -a option will be ignored.')
+            if dir_in_repo:
+                warnings.warn('File in repository assigned, value of -d option will be ignored.')
+            download_file_to_file(
+                local_file=output_path,
+                repo_id=repo_id,
+                file_in_repo=file_in_repo,
+                repo_type=repo_type,
+                revision=revision,
+                silent=False,
+            )
+
+        elif archive_in_repo:
+            if dir_in_repo:
+                warnings.warn('Archive in repository assigned, value of -d option will be ignored.')
+            download_archive_as_directory(
+                local_directory=output_path,
+                repo_id=repo_id,
+                file_in_repo=archive_in_repo,
+                repo_type=repo_type,
+                revision=revision,
+                silent=False,
+            )
+
+        elif dir_in_repo:
+            download_directory_as_directory(
+                local_directory=output_path,
+                repo_id=repo_id,
+                dir_in_repo=dir_in_repo,
+                repo_type=repo_type,
+                revision=revision,
+                silent=False,
+            )
+
+        else:
+            assert False, 'Should not reach this line, it must be a bug!'  # pragma: no cover
+
+    return cli

hfutils/operate/base.py

@@ -6,6 +6,7 @@
 from huggingface_hub import HfApi, HfFileSystem
 
 RepoTypeTyping = Literal['dataset', 'model', 'space']
+REPO_TYPES = ['dataset', 'model', 'space']
 
 
 @lru_cache()

test/entry/test_dispatch.py

@@ -8,6 +8,6 @@
 @pytest.mark.unittest
 class TestEntryDispatch:
     def test_version(self):
-        result = simulate_entry(hfutilscli, ['hfutilscli', '-v'])
+        result = simulate_entry(hfutilscli, ['hfutils', '-v'])
         assert result.exitcode == 0
         assert __VERSION__ in result.stdout

test/entry/test_download.py

@@ -0,0 +1,52 @@
+import pytest
+from hbutils.testing import simulate_entry, isolated_directory
+
+from hfutils.entry import hfutilscli
+from test.testings import get_testfile, file_compare, dir_compare
+
+
+@pytest.mark.unittest
+class TestEntryDownload:
+    def test_download_file_to_file(self):
+        target_file = get_testfile('mashu.png')
+        with isolated_directory():
+            result = simulate_entry(hfutilscli, [
+                'hfutils', 'download',
+                '-r', 'deepghs/game_character_skins',
+                '-f', 'fgo/1/常夏的泳装Ver_02.png',
+                '-o', 'mashu_download.png'
+            ])
+            assert result.exitcode == 0
+            file_compare(target_file, 'mashu_download.png')
+
+    def test_download_archive_as_directory(self):
+        target_dir = get_testfile('surtr_ds')
+        with isolated_directory():
+            result = simulate_entry(hfutilscli, [
+                'hfutils', 'download',
+                '-r', 'narugo/test_ds_repo',
+                '-a', 'surtr_dataset.zip',
+                '-o', 'download_dir'
+            ])
+            assert result.exitcode == 0
+            dir_compare(target_dir, 'download_dir')
+
+    def test_download_directory_as_directory(self):
+        target_dir = get_testfile('skin_mashu')
+        with isolated_directory():
+            result = simulate_entry(hfutilscli, [
+                'hfutils', 'download',
+                '-r', 'deepghs/game_character_skins',
+                '-d', 'fgo/1',
+                '-o', 'download_dir'
+            ])
+            assert result.exitcode == 0
+            dir_compare(target_dir, 'download_dir')
+
+    def test_no_assign(self):
+        result = simulate_entry(hfutilscli, [
+            'hfutils', 'download',
+            '-r', 'deepghs/game_character_skins',
+            '-o', 'download_dir'
+        ])
+        assert result.exitcode == 0x11