aboutsummaryrefslogtreecommitdiff
path: root/.venv/lib/python3.12/site-packages/huggingface_hub/commands
diff options
context:
space:
mode:
authorS. Solomon Darnell2025-03-28 21:52:21 -0500
committerS. Solomon Darnell2025-03-28 21:52:21 -0500
commit4a52a71956a8d46fcb7294ac71734504bb09bcc2 (patch)
treeee3dc5af3b6313e921cd920906356f5d4febc4ed /.venv/lib/python3.12/site-packages/huggingface_hub/commands
parentcc961e04ba734dd72309fb548a2f97d67d578813 (diff)
downloadgn-ai-master.tar.gz
two version of R2R are hereHEADmaster
Diffstat (limited to '.venv/lib/python3.12/site-packages/huggingface_hub/commands')
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/__init__.py27
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/_cli_utils.py69
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/delete_cache.py428
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/download.py200
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/env.py36
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/huggingface_cli.py61
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/lfs.py200
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo_files.py128
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/scan_cache.py181
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/tag.py159
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload.py299
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py129
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/user.py304
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/version.py37
14 files changed, 2258 insertions, 0 deletions
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/__init__.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/__init__.py
new file mode 100644
index 00000000..49d08821
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/__init__.py
@@ -0,0 +1,27 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from argparse import _SubParsersAction
+
+
+class BaseHuggingfaceCLICommand(ABC):
+ @staticmethod
+ @abstractmethod
+ def register_subcommand(parser: _SubParsersAction):
+ raise NotImplementedError()
+
+ @abstractmethod
+ def run(self):
+ raise NotImplementedError()
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/_cli_utils.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/_cli_utils.py
new file mode 100644
index 00000000..bd56ad68
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/_cli_utils.py
@@ -0,0 +1,69 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains a utility for good-looking prints."""
+
+import os
+from typing import List, Union
+
+
+class ANSI:
+ """
+ Helper for en.wikipedia.org/wiki/ANSI_escape_code
+ """
+
+ _bold = "\u001b[1m"
+ _gray = "\u001b[90m"
+ _red = "\u001b[31m"
+ _reset = "\u001b[0m"
+ _yellow = "\u001b[33m"
+
+ @classmethod
+ def bold(cls, s: str) -> str:
+ return cls._format(s, cls._bold)
+
+ @classmethod
+ def gray(cls, s: str) -> str:
+ return cls._format(s, cls._gray)
+
+ @classmethod
+ def red(cls, s: str) -> str:
+ return cls._format(s, cls._bold + cls._red)
+
+ @classmethod
+ def yellow(cls, s: str) -> str:
+ return cls._format(s, cls._yellow)
+
+ @classmethod
+ def _format(cls, s: str, code: str) -> str:
+ if os.environ.get("NO_COLOR"):
+ # See https://no-color.org/
+ return s
+ return f"{code}{s}{cls._reset}"
+
+
+def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
+ """
+ Inspired by:
+
+ - stackoverflow.com/a/8356620/593036
+ - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
+ """
+ col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)]
+ row_format = ("{{:{}}} " * len(headers)).format(*col_widths)
+ lines = []
+ lines.append(row_format.format(*headers))
+ lines.append(row_format.format(*["-" * w for w in col_widths]))
+ for row in rows:
+ lines.append(row_format.format(*row))
+ return "\n".join(lines)
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/delete_cache.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/delete_cache.py
new file mode 100644
index 00000000..b2fc44d3
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/delete_cache.py
@@ -0,0 +1,428 @@
+# coding=utf-8
+# Copyright 2022-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to delete some revisions from the HF cache directory.
+
+Usage:
+ huggingface-cli delete-cache
+ huggingface-cli delete-cache --disable-tui
+ huggingface-cli delete-cache --dir ~/.cache/huggingface/hub
+
+NOTE:
+ This command is based on `InquirerPy` to build the multiselect menu in the terminal.
+ This dependency has to be installed with `pip install huggingface_hub[cli]`. Since
+ we want to avoid as much as possible cross-platform issues, I chose a library that
+ is built on top of `python-prompt-toolkit` which seems to be a reference in terminal
+ GUI (actively maintained on both Unix and Windows, 7.9k stars).
+
+ For the moment, the TUI feature is in beta.
+
+ See:
+ - https://github.com/kazhala/InquirerPy
+ - https://inquirerpy.readthedocs.io/en/latest/
+ - https://github.com/prompt-toolkit/python-prompt-toolkit
+
+ Other solutions could have been:
+ - `simple_term_menu`: would be good as well for our use case but some issues suggest
+ that Windows is less supported.
+ See: https://github.com/IngoMeyer441/simple-term-menu
+ - `PyInquirer`: very similar to `InquirerPy` but older and not maintained anymore.
+ In particular, no support of Python3.10.
+ See: https://github.com/CITGuru/PyInquirer
+ - `pick` (or `pickpack`): easy to use and flexible but built on top of Python's
+ standard library `curses` that is specific to Unix (not implemented on Windows).
+ See https://github.com/wong2/pick and https://github.com/anafvana/pickpack.
+ - `inquirer`: lot of traction (700 stars) but explicitly states "experimental
+ support of Windows". Not built on top of `python-prompt-toolkit`.
+ See https://github.com/magmax/python-inquirer
+
+TODO: add support for `huggingface-cli delete-cache aaaaaa bbbbbb cccccc (...)` ?
+TODO: add "--keep-last" arg to delete revisions that are not on `main` ref
+TODO: add "--filter" arg to filter repositories by name ?
+TODO: add "--sort" arg to sort by size ?
+TODO: add "--limit" arg to limit to X repos ?
+TODO: add "-y" arg for immediate deletion ?
+See discussions in https://github.com/huggingface/huggingface_hub/issues/1025.
+"""
+
+import os
+from argparse import Namespace, _SubParsersAction
+from functools import wraps
+from tempfile import mkstemp
+from typing import Any, Callable, Iterable, List, Optional, Union
+
+from ..utils import CachedRepoInfo, CachedRevisionInfo, HFCacheInfo, scan_cache_dir
+from . import BaseHuggingfaceCLICommand
+from ._cli_utils import ANSI
+
+
+try:
+ from InquirerPy import inquirer
+ from InquirerPy.base.control import Choice
+ from InquirerPy.separator import Separator
+
+ _inquirer_py_available = True
+except ImportError:
+ _inquirer_py_available = False
+
+
+def require_inquirer_py(fn: Callable) -> Callable:
+ """Decorator to flag methods that require `InquirerPy`."""
+
+ # TODO: refactor this + imports in a unified pattern across codebase
+ @wraps(fn)
+ def _inner(*args, **kwargs):
+ if not _inquirer_py_available:
+ raise ImportError(
+ "The `delete-cache` command requires extra dependencies to work with"
+ " the TUI.\nPlease run `pip install huggingface_hub[cli]` to install"
+ " them.\nOtherwise, disable TUI using the `--disable-tui` flag."
+ )
+
+ return fn(*args, **kwargs)
+
+ return _inner
+
+
+# Possibility for the user to cancel deletion
+_CANCEL_DELETION_STR = "CANCEL_DELETION"
+
+
+class DeleteCacheCommand(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ delete_cache_parser = parser.add_parser("delete-cache", help="Delete revisions from the cache directory.")
+
+ delete_cache_parser.add_argument(
+ "--dir",
+ type=str,
+ default=None,
+ help="cache directory (optional). Default to the default HuggingFace cache.",
+ )
+
+ delete_cache_parser.add_argument(
+ "--disable-tui",
+ action="store_true",
+ help=(
+ "Disable Terminal User Interface (TUI) mode. Useful if your"
+ " platform/terminal doesn't support the multiselect menu."
+ ),
+ )
+
+ delete_cache_parser.set_defaults(func=DeleteCacheCommand)
+
+ def __init__(self, args: Namespace) -> None:
+ self.cache_dir: Optional[str] = args.dir
+ self.disable_tui: bool = args.disable_tui
+
+ def run(self):
+ """Run `delete-cache` command with or without TUI."""
+ # Scan cache directory
+ hf_cache_info = scan_cache_dir(self.cache_dir)
+
+ # Manual review from the user
+ if self.disable_tui:
+ selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[])
+ else:
+ selected_hashes = _manual_review_tui(hf_cache_info, preselected=[])
+
+ # If deletion is not cancelled
+ if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes:
+ confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?"
+
+ # Confirm deletion
+ if self.disable_tui:
+ confirmed = _ask_for_confirmation_no_tui(confirm_message)
+ else:
+ confirmed = _ask_for_confirmation_tui(confirm_message)
+
+ # Deletion is confirmed
+ if confirmed:
+ strategy = hf_cache_info.delete_revisions(*selected_hashes)
+ print("Start deletion.")
+ strategy.execute()
+ print(
+ f"Done. Deleted {len(strategy.repos)} repo(s) and"
+ f" {len(strategy.snapshots)} revision(s) for a total of"
+ f" {strategy.expected_freed_size_str}."
+ )
+ return
+
+ # Deletion is cancelled
+ print("Deletion is cancelled. Do nothing.")
+
+
+@require_inquirer_py
+def _manual_review_tui(hf_cache_info: HFCacheInfo, preselected: List[str]) -> List[str]:
+ """Ask the user for a manual review of the revisions to delete.
+
+ Displays a multi-select menu in the terminal (TUI).
+ """
+ # Define multiselect list
+ choices = _get_tui_choices_from_scan(repos=hf_cache_info.repos, preselected=preselected)
+ checkbox = inquirer.checkbox(
+ message="Select revisions to delete:",
+ choices=choices, # List of revisions with some pre-selection
+ cycle=False, # No loop between top and bottom
+ height=100, # Large list if possible
+ # We use the instruction to display to the user the expected effect of the
+ # deletion.
+ instruction=_get_expectations_str(
+ hf_cache_info,
+ selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled],
+ ),
+ # We use the long instruction to should keybindings instructions to the user
+ long_instruction="Press <space> to select, <enter> to validate and <ctrl+c> to quit without modification.",
+ # Message that is displayed once the user validates its selection.
+ transformer=lambda result: f"{len(result)} revision(s) selected.",
+ )
+
+ # Add a callback to update the information line when a revision is
+ # selected/unselected
+ def _update_expectations(_) -> None:
+ # Hacky way to dynamically set an instruction message to the checkbox when
+ # a revision hash is selected/unselected.
+ checkbox._instruction = _get_expectations_str(
+ hf_cache_info,
+ selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]],
+ )
+
+ checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations})
+
+ # Finally display the form to the user.
+ try:
+ return checkbox.execute()
+ except KeyboardInterrupt:
+ return [] # Quit without deletion
+
+
+@require_inquirer_py
+def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool:
+ """Ask for confirmation using Inquirer."""
+ return inquirer.confirm(message, default=default).execute()
+
+
+def _get_tui_choices_from_scan(repos: Iterable[CachedRepoInfo], preselected: List[str]) -> List:
+ """Build a list of choices from the scanned repos.
+
+ Args:
+ repos (*Iterable[`CachedRepoInfo`]*):
+ List of scanned repos on which we want to delete revisions.
+ preselected (*List[`str`]*):
+ List of revision hashes that will be preselected.
+
+ Return:
+ The list of choices to pass to `inquirer.checkbox`.
+ """
+ choices: List[Union[Choice, Separator]] = []
+
+ # First choice is to cancel the deletion. If selected, nothing will be deleted,
+ # no matter the other selected items.
+ choices.append(
+ Choice(
+ _CANCEL_DELETION_STR,
+ name="None of the following (if selected, nothing will be deleted).",
+ enabled=False,
+ )
+ )
+
+ # Display a separator per repo and a Choice for each revisions of the repo
+ for repo in sorted(repos, key=_repo_sorting_order):
+ # Repo as separator
+ choices.append(
+ Separator(
+ f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str},"
+ f" used {repo.last_accessed_str})"
+ )
+ )
+ for revision in sorted(repo.revisions, key=_revision_sorting_order):
+ # Revision as choice
+ choices.append(
+ Choice(
+ revision.commit_hash,
+ name=(
+ f"{revision.commit_hash[:8]}:"
+ f" {', '.join(sorted(revision.refs)) or '(detached)'} #"
+ f" modified {revision.last_modified_str}"
+ ),
+ enabled=revision.commit_hash in preselected,
+ )
+ )
+
+ # Return choices
+ return choices
+
+
+def _manual_review_no_tui(hf_cache_info: HFCacheInfo, preselected: List[str]) -> List[str]:
+ """Ask the user for a manual review of the revisions to delete.
+
+ Used when TUI is disabled. Manual review happens in a separate tmp file that the
+ user can manually edit.
+ """
+ # 1. Generate temporary file with delete commands.
+ fd, tmp_path = mkstemp(suffix=".txt") # suffix to make it easier to find by editors
+ os.close(fd)
+
+ lines = []
+ for repo in sorted(hf_cache_info.repos, key=_repo_sorting_order):
+ lines.append(
+ f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str},"
+ f" used {repo.last_accessed_str})"
+ )
+ for revision in sorted(repo.revisions, key=_revision_sorting_order):
+ lines.append(
+ # Deselect by prepending a '#'
+ f"{'' if revision.commit_hash in preselected else '#'} "
+ f" {revision.commit_hash} # Refs:"
+ # Print `refs` as comment on same line
+ f" {', '.join(sorted(revision.refs)) or '(detached)'} # modified"
+ # Print `last_modified` as comment on same line
+ f" {revision.last_modified_str}"
+ )
+
+ with open(tmp_path, "w") as f:
+ f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS)
+ f.write("\n".join(lines))
+
+ # 2. Prompt instructions to user.
+ instructions = f"""
+ TUI is disabled. In order to select which revisions you want to delete, please edit
+ the following file using the text editor of your choice. Instructions for manual
+ editing are located at the beginning of the file. Edit the file, save it and confirm
+ to continue.
+ File to edit: {ANSI.bold(tmp_path)}
+ """
+ print("\n".join(line.strip() for line in instructions.strip().split("\n")))
+
+ # 3. Wait for user confirmation.
+ while True:
+ selected_hashes = _read_manual_review_tmp_file(tmp_path)
+ if _ask_for_confirmation_no_tui(
+ _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?",
+ default=False,
+ ):
+ break
+
+ # 4. Return selected_hashes
+ os.remove(tmp_path)
+ return selected_hashes
+
+
+def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool:
+ """Ask for confirmation using pure-python."""
+ YES = ("y", "yes", "1")
+ NO = ("n", "no", "0")
+ DEFAULT = ""
+ ALL = YES + NO + (DEFAULT,)
+ full_message = message + (" (Y/n) " if default else " (y/N) ")
+ while True:
+ answer = input(full_message).lower()
+ if answer == DEFAULT:
+ return default
+ if answer in YES:
+ return True
+ if answer in NO:
+ return False
+ print(f"Invalid input. Must be one of {ALL}")
+
+
+def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: List[str]) -> str:
+ """Format a string to display to the user how much space would be saved.
+
+ Example:
+ ```
+ >>> _get_expectations_str(hf_cache_info, selected_hashes)
+ '7 revisions selected counting for 4.3G.'
+ ```
+ """
+ if _CANCEL_DELETION_STR in selected_hashes:
+ return "Nothing will be deleted."
+ strategy = hf_cache_info.delete_revisions(*selected_hashes)
+ return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}."
+
+
+def _read_manual_review_tmp_file(tmp_path: str) -> List[str]:
+ """Read the manually reviewed instruction file and return a list of revision hash.
+
+ Example:
+ ```txt
+ # This is the tmp file content
+ ###
+
+ # Commented out line
+ 123456789 # revision hash
+
+ # Something else
+ # a_newer_hash # 2 days ago
+ an_older_hash # 3 days ago
+ ```
+
+ ```py
+ >>> _read_manual_review_tmp_file(tmp_path)
+ ['123456789', 'an_older_hash']
+ ```
+ """
+ with open(tmp_path) as f:
+ content = f.read()
+
+ # Split lines
+ lines = [line.strip() for line in content.split("\n")]
+
+ # Filter commented lines
+ selected_lines = [line for line in lines if not line.startswith("#")]
+
+ # Select only before comment
+ selected_hashes = [line.split("#")[0].strip() for line in selected_lines]
+
+ # Return revision hashes
+ return [hash for hash in selected_hashes if len(hash) > 0]
+
+
+_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f"""
+# INSTRUCTIONS
+# ------------
+# This is a temporary file created by running `huggingface-cli delete-cache` with the
+# `--disable-tui` option. It contains a set of revisions that can be deleted from your
+# local cache directory.
+#
+# Please manually review the revisions you want to delete:
+# - Revision hashes can be commented out with '#'.
+# - Only non-commented revisions in this file will be deleted.
+# - Revision hashes that are removed from this file are ignored as well.
+# - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and
+# no changes will be applied.
+#
+# Once you've manually reviewed this file, please confirm deletion in the terminal. This
+# file will be automatically removed once done.
+# ------------
+
+# KILL SWITCH
+# ------------
+# Un-comment following line to completely cancel the deletion process
+# {_CANCEL_DELETION_STR}
+# ------------
+
+# REVISIONS
+# ------------
+""".strip()
+
+
+def _repo_sorting_order(repo: CachedRepoInfo) -> Any:
+ # First split by Dataset/Model, then sort by last accessed (oldest first)
+ return (repo.repo_type, repo.last_accessed)
+
+
+def _revision_sorting_order(revision: CachedRevisionInfo) -> Any:
+ # Sort by last modified (oldest first)
+ return revision.last_modified
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/download.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/download.py
new file mode 100644
index 00000000..10e22c3d
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/download.py
@@ -0,0 +1,200 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to download files from the Hub with the CLI.
+
+Usage:
+ huggingface-cli download --help
+
+ # Download file
+ huggingface-cli download gpt2 config.json
+
+ # Download entire repo
+ huggingface-cli download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78
+
+ # Download repo with filters
+ huggingface-cli download gpt2 --include="*.safetensors"
+
+ # Download with token
+ huggingface-cli download Wauplin/private-model --token=hf_***
+
+ # Download quietly (no progress bar, no warnings, only the returned path)
+ huggingface-cli download gpt2 config.json --quiet
+
+ # Download to local dir
+ huggingface-cli download gpt2 --local-dir=./models/gpt2
+"""
+
+import warnings
+from argparse import Namespace, _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub._snapshot_download import snapshot_download
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.file_download import hf_hub_download
+from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
+
+
+logger = logging.get_logger(__name__)
+
+
+class DownloadCommand(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ download_parser = parser.add_parser("download", help="Download files from the Hub")
+ download_parser.add_argument(
+ "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)."
+ )
+ download_parser.add_argument(
+ "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)."
+ )
+ download_parser.add_argument(
+ "--repo-type",
+ choices=["model", "dataset", "space"],
+ default="model",
+ help="Type of repo to download from (defaults to 'model').",
+ )
+ download_parser.add_argument(
+ "--revision",
+ type=str,
+ help="An optional Git revision id which can be a branch name, a tag, or a commit hash.",
+ )
+ download_parser.add_argument(
+ "--include", nargs="*", type=str, help="Glob patterns to match files to download."
+ )
+ download_parser.add_argument(
+ "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download."
+ )
+ download_parser.add_argument(
+ "--cache-dir", type=str, help="Path to the directory where to save the downloaded files."
+ )
+ download_parser.add_argument(
+ "--local-dir",
+ type=str,
+ help=(
+ "If set, the downloaded file will be placed under this directory. Check out"
+ " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more"
+ " details."
+ ),
+ )
+ download_parser.add_argument(
+ "--local-dir-use-symlinks",
+ choices=["auto", "True", "False"],
+ help=("Deprecated and ignored. Downloading to a local directory does not use symlinks anymore."),
+ )
+ download_parser.add_argument(
+ "--force-download",
+ action="store_true",
+ help="If True, the files will be downloaded even if they are already cached.",
+ )
+ download_parser.add_argument(
+ "--resume-download",
+ action="store_true",
+ help="Deprecated and ignored. Downloading a file to local dir always attempts to resume previously interrupted downloads (unless hf-transfer is enabled).",
+ )
+ download_parser.add_argument(
+ "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
+ )
+ download_parser.add_argument(
+ "--quiet",
+ action="store_true",
+ help="If True, progress bars are disabled and only the path to the download files is printed.",
+ )
+ download_parser.add_argument(
+ "--max-workers",
+ type=int,
+ default=8,
+ help="Maximum number of workers to use for downloading files. Default is 8.",
+ )
+ download_parser.set_defaults(func=DownloadCommand)
+
+ def __init__(self, args: Namespace) -> None:
+ self.token = args.token
+ self.repo_id: str = args.repo_id
+ self.filenames: List[str] = args.filenames
+ self.repo_type: str = args.repo_type
+ self.revision: Optional[str] = args.revision
+ self.include: Optional[List[str]] = args.include
+ self.exclude: Optional[List[str]] = args.exclude
+ self.cache_dir: Optional[str] = args.cache_dir
+ self.local_dir: Optional[str] = args.local_dir
+ self.force_download: bool = args.force_download
+ self.resume_download: Optional[bool] = args.resume_download or None
+ self.quiet: bool = args.quiet
+ self.max_workers: int = args.max_workers
+
+ if args.local_dir_use_symlinks is not None:
+ warnings.warn(
+ "Ignoring --local-dir-use-symlinks. Downloading to a local directory does not use symlinks anymore.",
+ FutureWarning,
+ )
+
+ def run(self) -> None:
+ if self.quiet:
+ disable_progress_bars()
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ print(self._download()) # Print path to downloaded files
+ enable_progress_bars()
+ else:
+ logging.set_verbosity_info()
+ print(self._download()) # Print path to downloaded files
+ logging.set_verbosity_warning()
+
+ def _download(self) -> str:
+ # Warn user if patterns are ignored
+ if len(self.filenames) > 0:
+ if self.include is not None and len(self.include) > 0:
+ warnings.warn("Ignoring `--include` since filenames have being explicitly set.")
+ if self.exclude is not None and len(self.exclude) > 0:
+ warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")
+
+ # Single file to download: use `hf_hub_download`
+ if len(self.filenames) == 1:
+ return hf_hub_download(
+ repo_id=self.repo_id,
+ repo_type=self.repo_type,
+ revision=self.revision,
+ filename=self.filenames[0],
+ cache_dir=self.cache_dir,
+ resume_download=self.resume_download,
+ force_download=self.force_download,
+ token=self.token,
+ local_dir=self.local_dir,
+ library_name="huggingface-cli",
+ )
+
+ # Otherwise: use `snapshot_download` to ensure all files comes from same revision
+ elif len(self.filenames) == 0:
+ allow_patterns = self.include
+ ignore_patterns = self.exclude
+ else:
+ allow_patterns = self.filenames
+ ignore_patterns = None
+
+ return snapshot_download(
+ repo_id=self.repo_id,
+ repo_type=self.repo_type,
+ revision=self.revision,
+ allow_patterns=allow_patterns,
+ ignore_patterns=ignore_patterns,
+ resume_download=self.resume_download,
+ force_download=self.force_download,
+ cache_dir=self.cache_dir,
+ token=self.token,
+ local_dir=self.local_dir,
+ library_name="huggingface-cli",
+ max_workers=self.max_workers,
+ )
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/env.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/env.py
new file mode 100644
index 00000000..23f2828b
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/env.py
@@ -0,0 +1,36 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to print information about the environment.
+
+Usage:
+ huggingface-cli env
+"""
+
+from argparse import _SubParsersAction
+
+from ..utils import dump_environment_info
+from . import BaseHuggingfaceCLICommand
+
+
+class EnvironmentCommand(BaseHuggingfaceCLICommand):
+ def __init__(self, args):
+ self.args = args
+
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ env_parser = parser.add_parser("env", help="Print information about the environment.")
+ env_parser.set_defaults(func=EnvironmentCommand)
+
+ def run(self) -> None:
+ dump_environment_info()
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/huggingface_cli.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/huggingface_cli.py
new file mode 100644
index 00000000..1e790b5e
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/huggingface_cli.py
@@ -0,0 +1,61 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from argparse import ArgumentParser
+
+from huggingface_hub.commands.delete_cache import DeleteCacheCommand
+from huggingface_hub.commands.download import DownloadCommand
+from huggingface_hub.commands.env import EnvironmentCommand
+from huggingface_hub.commands.lfs import LfsCommands
+from huggingface_hub.commands.repo_files import RepoFilesCommand
+from huggingface_hub.commands.scan_cache import ScanCacheCommand
+from huggingface_hub.commands.tag import TagCommands
+from huggingface_hub.commands.upload import UploadCommand
+from huggingface_hub.commands.upload_large_folder import UploadLargeFolderCommand
+from huggingface_hub.commands.user import UserCommands
+from huggingface_hub.commands.version import VersionCommand
+
+
+def main():
+ parser = ArgumentParser("huggingface-cli", usage="huggingface-cli <command> [<args>]")
+ commands_parser = parser.add_subparsers(help="huggingface-cli command helpers")
+
+ # Register commands
+ DownloadCommand.register_subcommand(commands_parser)
+ UploadCommand.register_subcommand(commands_parser)
+ RepoFilesCommand.register_subcommand(commands_parser)
+ EnvironmentCommand.register_subcommand(commands_parser)
+ UserCommands.register_subcommand(commands_parser)
+ LfsCommands.register_subcommand(commands_parser)
+ ScanCacheCommand.register_subcommand(commands_parser)
+ DeleteCacheCommand.register_subcommand(commands_parser)
+ TagCommands.register_subcommand(commands_parser)
+ VersionCommand.register_subcommand(commands_parser)
+
+ # Experimental
+ UploadLargeFolderCommand.register_subcommand(commands_parser)
+
+ # Let's go
+ args = parser.parse_args()
+ if not hasattr(args, "func"):
+ parser.print_help()
+ exit(1)
+
+ # Run
+ service = args.func(args)
+ service.run()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/lfs.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/lfs.py
new file mode 100644
index 00000000..e510e345
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/lfs.py
@@ -0,0 +1,200 @@
+"""
+Implementation of a custom transfer agent for the transfer type "multipart" for
+git-lfs.
+
+Inspired by:
+github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py
+
+Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
+
+
+To launch debugger while developing:
+
+``` [lfs "customtransfer.multipart"]
+path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
+--wait-for-client
+/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
+lfs-multipart-upload ```"""
+
+import json
+import os
+import subprocess
+import sys
+from argparse import _SubParsersAction
+from typing import Dict, List, Optional
+
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
+
+from ..utils import get_session, hf_raise_for_status, logging
+from ..utils._lfs import SliceFileObj
+
+
+logger = logging.get_logger(__name__)
+
+
+class LfsCommands(BaseHuggingfaceCLICommand):
+ """
+ Implementation of a custom transfer agent for the transfer type "multipart"
+ for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom
+ transfer agent is:
+ https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
+
+ This introduces two commands to the CLI:
+
+ 1. $ huggingface-cli lfs-enable-largefiles
+
+ This should be executed once for each model repo that contains a model file
+ >5GB. It's documented in the error message you get if you just try to git
+ push a 5GB file without having enabled it before.
+
+ 2. $ huggingface-cli lfs-multipart-upload
+
+ This command is called by lfs directly and is not meant to be called by the
+ user.
+ """
+
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ enable_parser = parser.add_parser(
+ "lfs-enable-largefiles", help="Configure your repository to enable upload of files > 5GB."
+ )
+ enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.")
+ enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args))
+
+ # Command will get called by git-lfs, do not call it directly.
+ upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False)
+ upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args))
+
+
+class LfsEnableCommand:
+ def __init__(self, args):
+ self.args = args
+
+ def run(self):
+ local_path = os.path.abspath(self.args.path)
+ if not os.path.isdir(local_path):
+ print("This does not look like a valid git repo.")
+ exit(1)
+ subprocess.run(
+ "git config lfs.customtransfer.multipart.path huggingface-cli".split(),
+ check=True,
+ cwd=local_path,
+ )
+ subprocess.run(
+ f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
+ check=True,
+ cwd=local_path,
+ )
+ print("Local repo set up for largefiles")
+
+
+def write_msg(msg: Dict):
+ """Write out the message in Line delimited JSON."""
+ msg_str = json.dumps(msg) + "\n"
+ sys.stdout.write(msg_str)
+ sys.stdout.flush()
+
+
+def read_msg() -> Optional[Dict]:
+ """Read Line delimited JSON from stdin."""
+ msg = json.loads(sys.stdin.readline().strip())
+
+ if "terminate" in (msg.get("type"), msg.get("event")):
+ # terminate message received
+ return None
+
+ if msg.get("event") not in ("download", "upload"):
+ logger.critical("Received unexpected message")
+ sys.exit(1)
+
+ return msg
+
+
+class LfsUploadCommand:
+ def __init__(self, args) -> None:
+ self.args = args
+
+ def run(self) -> None:
+ # Immediately after invoking a custom transfer process, git-lfs
+ # sends initiation data to the process over stdin.
+ # This tells the process useful information about the configuration.
+ init_msg = json.loads(sys.stdin.readline().strip())
+ if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"):
+ write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
+ sys.exit(1)
+
+ # The transfer process should use the information it needs from the
+ # initiation structure, and also perform any one-off setup tasks it
+ # needs to do. It should then respond on stdout with a simple empty
+ # confirmation structure, as follows:
+ write_msg({})
+
+ # After the initiation exchange, git-lfs will send any number of
+ # transfer requests to the stdin of the transfer process, in a serial sequence.
+ while True:
+ msg = read_msg()
+ if msg is None:
+ # When all transfers have been processed, git-lfs will send
+ # a terminate event to the stdin of the transfer process.
+ # On receiving this message the transfer process should
+ # clean up and terminate. No response is expected.
+ sys.exit(0)
+
+ oid = msg["oid"]
+ filepath = msg["path"]
+ completion_url = msg["action"]["href"]
+ header = msg["action"]["header"]
+ chunk_size = int(header.pop("chunk_size"))
+ presigned_urls: List[str] = list(header.values())
+
+ # Send a "started" progress event to allow other workers to start.
+ # Otherwise they're delayed until first "progress" event is reported,
+ # i.e. after the first 5GB by default (!)
+ write_msg(
+ {
+ "event": "progress",
+ "oid": oid,
+ "bytesSoFar": 1,
+ "bytesSinceLast": 0,
+ }
+ )
+
+ parts = []
+ with open(filepath, "rb") as file:
+ for i, presigned_url in enumerate(presigned_urls):
+ with SliceFileObj(
+ file,
+ seek_from=i * chunk_size,
+ read_limit=chunk_size,
+ ) as data:
+ r = get_session().put(presigned_url, data=data)
+ hf_raise_for_status(r)
+ parts.append(
+ {
+ "etag": r.headers.get("etag"),
+ "partNumber": i + 1,
+ }
+ )
+ # In order to support progress reporting while data is uploading / downloading,
+ # the transfer process should post messages to stdout
+ write_msg(
+ {
+ "event": "progress",
+ "oid": oid,
+ "bytesSoFar": (i + 1) * chunk_size,
+ "bytesSinceLast": chunk_size,
+ }
+ )
+ # Not precise but that's ok.
+
+ r = get_session().post(
+ completion_url,
+ json={
+ "oid": oid,
+ "parts": parts,
+ },
+ )
+ hf_raise_for_status(r)
+
+ write_msg({"event": "complete", "oid": oid})
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo_files.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo_files.py
new file mode 100644
index 00000000..f15bbed0
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo_files.py
@@ -0,0 +1,128 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to update or delete files in a repository using the CLI.
+
+Usage:
+ # delete all
+ huggingface-cli repo-files <repo_id> delete "*"
+
+ # delete single file
+ huggingface-cli repo-files <repo_id> delete file.txt
+
+ # delete single folder
+ huggingface-cli repo-files <repo_id> delete folder/
+
+ # delete multiple
+ huggingface-cli repo-files <repo_id> delete file.txt folder/ file2.txt
+
+ # delete multiple patterns
+ huggingface-cli repo-files <repo_id> delete file.txt "*.json" "folder/*.parquet"
+
+ # delete from different revision / repo-type
+ huggingface-cli repo-files <repo_id> delete file.txt --revision=refs/pr/1 --repo-type=dataset
+"""
+
+from argparse import _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.hf_api import HfApi
+
+
+logger = logging.get_logger(__name__)
+
+
+class DeleteFilesSubCommand:
+ def __init__(self, args) -> None:
+ self.args = args
+ self.repo_id: str = args.repo_id
+ self.repo_type: Optional[str] = args.repo_type
+ self.revision: Optional[str] = args.revision
+ self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
+ self.patterns: List[str] = args.patterns
+ self.commit_message: Optional[str] = args.commit_message
+ self.commit_description: Optional[str] = args.commit_description
+ self.create_pr: bool = args.create_pr
+ self.token: Optional[str] = args.token
+
+ def run(self) -> None:
+ logging.set_verbosity_info()
+ url = self.api.delete_files(
+ delete_patterns=self.patterns,
+ repo_id=self.repo_id,
+ repo_type=self.repo_type,
+ revision=self.revision,
+ commit_message=self.commit_message,
+ commit_description=self.commit_description,
+ create_pr=self.create_pr,
+ )
+ print(f"Files correctly deleted from repo. Commit: {url}.")
+ logging.set_verbosity_warning()
+
+
+class RepoFilesCommand(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub")
+ repo_files_parser.add_argument(
+ "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)."
+ )
+ repo_files_subparsers = repo_files_parser.add_subparsers(
+ help="Action to execute against the files.",
+ required=True,
+ )
+ delete_subparser = repo_files_subparsers.add_parser(
+ "delete",
+ help="Delete files from a repo on the Hub",
+ )
+ delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args))
+ delete_subparser.add_argument(
+ "patterns",
+ nargs="+",
+ type=str,
+ help="Glob patterns to match files to delete.",
+ )
+ delete_subparser.add_argument(
+ "--repo-type",
+ choices=["model", "dataset", "space"],
+ default="model",
+ help="Type of the repo to upload to (e.g. `dataset`).",
+ )
+ delete_subparser.add_argument(
+ "--revision",
+ type=str,
+ help=(
+ "An optional Git revision to push to. It can be a branch name "
+ "or a PR reference. If revision does not"
+ " exist and `--create-pr` is not set, a branch will be automatically created."
+ ),
+ )
+ delete_subparser.add_argument(
+ "--commit-message", type=str, help="The summary / title / first line of the generated commit."
+ )
+ delete_subparser.add_argument(
+ "--commit-description", type=str, help="The description of the generated commit."
+ )
+ delete_subparser.add_argument(
+ "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes."
+ )
+ repo_files_parser.add_argument(
+ "--token",
+ type=str,
+ help="A User Access Token generated from https://huggingface.co/settings/tokens",
+ )
+
+ repo_files_parser.set_defaults(func=RepoFilesCommand)
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/scan_cache.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/scan_cache.py
new file mode 100644
index 00000000..799b9ba5
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/scan_cache.py
@@ -0,0 +1,181 @@
+# coding=utf-8
+# Copyright 2022-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to scan the HF cache directory.
+
+Usage:
+ huggingface-cli scan-cache
+ huggingface-cli scan-cache -v
+ huggingface-cli scan-cache -vvv
+ huggingface-cli scan-cache --dir ~/.cache/huggingface/hub
+"""
+
+import time
+from argparse import Namespace, _SubParsersAction
+from typing import Optional
+
+from ..utils import CacheNotFound, HFCacheInfo, scan_cache_dir
+from . import BaseHuggingfaceCLICommand
+from ._cli_utils import ANSI, tabulate
+
+
+class ScanCacheCommand(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ scan_cache_parser = parser.add_parser("scan-cache", help="Scan cache directory.")
+
+ scan_cache_parser.add_argument(
+ "--dir",
+ type=str,
+ default=None,
+ help="cache directory to scan (optional). Default to the default HuggingFace cache.",
+ )
+ scan_cache_parser.add_argument(
+ "-v",
+ "--verbose",
+ action="count",
+ default=0,
+ help="show a more verbose output",
+ )
+ scan_cache_parser.set_defaults(func=ScanCacheCommand)
+
+ def __init__(self, args: Namespace) -> None:
+ self.verbosity: int = args.verbose
+ self.cache_dir: Optional[str] = args.dir
+
+ def run(self):
+ try:
+ t0 = time.time()
+ hf_cache_info = scan_cache_dir(self.cache_dir)
+ t1 = time.time()
+ except CacheNotFound as exc:
+ cache_dir = exc.cache_dir
+ print(f"Cache directory not found: {cache_dir}")
+ return
+
+ self._print_hf_cache_info_as_table(hf_cache_info)
+
+ print(
+ f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)"
+ f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}."
+ )
+ if len(hf_cache_info.warnings) > 0:
+ message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning."
+ if self.verbosity >= 3:
+ print(ANSI.gray(message))
+ for warning in hf_cache_info.warnings:
+ print(ANSI.gray(warning))
+ else:
+ print(ANSI.gray(message + " Use -vvv to print details."))
+
+ def _print_hf_cache_info_as_table(self, hf_cache_info: HFCacheInfo) -> None:
+ print(get_table(hf_cache_info, verbosity=self.verbosity))
+
+
+def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str:
+ """Generate a table from the [`HFCacheInfo`] object.
+
+ Pass `verbosity=0` to get a table with a single row per repo, with columns
+ "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path".
+
+ Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns
+ "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path".
+
+ Example:
+ ```py
+ >>> from huggingface_hub.utils import scan_cache_dir
+ >>> from huggingface_hub.commands.scan_cache import get_table
+
+ >>> hf_cache_info = scan_cache_dir()
+ HFCacheInfo(...)
+
+ >>> print(get_table(hf_cache_info, verbosity=0))
+ REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH
+ --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- --------------------------------------------------------------------------------------------------
+ roberta-base model 2.7M 5 1 day ago 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base
+ suno/bark model 8.8K 1 1 week ago 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark
+ t5-base model 893.8M 4 4 days ago 7 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base
+ t5-large model 3.0G 4 5 weeks ago 5 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large
+
+ >>> print(get_table(hf_cache_info, verbosity=1))
+ REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH
+ --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- -----------------------------------------------------------------------------------------------------------------------------------------------------
+ roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base\\snapshots\\e2da8e2f811d1448a5b465c236feacd80ffbac7b
+ suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark\\snapshots\\70a8a7d34168586dc5d028fa9666aceade177992
+ t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base\\snapshots\\a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1
+ t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large\\snapshots\\150ebc2c4b72291e770f58e6057481c8d2ed331a ```
+ ```
+
+ Args:
+ hf_cache_info ([`HFCacheInfo`]):
+ The HFCacheInfo object to print.
+ verbosity (`int`, *optional*):
+ The verbosity level. Defaults to 0.
+
+ Returns:
+ `str`: The table as a string.
+ """
+ if verbosity == 0:
+ return tabulate(
+ rows=[
+ [
+ repo.repo_id,
+ repo.repo_type,
+ "{:>12}".format(repo.size_on_disk_str),
+ repo.nb_files,
+ repo.last_accessed_str,
+ repo.last_modified_str,
+ ", ".join(sorted(repo.refs)),
+ str(repo.repo_path),
+ ]
+ for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path)
+ ],
+ headers=[
+ "REPO ID",
+ "REPO TYPE",
+ "SIZE ON DISK",
+ "NB FILES",
+ "LAST_ACCESSED",
+ "LAST_MODIFIED",
+ "REFS",
+ "LOCAL PATH",
+ ],
+ )
+ else:
+ return tabulate(
+ rows=[
+ [
+ repo.repo_id,
+ repo.repo_type,
+ revision.commit_hash,
+ "{:>12}".format(revision.size_on_disk_str),
+ revision.nb_files,
+ revision.last_modified_str,
+ ", ".join(sorted(revision.refs)),
+ str(revision.snapshot_path),
+ ]
+ for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path)
+ for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash)
+ ],
+ headers=[
+ "REPO ID",
+ "REPO TYPE",
+ "REVISION",
+ "SIZE ON DISK",
+ "NB FILES",
+ "LAST_MODIFIED",
+ "REFS",
+ "LOCAL PATH",
+ ],
+ )
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/tag.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/tag.py
new file mode 100644
index 00000000..c3beab90
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/tag.py
@@ -0,0 +1,159 @@
+# coding=utf-8
+# Copyright 2024-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains commands to perform tag management with the CLI.
+
+Usage Examples:
+ - Create a tag:
+ $ huggingface-cli tag user/my-model 1.0 --message "First release"
+ $ huggingface-cli tag user/my-model 1.0 -m "First release" --revision develop
+ $ huggingface-cli tag user/my-dataset 1.0 -m "First release" --repo-type dataset
+ $ huggingface-cli tag user/my-space 1.0
+ - List all tags:
+ $ huggingface-cli tag -l user/my-model
+ $ huggingface-cli tag --list user/my-dataset --repo-type dataset
+ - Delete a tag:
+ $ huggingface-cli tag -d user/my-model 1.0
+ $ huggingface-cli tag --delete user/my-dataset 1.0 --repo-type dataset
+ $ huggingface-cli tag -d user/my-space 1.0 -y
+"""
+
+from argparse import Namespace, _SubParsersAction
+
+from requests.exceptions import HTTPError
+
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.constants import (
+ REPO_TYPES,
+)
+from huggingface_hub.hf_api import HfApi
+
+from ..errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
+from ._cli_utils import ANSI
+
+
+class TagCommands(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ tag_parser = parser.add_parser("tag", help="(create, list, delete) tags for a repo in the hub")
+
+ tag_parser.add_argument("repo_id", type=str, help="The ID of the repo to tag (e.g. `username/repo-name`).")
+ tag_parser.add_argument("tag", nargs="?", type=str, help="The name of the tag for creation or deletion.")
+ tag_parser.add_argument("-m", "--message", type=str, help="The description of the tag to create.")
+ tag_parser.add_argument("--revision", type=str, help="The git revision to tag.")
+ tag_parser.add_argument(
+ "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens."
+ )
+ tag_parser.add_argument(
+ "--repo-type",
+ choices=["model", "dataset", "space"],
+ default="model",
+ help="Set the type of repository (model, dataset, or space).",
+ )
+ tag_parser.add_argument("-y", "--yes", action="store_true", help="Answer Yes to prompts automatically.")
+
+ tag_parser.add_argument("-l", "--list", action="store_true", help="List tags for a repository.")
+ tag_parser.add_argument("-d", "--delete", action="store_true", help="Delete a tag for a repository.")
+
+ tag_parser.set_defaults(func=lambda args: handle_commands(args))
+
+
+def handle_commands(args: Namespace):
+ if args.list:
+ return TagListCommand(args)
+ elif args.delete:
+ return TagDeleteCommand(args)
+ else:
+ return TagCreateCommand(args)
+
+
+class TagCommand:
+ def __init__(self, args: Namespace):
+ self.args = args
+ self.api = HfApi(token=self.args.token)
+ self.repo_id = self.args.repo_id
+ self.repo_type = self.args.repo_type
+ if self.repo_type not in REPO_TYPES:
+ print("Invalid repo --repo-type")
+ exit(1)
+
+
+class TagCreateCommand(TagCommand):
+ def run(self):
+ print(f"You are about to create tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}")
+
+ try:
+ self.api.create_tag(
+ repo_id=self.repo_id,
+ tag=self.args.tag,
+ tag_message=self.args.message,
+ revision=self.args.revision,
+ repo_type=self.repo_type,
+ )
+ except RepositoryNotFoundError:
+ print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+ exit(1)
+ except RevisionNotFoundError:
+ print(f"Revision {ANSI.bold(self.args.revision)} not found.")
+ exit(1)
+ except HfHubHTTPError as e:
+ if e.response.status_code == 409:
+ print(f"Tag {ANSI.bold(self.args.tag)} already exists on {ANSI.bold(self.repo_id)}")
+ exit(1)
+ raise e
+
+ print(f"Tag {ANSI.bold(self.args.tag)} created on {ANSI.bold(self.repo_id)}")
+
+
+class TagListCommand(TagCommand):
+ def run(self):
+ try:
+ refs = self.api.list_repo_refs(
+ repo_id=self.repo_id,
+ repo_type=self.repo_type,
+ )
+ except RepositoryNotFoundError:
+ print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+ exit(1)
+ except HTTPError as e:
+ print(e)
+ print(ANSI.red(e.response.text))
+ exit(1)
+ if len(refs.tags) == 0:
+ print("No tags found")
+ exit(0)
+ print(f"Tags for {self.repo_type} {ANSI.bold(self.repo_id)}:")
+ for tag in refs.tags:
+ print(tag.name)
+
+
+class TagDeleteCommand(TagCommand):
+ def run(self):
+ print(f"You are about to delete tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}")
+
+ if not self.args.yes:
+ choice = input("Proceed? [Y/n] ").lower()
+ if choice not in ("", "y", "yes"):
+ print("Abort")
+ exit()
+ try:
+ self.api.delete_tag(repo_id=self.repo_id, tag=self.args.tag, repo_type=self.repo_type)
+ except RepositoryNotFoundError:
+ print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+ exit(1)
+ except RevisionNotFoundError:
+ print(f"Tag {ANSI.bold(self.args.tag)} not found on {ANSI.bold(self.repo_id)}")
+ exit(1)
+ print(f"Tag {ANSI.bold(self.args.tag)} deleted on {ANSI.bold(self.repo_id)}")
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload.py
new file mode 100644
index 00000000..c5db1118
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload.py
@@ -0,0 +1,299 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to upload a repo or file with the CLI.
+
+Usage:
+ # Upload file (implicit)
+ huggingface-cli upload my-cool-model ./my-cool-model.safetensors
+
+ # Upload file (explicit)
+ huggingface-cli upload my-cool-model ./my-cool-model.safetensors model.safetensors
+
+ # Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised.
+ huggingface-cli upload my-cool-model
+
+ # Upload directory (explicit)
+ huggingface-cli upload my-cool-model ./models/my-cool-model .
+
+ # Upload filtered directory (example: tensorboard logs except for the last run)
+ huggingface-cli upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*"
+
+ # Upload private dataset
+ huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private
+
+ # Upload with token
+ huggingface-cli upload Wauplin/my-cool-model --token=hf_****
+
+ # Sync local Space with Hub (upload new files, delete removed files)
+ huggingface-cli upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub"
+
+ # Schedule commits every 30 minutes
+ huggingface-cli upload Wauplin/my-cool-model --every=30
+"""
+
+import os
+import time
+import warnings
+from argparse import Namespace, _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub._commit_scheduler import CommitScheduler
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER
+from huggingface_hub.errors import RevisionNotFoundError
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
+
+
+logger = logging.get_logger(__name__)
+
+
+class UploadCommand(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ upload_parser = parser.add_parser("upload", help="Upload a file or a folder to a repo on the Hub")
+ upload_parser.add_argument(
+ "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
+ )
+ upload_parser.add_argument(
+ "local_path", nargs="?", help="Local path to the file or folder to upload. Defaults to current directory."
+ )
+ upload_parser.add_argument(
+ "path_in_repo",
+ nargs="?",
+ help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.",
+ )
+ upload_parser.add_argument(
+ "--repo-type",
+ choices=["model", "dataset", "space"],
+ default="model",
+ help="Type of the repo to upload to (e.g. `dataset`).",
+ )
+ upload_parser.add_argument(
+ "--revision",
+ type=str,
+ help=(
+ "An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not"
+ " exist and `--create-pr` is not set, a branch will be automatically created."
+ ),
+ )
+ upload_parser.add_argument(
+ "--private",
+ action="store_true",
+ help=(
+ "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already"
+ " exists."
+ ),
+ )
+ upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
+ upload_parser.add_argument(
+ "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload."
+ )
+ upload_parser.add_argument(
+ "--delete",
+ nargs="*",
+ type=str,
+ help="Glob patterns for file to be deleted from the repo while committing.",
+ )
+ upload_parser.add_argument(
+ "--commit-message", type=str, help="The summary / title / first line of the generated commit."
+ )
+ upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.")
+ upload_parser.add_argument(
+ "--create-pr", action="store_true", help="Whether to upload content as a new Pull Request."
+ )
+ upload_parser.add_argument(
+ "--every",
+ type=float,
+ help="If set, a background job is scheduled to create commits every `every` minutes.",
+ )
+ upload_parser.add_argument(
+ "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
+ )
+ upload_parser.add_argument(
+ "--quiet",
+ action="store_true",
+ help="If True, progress bars are disabled and only the path to the uploaded files is printed.",
+ )
+ upload_parser.set_defaults(func=UploadCommand)
+
+ def __init__(self, args: Namespace) -> None:
+ self.repo_id: str = args.repo_id
+ self.repo_type: Optional[str] = args.repo_type
+ self.revision: Optional[str] = args.revision
+ self.private: bool = args.private
+
+ self.include: Optional[List[str]] = args.include
+ self.exclude: Optional[List[str]] = args.exclude
+ self.delete: Optional[List[str]] = args.delete
+
+ self.commit_message: Optional[str] = args.commit_message
+ self.commit_description: Optional[str] = args.commit_description
+ self.create_pr: bool = args.create_pr
+ self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
+ self.quiet: bool = args.quiet # disable warnings and progress bars
+
+ # Check `--every` is valid
+ if args.every is not None and args.every <= 0:
+ raise ValueError(f"`every` must be a positive value (got '{args.every}')")
+ self.every: Optional[float] = args.every
+
+ # Resolve `local_path` and `path_in_repo`
+ repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model"
+ self.local_path: str
+ self.path_in_repo: str
+ if args.local_path is None and os.path.isfile(repo_name):
+ # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name
+ self.local_path = repo_name
+ self.path_in_repo = repo_name
+ elif args.local_path is None and os.path.isdir(repo_name):
+ # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root
+ self.local_path = repo_name
+ self.path_in_repo = "."
+ elif args.local_path is None:
+ # Implicit case 3: user provided only a repo_id that does not match a local file or folder
+ # => the user must explicitly provide a local_path => raise exception
+ raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.")
+ elif args.path_in_repo is None and os.path.isfile(args.local_path):
+ # Explicit local path to file, no path in repo => upload it at root with same name
+ self.local_path = args.local_path
+ self.path_in_repo = os.path.basename(args.local_path)
+ elif args.path_in_repo is None:
+ # Explicit local path to folder, no path in repo => upload at root
+ self.local_path = args.local_path
+ self.path_in_repo = "."
+ else:
+ # Finally, if both paths are explicit
+ self.local_path = args.local_path
+ self.path_in_repo = args.path_in_repo
+
+ def run(self) -> None:
+ if self.quiet:
+ disable_progress_bars()
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ print(self._upload())
+ enable_progress_bars()
+ else:
+ logging.set_verbosity_info()
+ print(self._upload())
+ logging.set_verbosity_warning()
+
+ def _upload(self) -> str:
+ if os.path.isfile(self.local_path):
+ if self.include is not None and len(self.include) > 0:
+ warnings.warn("Ignoring `--include` since a single file is uploaded.")
+ if self.exclude is not None and len(self.exclude) > 0:
+ warnings.warn("Ignoring `--exclude` since a single file is uploaded.")
+ if self.delete is not None and len(self.delete) > 0:
+ warnings.warn("Ignoring `--delete` since a single file is uploaded.")
+
+ if not HF_HUB_ENABLE_HF_TRANSFER:
+ logger.info(
+ "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See"
+ " https://huggingface.co/docs/huggingface_hub/hf_transfer for more details."
+ )
+
+ # Schedule commits if `every` is set
+ if self.every is not None:
+ if os.path.isfile(self.local_path):
+ # If file => watch entire folder + use allow_patterns
+ folder_path = os.path.dirname(self.local_path)
+ path_in_repo = (
+ self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo
+ if self.path_in_repo.endswith(self.local_path)
+ else self.path_in_repo
+ )
+ allow_patterns = [self.local_path]
+ ignore_patterns = []
+ else:
+ folder_path = self.local_path
+ path_in_repo = self.path_in_repo
+ allow_patterns = self.include or []
+ ignore_patterns = self.exclude or []
+ if self.delete is not None and len(self.delete) > 0:
+ warnings.warn("Ignoring `--delete` when uploading with scheduled commits.")
+
+ scheduler = CommitScheduler(
+ folder_path=folder_path,
+ repo_id=self.repo_id,
+ repo_type=self.repo_type,
+ revision=self.revision,
+ allow_patterns=allow_patterns,
+ ignore_patterns=ignore_patterns,
+ path_in_repo=path_in_repo,
+ private=self.private,
+ every=self.every,
+ hf_api=self.api,
+ )
+ print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.")
+ try: # Block main thread until KeyboardInterrupt
+ while True:
+ time.sleep(100)
+ except KeyboardInterrupt:
+ scheduler.stop()
+ return "Stopped scheduled commits."
+
+ # Otherwise, create repo and proceed with the upload
+ if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path):
+ raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.")
+ repo_id = self.api.create_repo(
+ repo_id=self.repo_id,
+ repo_type=self.repo_type,
+ exist_ok=True,
+ private=self.private,
+ space_sdk="gradio" if self.repo_type == "space" else None,
+ # ^ We don't want it to fail when uploading to a Space => let's set Gradio by default.
+ # ^ I'd rather not add CLI args to set it explicitly as we already have `huggingface-cli repo create` for that.
+ ).repo_id
+
+ # Check if branch already exists and if not, create it
+ if self.revision is not None and not self.create_pr:
+ try:
+ self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision)
+ except RevisionNotFoundError:
+ logger.info(f"Branch '{self.revision}' not found. Creating it...")
+ self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True)
+ # ^ `exist_ok=True` to avoid race concurrency issues
+
+ # File-based upload
+ if os.path.isfile(self.local_path):
+ return self.api.upload_file(
+ path_or_fileobj=self.local_path,
+ path_in_repo=self.path_in_repo,
+ repo_id=repo_id,
+ repo_type=self.repo_type,
+ revision=self.revision,
+ commit_message=self.commit_message,
+ commit_description=self.commit_description,
+ create_pr=self.create_pr,
+ )
+
+ # Folder-based upload
+ else:
+ return self.api.upload_folder(
+ folder_path=self.local_path,
+ path_in_repo=self.path_in_repo,
+ repo_id=repo_id,
+ repo_type=self.repo_type,
+ revision=self.revision,
+ commit_message=self.commit_message,
+ commit_description=self.commit_description,
+ create_pr=self.create_pr,
+ allow_patterns=self.include,
+ ignore_patterns=self.exclude,
+ delete_patterns=self.delete,
+ )
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py
new file mode 100644
index 00000000..61c12a9f
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py
@@ -0,0 +1,129 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to upload a large folder with the CLI."""
+
+import os
+from argparse import Namespace, _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import disable_progress_bars
+
+from ._cli_utils import ANSI
+
+
+logger = logging.get_logger(__name__)
+
+
+class UploadLargeFolderCommand(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ subparser = parser.add_parser("upload-large-folder", help="Upload a large folder to a repo on the Hub")
+ subparser.add_argument(
+ "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
+ )
+ subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.")
+ subparser.add_argument(
+ "--repo-type",
+ choices=["model", "dataset", "space"],
+ help="Type of the repo to upload to (e.g. `dataset`).",
+ )
+ subparser.add_argument(
+ "--revision",
+ type=str,
+ help=("An optional Git revision to push to. It can be a branch name or a PR reference."),
+ )
+ subparser.add_argument(
+ "--private",
+ action="store_true",
+ help=(
+ "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists."
+ ),
+ )
+ subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
+ subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.")
+ subparser.add_argument(
+ "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
+ )
+ subparser.add_argument(
+ "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files."
+ )
+ subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.")
+ subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.")
+ subparser.set_defaults(func=UploadLargeFolderCommand)
+
+ def __init__(self, args: Namespace) -> None:
+ self.repo_id: str = args.repo_id
+ self.local_path: str = args.local_path
+ self.repo_type: str = args.repo_type
+ self.revision: Optional[str] = args.revision
+ self.private: bool = args.private
+
+ self.include: Optional[List[str]] = args.include
+ self.exclude: Optional[List[str]] = args.exclude
+
+ self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
+
+ self.num_workers: Optional[int] = args.num_workers
+ self.no_report: bool = args.no_report
+ self.no_bars: bool = args.no_bars
+
+ if not os.path.isdir(self.local_path):
+ raise ValueError("Large upload is only supported for folders.")
+
+ def run(self) -> None:
+ logging.set_verbosity_info()
+
+ print(
+ ANSI.yellow(
+ "You are about to upload a large folder to the Hub using `huggingface-cli upload-large-folder`. "
+ "This is a new feature so feedback is very welcome!\n"
+ "\n"
+ "A few things to keep in mind:\n"
+ " - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"
+ " - Do not start several processes in parallel.\n"
+ " - You can interrupt and resume the process at any time. "
+ "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"
+ " - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"
+ "\n"
+ f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n"
+ " - You must not modify those files manually.\n"
+ " - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"
+ " - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"
+ "\n"
+ "If the process output is too verbose, you can disable the progress bars with `--no-bars`. "
+ "You can also entirely disable the status report with `--no-report`.\n"
+ "\n"
+ "For more details, run `huggingface-cli upload-large-folder --help` or check the documentation at "
+ "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."
+ )
+ )
+
+ if self.no_bars:
+ disable_progress_bars()
+
+ self.api.upload_large_folder(
+ repo_id=self.repo_id,
+ folder_path=self.local_path,
+ repo_type=self.repo_type,
+ revision=self.revision,
+ private=self.private,
+ allow_patterns=self.include,
+ ignore_patterns=self.exclude,
+ num_workers=self.num_workers,
+ print_report=not self.no_report,
+ )
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/user.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/user.py
new file mode 100644
index 00000000..9741a219
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/user.py
@@ -0,0 +1,304 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories.
+
+Usage:
+ # login and save token locally.
+ huggingface-cli login --token=hf_*** --add-to-git-credential
+
+ # switch between tokens
+ huggingface-cli auth switch
+
+ # list all tokens
+ huggingface-cli auth list
+
+ # logout from a specific token, if no token-name is provided, all tokens will be deleted from your machine.
+ huggingface-cli logout --token-name=your_token_name
+
+ # find out which huggingface.co account you are logged in as
+ huggingface-cli whoami
+
+ # create a new dataset repo on the Hub
+ huggingface-cli repo create mydataset --type=dataset
+
+"""
+
+import subprocess
+from argparse import _SubParsersAction
+from typing import List, Optional
+
+from requests.exceptions import HTTPError
+
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.constants import ENDPOINT, REPO_TYPES, REPO_TYPES_URL_PREFIXES, SPACES_SDK_TYPES
+from huggingface_hub.hf_api import HfApi
+
+from .._login import ( # noqa: F401 # for backward compatibility # noqa: F401 # for backward compatibility
+ NOTEBOOK_LOGIN_PASSWORD_HTML,
+ NOTEBOOK_LOGIN_TOKEN_HTML_END,
+ NOTEBOOK_LOGIN_TOKEN_HTML_START,
+ auth_list,
+ auth_switch,
+ login,
+ logout,
+ notebook_login,
+)
+from ..utils import get_stored_tokens, get_token, logging
+from ._cli_utils import ANSI
+
+
+logger = logging.get_logger(__name__)
+
+try:
+ from InquirerPy import inquirer
+ from InquirerPy.base.control import Choice
+
+ _inquirer_py_available = True
+except ImportError:
+ _inquirer_py_available = False
+
+
+class UserCommands(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ login_parser = parser.add_parser("login", help="Log in using a token from huggingface.co/settings/tokens")
+ login_parser.add_argument(
+ "--token",
+ type=str,
+ help="Token generated from https://huggingface.co/settings/tokens",
+ )
+ login_parser.add_argument(
+ "--add-to-git-credential",
+ action="store_true",
+ help="Optional: Save token to git credential helper.",
+ )
+ login_parser.set_defaults(func=lambda args: LoginCommand(args))
+ whoami_parser = parser.add_parser("whoami", help="Find out which huggingface.co account you are logged in as.")
+ whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args))
+
+ logout_parser = parser.add_parser("logout", help="Log out")
+ logout_parser.add_argument(
+ "--token-name",
+ type=str,
+ help="Optional: Name of the access token to log out from.",
+ )
+ logout_parser.set_defaults(func=lambda args: LogoutCommand(args))
+
+ auth_parser = parser.add_parser("auth", help="Other authentication related commands")
+ auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands")
+ auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens")
+ auth_switch_parser.add_argument(
+ "--token-name",
+ type=str,
+ help="Optional: Name of the access token to switch to.",
+ )
+ auth_switch_parser.add_argument(
+ "--add-to-git-credential",
+ action="store_true",
+ help="Optional: Save token to git credential helper.",
+ )
+ auth_switch_parser.set_defaults(func=lambda args: AuthSwitchCommand(args))
+ auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens")
+ auth_list_parser.set_defaults(func=lambda args: AuthListCommand(args))
+ # new system: git-based repo system
+ repo_parser = parser.add_parser("repo", help="{create} Commands to interact with your huggingface.co repos.")
+ repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands")
+ repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co")
+ repo_create_parser.add_argument(
+ "name",
+ type=str,
+ help="Name for your repo. Will be namespaced under your username to build the repo id.",
+ )
+ repo_create_parser.add_argument(
+ "--type",
+ type=str,
+ help='Optional: repo_type: set to "dataset" or "space" if creating a dataset or space, default is model.',
+ )
+ repo_create_parser.add_argument("--organization", type=str, help="Optional: organization namespace.")
+ repo_create_parser.add_argument(
+ "--space_sdk",
+ type=str,
+ help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".',
+ choices=SPACES_SDK_TYPES,
+ )
+ repo_create_parser.add_argument(
+ "-y",
+ "--yes",
+ action="store_true",
+ help="Optional: answer Yes to the prompt",
+ )
+ repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args))
+
+
+class BaseUserCommand:
+ def __init__(self, args):
+ self.args = args
+ self._api = HfApi()
+
+
+class LoginCommand(BaseUserCommand):
+ def run(self):
+ logging.set_verbosity_info()
+ login(
+ token=self.args.token,
+ add_to_git_credential=self.args.add_to_git_credential,
+ )
+
+
+class LogoutCommand(BaseUserCommand):
+ def run(self):
+ logging.set_verbosity_info()
+ logout(token_name=self.args.token_name)
+
+
+class AuthSwitchCommand(BaseUserCommand):
+ def run(self):
+ logging.set_verbosity_info()
+ token_name = self.args.token_name
+ if token_name is None:
+ token_name = self._select_token_name()
+
+ if token_name is None:
+ print("No token name provided. Aborting.")
+ exit()
+ auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential)
+
+ def _select_token_name(self) -> Optional[str]:
+ token_names = list(get_stored_tokens().keys())
+
+ if not token_names:
+ logger.error("No stored tokens found. Please login first.")
+ return None
+
+ if _inquirer_py_available:
+ return self._select_token_name_tui(token_names)
+ # if inquirer is not available, use a simpler terminal UI
+ print("Available stored tokens:")
+ for i, token_name in enumerate(token_names, 1):
+ print(f"{i}. {token_name}")
+ while True:
+ try:
+ choice = input("Enter the number of the token to switch to (or 'q' to quit): ")
+ if choice.lower() == "q":
+ return None
+ index = int(choice) - 1
+ if 0 <= index < len(token_names):
+ return token_names[index]
+ else:
+ print("Invalid selection. Please try again.")
+ except ValueError:
+ print("Invalid input. Please enter a number or 'q' to quit.")
+
+ def _select_token_name_tui(self, token_names: List[str]) -> Optional[str]:
+ choices = [Choice(token_name, name=token_name) for token_name in token_names]
+ try:
+ return inquirer.select(
+ message="Select a token to switch to:",
+ choices=choices,
+ default=None,
+ ).execute()
+ except KeyboardInterrupt:
+ logger.info("Token selection cancelled.")
+ return None
+
+
+class AuthListCommand(BaseUserCommand):
+ def run(self):
+ logging.set_verbosity_info()
+ auth_list()
+
+
+class WhoamiCommand(BaseUserCommand):
+ def run(self):
+ token = get_token()
+ if token is None:
+ print("Not logged in")
+ exit()
+ try:
+ info = self._api.whoami(token)
+ print(info["name"])
+ orgs = [org["name"] for org in info["orgs"]]
+ if orgs:
+ print(ANSI.bold("orgs: "), ",".join(orgs))
+
+ if ENDPOINT != "https://huggingface.co":
+ print(f"Authenticated through private endpoint: {ENDPOINT}")
+ except HTTPError as e:
+ print(e)
+ print(ANSI.red(e.response.text))
+ exit(1)
+
+
+class RepoCreateCommand(BaseUserCommand):
+ def run(self):
+ token = get_token()
+ if token is None:
+ print("Not logged in")
+ exit(1)
+ try:
+ stdout = subprocess.check_output(["git", "--version"]).decode("utf-8")
+ print(ANSI.gray(stdout.strip()))
+ except FileNotFoundError:
+ print("Looks like you do not have git installed, please install.")
+
+ try:
+ stdout = subprocess.check_output(["git-lfs", "--version"]).decode("utf-8")
+ print(ANSI.gray(stdout.strip()))
+ except FileNotFoundError:
+ print(
+ ANSI.red(
+ "Looks like you do not have git-lfs installed, please install."
+ " You can install from https://git-lfs.github.com/."
+ " Then run `git lfs install` (you only have to do this once)."
+ )
+ )
+ print("")
+
+ user = self._api.whoami(token)["name"]
+ namespace = self.args.organization if self.args.organization is not None else user
+
+ repo_id = f"{namespace}/{self.args.name}"
+
+ if self.args.type not in REPO_TYPES:
+ print("Invalid repo --type")
+ exit(1)
+
+ if self.args.type in REPO_TYPES_URL_PREFIXES:
+ prefixed_repo_id = REPO_TYPES_URL_PREFIXES[self.args.type] + repo_id
+ else:
+ prefixed_repo_id = repo_id
+
+ print(f"You are about to create {ANSI.bold(prefixed_repo_id)}")
+
+ if not self.args.yes:
+ choice = input("Proceed? [Y/n] ").lower()
+ if not (choice == "" or choice == "y" or choice == "yes"):
+ print("Abort")
+ exit()
+ try:
+ url = self._api.create_repo(
+ repo_id=repo_id,
+ token=token,
+ repo_type=self.args.type,
+ space_sdk=self.args.space_sdk,
+ )
+ except HTTPError as e:
+ print(e)
+ print(ANSI.red(e.response.text))
+ exit(1)
+ print("\nYour repo now lives at:")
+ print(f" {ANSI.bold(url)}")
+ print("\nYou can clone it locally with the command below, and commit/push as usual.")
+ print(f"\n git clone {url}")
+ print("")
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/version.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/version.py
new file mode 100644
index 00000000..f7e866b7
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/version.py
@@ -0,0 +1,37 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to print information about the version.
+
+Usage:
+ huggingface-cli version
+"""
+
+from argparse import _SubParsersAction
+
+from huggingface_hub import __version__
+
+from . import BaseHuggingfaceCLICommand
+
+
+class VersionCommand(BaseHuggingfaceCLICommand):
+ def __init__(self, args):
+ self.args = args
+
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ version_parser = parser.add_parser("version", help="Print information about the huggingface-cli version.")
+ version_parser.set_defaults(func=VersionCommand)
+
+ def run(self) -> None:
+ print(f"huggingface_hub version: {__version__}")