aboutsummaryrefslogtreecommitdiff
path: root/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py
diff options
context:
space:
mode:
authorS. Solomon Darnell2025-03-28 21:52:21 -0500
committerS. Solomon Darnell2025-03-28 21:52:21 -0500
commit4a52a71956a8d46fcb7294ac71734504bb09bcc2 (patch)
treeee3dc5af3b6313e921cd920906356f5d4febc4ed /.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py
parentcc961e04ba734dd72309fb548a2f97d67d578813 (diff)
downloadgn-ai-master.tar.gz
two version of R2R are hereHEADmaster
Diffstat (limited to '.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py')
-rw-r--r--.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py129
1 files changed, 129 insertions, 0 deletions
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py
new file mode 100644
index 00000000..61c12a9f
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py
@@ -0,0 +1,129 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to upload a large folder with the CLI."""
+
+import os
+from argparse import Namespace, _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import disable_progress_bars
+
+from ._cli_utils import ANSI
+
+
+logger = logging.get_logger(__name__)
+
+
+class UploadLargeFolderCommand(BaseHuggingfaceCLICommand):
+ @staticmethod
+ def register_subcommand(parser: _SubParsersAction):
+ subparser = parser.add_parser("upload-large-folder", help="Upload a large folder to a repo on the Hub")
+ subparser.add_argument(
+ "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
+ )
+ subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.")
+ subparser.add_argument(
+ "--repo-type",
+ choices=["model", "dataset", "space"],
+ help="Type of the repo to upload to (e.g. `dataset`).",
+ )
+ subparser.add_argument(
+ "--revision",
+ type=str,
+ help=("An optional Git revision to push to. It can be a branch name or a PR reference."),
+ )
+ subparser.add_argument(
+ "--private",
+ action="store_true",
+ help=(
+ "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists."
+ ),
+ )
+ subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
+ subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.")
+ subparser.add_argument(
+ "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
+ )
+ subparser.add_argument(
+ "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files."
+ )
+ subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.")
+ subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.")
+ subparser.set_defaults(func=UploadLargeFolderCommand)
+
+ def __init__(self, args: Namespace) -> None:
+ self.repo_id: str = args.repo_id
+ self.local_path: str = args.local_path
+ self.repo_type: str = args.repo_type
+ self.revision: Optional[str] = args.revision
+ self.private: bool = args.private
+
+ self.include: Optional[List[str]] = args.include
+ self.exclude: Optional[List[str]] = args.exclude
+
+ self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
+
+ self.num_workers: Optional[int] = args.num_workers
+ self.no_report: bool = args.no_report
+ self.no_bars: bool = args.no_bars
+
+ if not os.path.isdir(self.local_path):
+ raise ValueError("Large upload is only supported for folders.")
+
+ def run(self) -> None:
+ logging.set_verbosity_info()
+
+ print(
+ ANSI.yellow(
+ "You are about to upload a large folder to the Hub using `huggingface-cli upload-large-folder`. "
+ "This is a new feature so feedback is very welcome!\n"
+ "\n"
+ "A few things to keep in mind:\n"
+ " - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"
+ " - Do not start several processes in parallel.\n"
+ " - You can interrupt and resume the process at any time. "
+ "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"
+ " - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"
+ "\n"
+ f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n"
+ " - You must not modify those files manually.\n"
+ " - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"
+ " - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"
+ "\n"
+ "If the process output is too verbose, you can disable the progress bars with `--no-bars`. "
+ "You can also entirely disable the status report with `--no-report`.\n"
+ "\n"
+ "For more details, run `huggingface-cli upload-large-folder --help` or check the documentation at "
+ "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."
+ )
+ )
+
+ if self.no_bars:
+ disable_progress_bars()
+
+ self.api.upload_large_folder(
+ repo_id=self.repo_id,
+ folder_path=self.local_path,
+ repo_type=self.repo_type,
+ revision=self.revision,
+ private=self.private,
+ allow_patterns=self.include,
+ ignore_patterns=self.exclude,
+ num_workers=self.num_workers,
+ print_report=not self.no_report,
+ )