diff options
Diffstat (limited to '.venv/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py')
| -rw-r--r-- | .venv/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py | 279 |
1 files changed, 279 insertions, 0 deletions
diff --git a/.venv/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py b/.venv/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py new file mode 100644 index 00000000..b22b797c --- /dev/null +++ b/.venv/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py @@ -0,0 +1,279 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""Utility helpers to handle progress bars in `huggingface_hub`. + +Example: + 1. Use `huggingface_hub.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`. + 2. To disable progress bars, either use `disable_progress_bars()` helper or set the + environment variable `HF_HUB_DISABLE_PROGRESS_BARS` to 1. + 3. To re-enable progress bars, use `enable_progress_bars()`. + 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`. + +NOTE: Environment variable `HF_HUB_DISABLE_PROGRESS_BARS` has the priority. + +Example: + ```py + >>> from huggingface_hub.utils import are_progress_bars_disabled, disable_progress_bars, enable_progress_bars, tqdm + + # Disable progress bars globally + >>> disable_progress_bars() + + # Use as normal `tqdm` + >>> for _ in tqdm(range(5)): + ... pass + + # Still not showing progress bars, as `disable=False` is overwritten to `True`. + >>> for _ in tqdm(range(5), disable=False): + ... pass + + >>> are_progress_bars_disabled() + True + + # Re-enable progress bars globally + >>> enable_progress_bars() + + # Progress bar will be shown ! + >>> for _ in tqdm(range(5)): + ... pass + 100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s] + ``` + +Group-based control: + ```python + # Disable progress bars for a specific group + >>> disable_progress_bars("peft.foo") + + # Check state of different groups + >>> assert not are_progress_bars_disabled("peft")) + >>> assert not are_progress_bars_disabled("peft.something") + >>> assert are_progress_bars_disabled("peft.foo")) + >>> assert are_progress_bars_disabled("peft.foo.bar")) + + # Enable progress bars for a subgroup + >>> enable_progress_bars("peft.foo.bar") + + # Check if enabling a subgroup affects the parent group + >>> assert are_progress_bars_disabled("peft.foo")) + >>> assert not are_progress_bars_disabled("peft.foo.bar")) + + # No progress bar for `name="peft.foo"` + >>> for _ in tqdm(range(5), name="peft.foo"): + ... pass + + # Progress bar will be shown for `name="peft.foo.bar"` + >>> for _ in tqdm(range(5), name="peft.foo.bar"): + ... pass + 100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s] + + ``` +""" + +import io +import logging +import os +import warnings +from contextlib import contextmanager +from pathlib import Path +from typing import Dict, Iterator, Optional, Union + +from tqdm.auto import tqdm as old_tqdm + +from ..constants import HF_HUB_DISABLE_PROGRESS_BARS + + +# The `HF_HUB_DISABLE_PROGRESS_BARS` environment variable can be True, False, or not set (None), +# allowing for control over progress bar visibility. When set, this variable takes precedence +# over programmatic settings, dictating whether progress bars should be shown or hidden globally. +# Essentially, the environment variable's setting overrides any code-based configurations. +# +# If `HF_HUB_DISABLE_PROGRESS_BARS` is not defined (None), it implies that users can manage +# progress bar visibility through code. By default, progress bars are turned on. + + +progress_bar_states: Dict[str, bool] = {} + + +def disable_progress_bars(name: Optional[str] = None) -> None: + """ + Disable progress bars either globally or for a specified group. + + This function updates the state of progress bars based on a group name. + If no group name is provided, all progress bars are disabled. The operation + respects the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable's setting. + + Args: + name (`str`, *optional*): + The name of the group for which to disable the progress bars. If None, + progress bars are disabled globally. + + Raises: + Warning: If the environment variable precludes changes. + """ + if HF_HUB_DISABLE_PROGRESS_BARS is False: + warnings.warn( + "Cannot disable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=0` is set and has priority." + ) + return + + if name is None: + progress_bar_states.clear() + progress_bar_states["_global"] = False + else: + keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")] + for key in keys_to_remove: + del progress_bar_states[key] + progress_bar_states[name] = False + + +def enable_progress_bars(name: Optional[str] = None) -> None: + """ + Enable progress bars either globally or for a specified group. + + This function sets the progress bars to enabled for the specified group or globally + if no group is specified. The operation is subject to the `HF_HUB_DISABLE_PROGRESS_BARS` + environment setting. + + Args: + name (`str`, *optional*): + The name of the group for which to enable the progress bars. If None, + progress bars are enabled globally. + + Raises: + Warning: If the environment variable precludes changes. + """ + if HF_HUB_DISABLE_PROGRESS_BARS is True: + warnings.warn( + "Cannot enable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=1` is set and has priority." + ) + return + + if name is None: + progress_bar_states.clear() + progress_bar_states["_global"] = True + else: + keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")] + for key in keys_to_remove: + del progress_bar_states[key] + progress_bar_states[name] = True + + +def are_progress_bars_disabled(name: Optional[str] = None) -> bool: + """ + Check if progress bars are disabled globally or for a specific group. + + This function returns whether progress bars are disabled for a given group or globally. + It checks the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable first, then the programmatic + settings. + + Args: + name (`str`, *optional*): + The group name to check; if None, checks the global setting. + + Returns: + `bool`: True if progress bars are disabled, False otherwise. + """ + if HF_HUB_DISABLE_PROGRESS_BARS is True: + return True + + if name is None: + return not progress_bar_states.get("_global", True) + + while name: + if name in progress_bar_states: + return not progress_bar_states[name] + name = ".".join(name.split(".")[:-1]) + + return not progress_bar_states.get("_global", True) + + +def is_tqdm_disabled(log_level: int) -> Optional[bool]: + """ + Determine if tqdm progress bars should be disabled based on logging level and environment settings. + + see https://github.com/huggingface/huggingface_hub/pull/2000 and https://github.com/huggingface/huggingface_hub/pull/2698. + """ + if log_level == logging.NOTSET: + return True + if os.getenv("TQDM_POSITION") == "-1": + return False + return None + + +class tqdm(old_tqdm): + """ + Class to override `disable` argument in case progress bars are globally disabled. + + Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324. + """ + + def __init__(self, *args, **kwargs): + name = kwargs.pop("name", None) # do not pass `name` to `tqdm` + if are_progress_bars_disabled(name): + kwargs["disable"] = True + super().__init__(*args, **kwargs) + + def __delattr__(self, attr: str) -> None: + """Fix for https://github.com/huggingface/huggingface_hub/issues/1603""" + try: + super().__delattr__(attr) + except AttributeError: + if attr != "_lock": + raise + + +@contextmanager +def tqdm_stream_file(path: Union[Path, str]) -> Iterator[io.BufferedReader]: + """ + Open a file as binary and wrap the `read` method to display a progress bar when it's streamed. + + First implemented in `transformers` in 2019 but removed when switched to git-lfs. Used in `huggingface_hub` to show + progress bar when uploading an LFS file to the Hub. See github.com/huggingface/transformers/pull/2078#discussion_r354739608 + for implementation details. + + Note: currently implementation handles only files stored on disk as it is the most common use case. Could be + extended to stream any `BinaryIO` object but we might have to debug some corner cases. + + Example: + ```py + >>> with tqdm_stream_file("config.json") as f: + >>> requests.put(url, data=f) + config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s] + ``` + """ + if isinstance(path, str): + path = Path(path) + + with path.open("rb") as f: + total_size = path.stat().st_size + pbar = tqdm( + unit="B", + unit_scale=True, + total=total_size, + initial=0, + desc=path.name, + ) + + f_read = f.read + + def _inner_read(size: Optional[int] = -1) -> bytes: + data = f_read(size) + pbar.update(len(data)) + return data + + f.read = _inner_read # type: ignore + + yield f + + pbar.close() |