diff options
Diffstat (limited to '.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils')
3 files changed, 299 insertions, 0 deletions
diff --git a/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/__init__.py b/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/__init__.py diff --git a/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/stream.py b/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/stream.py new file mode 100644 index 00000000..7253d96e --- /dev/null +++ b/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/stream.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +"""Utilities for dealing with streamed requests.""" +import os.path +import re + +from .. import exceptions as exc + +# Regular expressions stolen from werkzeug/http.py +# cd2c97bb0a076da2322f11adce0b2731f9193396 L62-L64 +_QUOTED_STRING_RE = r'"[^"\\]*(?:\\.[^"\\]*)*"' +_OPTION_HEADER_PIECE_RE = re.compile( + r';\s*(%s|[^\s;=]+)\s*(?:=\s*(%s|[^;]+))?\s*' % (_QUOTED_STRING_RE, + _QUOTED_STRING_RE) +) +_DEFAULT_CHUNKSIZE = 512 + + +def _get_filename(content_disposition): + for match in _OPTION_HEADER_PIECE_RE.finditer(content_disposition): + k, v = match.groups() + if k == 'filename': + # ignore any directory paths in the filename + return os.path.split(v)[1] + return None + + +def get_download_file_path(response, path): + """ + Given a response and a path, return a file path for a download. + + If a ``path`` parameter is a directory, this function will parse the + ``Content-Disposition`` header on the response to determine the name of the + file as reported by the server, and return a file path in the specified + directory. + + If ``path`` is empty or None, this function will return a path relative + to the process' current working directory. + + If path is a full file path, return it. + + :param response: A Response object from requests + :type response: requests.models.Response + :param str path: Directory or file path. + :returns: full file path to download as + :rtype: str + :raises: :class:`requests_toolbelt.exceptions.StreamingError` + """ + path_is_dir = path and os.path.isdir(path) + + if path and not path_is_dir: + # fully qualified file path + filepath = path + else: + response_filename = _get_filename( + response.headers.get('content-disposition', '') + ) + if not response_filename: + raise exc.StreamingError('No filename given to stream response to') + + if path_is_dir: + # directory to download to + filepath = os.path.join(path, response_filename) + else: + # fallback to downloading to current working directory + filepath = response_filename + + return filepath + + +def stream_response_to_file(response, path=None, chunksize=_DEFAULT_CHUNKSIZE): + """Stream a response body to the specified file. + + Either use the ``path`` provided or use the name provided in the + ``Content-Disposition`` header. + + .. warning:: + + If you pass this function an open file-like object as the ``path`` + parameter, the function will not close that file for you. + + .. warning:: + + This function will not automatically close the response object + passed in as the ``response`` parameter. + + If a ``path`` parameter is a directory, this function will parse the + ``Content-Disposition`` header on the response to determine the name of the + file as reported by the server, and return a file path in the specified + directory. If no ``path`` parameter is supplied, this function will default + to the process' current working directory. + + .. code-block:: python + + import requests + from requests_toolbelt import exceptions + from requests_toolbelt.downloadutils import stream + + r = requests.get(url, stream=True) + try: + filename = stream.stream_response_to_file(r) + except exceptions.StreamingError as e: + # The toolbelt could not find the filename in the + # Content-Disposition + print(e.message) + + You can also specify the filename as a string. This will be passed to + the built-in :func:`open` and we will read the content into the file. + + .. code-block:: python + + import requests + from requests_toolbelt.downloadutils import stream + + r = requests.get(url, stream=True) + filename = stream.stream_response_to_file(r, path='myfile') + + If the calculated download file path already exists, this function will + raise a StreamingError. + + Instead, if you want to manage the file object yourself, you need to + provide either a :class:`io.BytesIO` object or a file opened with the + `'b'` flag. See the two examples below for more details. + + .. code-block:: python + + import requests + from requests_toolbelt.downloadutils import stream + + with open('myfile', 'wb') as fd: + r = requests.get(url, stream=True) + filename = stream.stream_response_to_file(r, path=fd) + + print('{} saved to {}'.format(url, filename)) + + .. code-block:: python + + import io + import requests + from requests_toolbelt.downloadutils import stream + + b = io.BytesIO() + r = requests.get(url, stream=True) + filename = stream.stream_response_to_file(r, path=b) + assert filename is None + + :param response: A Response object from requests + :type response: requests.models.Response + :param path: *(optional)*, Either a string with the path to the location + to save the response content, or a file-like object expecting bytes. + :type path: :class:`str`, or object with a :meth:`write` + :param int chunksize: (optional), Size of chunk to attempt to stream + (default 512B). + :returns: The name of the file, if one can be determined, else None + :rtype: str + :raises: :class:`requests_toolbelt.exceptions.StreamingError` + """ + pre_opened = False + fd = None + filename = None + if path and callable(getattr(path, 'write', None)): + pre_opened = True + fd = path + filename = getattr(fd, 'name', None) + else: + filename = get_download_file_path(response, path) + if os.path.exists(filename): + raise exc.StreamingError("File already exists: %s" % filename) + fd = open(filename, 'wb') + + for chunk in response.iter_content(chunk_size=chunksize): + fd.write(chunk) + + if not pre_opened: + fd.close() + + return filename diff --git a/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/tee.py b/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/tee.py new file mode 100644 index 00000000..ecc7d0cd --- /dev/null +++ b/.venv/lib/python3.12/site-packages/requests_toolbelt/downloadutils/tee.py @@ -0,0 +1,123 @@ +"""Tee function implementations.""" +import io + +_DEFAULT_CHUNKSIZE = 65536 + +__all__ = ['tee', 'tee_to_file', 'tee_to_bytearray'] + + +def _tee(response, callback, chunksize, decode_content): + for chunk in response.raw.stream(amt=chunksize, + decode_content=decode_content): + callback(chunk) + yield chunk + + +def tee(response, fileobject, chunksize=_DEFAULT_CHUNKSIZE, + decode_content=None): + """Stream the response both to the generator and a file. + + This will stream the response body while writing the bytes to + ``fileobject``. + + Example usage: + + .. code-block:: python + + resp = requests.get(url, stream=True) + with open('save_file', 'wb') as save_file: + for chunk in tee(resp, save_file): + # do stuff with chunk + + .. code-block:: python + + import io + + resp = requests.get(url, stream=True) + fileobject = io.BytesIO() + + for chunk in tee(resp, fileobject): + # do stuff with chunk + + :param response: Response from requests. + :type response: requests.Response + :param fileobject: Writable file-like object. + :type fileobject: file, io.BytesIO + :param int chunksize: (optional), Size of chunk to attempt to stream. + :param bool decode_content: (optional), If True, this will decode the + compressed content of the response. + :raises: TypeError if the fileobject wasn't opened with the right mode + or isn't a BytesIO object. + """ + # We will be streaming the raw bytes from over the wire, so we need to + # ensure that writing to the fileobject will preserve those bytes. On + # Python3, if the user passes an io.StringIO, this will fail, so we need + # to check for BytesIO instead. + if not ('b' in getattr(fileobject, 'mode', '') or + isinstance(fileobject, io.BytesIO)): + raise TypeError('tee() will write bytes directly to this fileobject' + ', it must be opened with the "b" flag if it is a file' + ' or inherit from io.BytesIO.') + + return _tee(response, fileobject.write, chunksize, decode_content) + + +def tee_to_file(response, filename, chunksize=_DEFAULT_CHUNKSIZE, + decode_content=None): + """Stream the response both to the generator and a file. + + This will open a file named ``filename`` and stream the response body + while writing the bytes to the opened file object. + + Example usage: + + .. code-block:: python + + resp = requests.get(url, stream=True) + for chunk in tee_to_file(resp, 'save_file'): + # do stuff with chunk + + :param response: Response from requests. + :type response: requests.Response + :param str filename: Name of file in which we write the response content. + :param int chunksize: (optional), Size of chunk to attempt to stream. + :param bool decode_content: (optional), If True, this will decode the + compressed content of the response. + """ + with open(filename, 'wb') as fd: + for chunk in tee(response, fd, chunksize, decode_content): + yield chunk + + +def tee_to_bytearray(response, bytearr, chunksize=_DEFAULT_CHUNKSIZE, + decode_content=None): + """Stream the response both to the generator and a bytearray. + + This will stream the response provided to the function, add them to the + provided :class:`bytearray` and yield them to the user. + + .. note:: + + This uses the :meth:`bytearray.extend` by default instead of passing + the bytearray into the ``readinto`` method. + + Example usage: + + .. code-block:: python + + b = bytearray() + resp = requests.get(url, stream=True) + for chunk in tee_to_bytearray(resp, b): + # do stuff with chunk + + :param response: Response from requests. + :type response: requests.Response + :param bytearray bytearr: Array to add the streamed bytes to. + :param int chunksize: (optional), Size of chunk to attempt to stream. + :param bool decode_content: (optional), If True, this will decode the + compressed content of the response. + """ + if not isinstance(bytearr, bytearray): + raise TypeError('tee_to_bytearray() expects bytearr to be a ' + 'bytearray') + return _tee(response, bytearr.extend, chunksize, decode_content) |