two version of R2R are hereHEADmaster

37 files changed, 7061 insertions, 0 deletions

diff --git a/.venv/lib/python3.12/site-packages/litellm/types/adapter.py b/.venv/lib/python3.12/site-packages/litellm/types/adapter.py
new file mode 100644
index 00000000..2995cfbc
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/adapter.py
@@ -0,0 +1,10 @@
+from typing import List
+
+from typing_extensions import Dict, Required, TypedDict, override
+
+from litellm.integrations.custom_logger import CustomLogger
+
+
+class AdapterItem(TypedDict):
+    id: str
+    adapter: CustomLogger
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/caching.py b/.venv/lib/python3.12/site-packages/litellm/types/caching.py
new file mode 100644
index 00000000..c15d4d15
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/caching.py
@@ -0,0 +1,78 @@
+from enum import Enum
+from typing import Any, Dict, Literal, Optional, TypedDict, Union
+
+from pydantic import BaseModel
+
+
+class LiteLLMCacheType(str, Enum):
+    LOCAL = "local"
+    REDIS = "redis"
+    REDIS_SEMANTIC = "redis-semantic"
+    S3 = "s3"
+    DISK = "disk"
+    QDRANT_SEMANTIC = "qdrant-semantic"
+
+
+CachingSupportedCallTypes = Literal[
+    "completion",
+    "acompletion",
+    "embedding",
+    "aembedding",
+    "atranscription",
+    "transcription",
+    "atext_completion",
+    "text_completion",
+    "arerank",
+    "rerank",
+]
+
+
+class RedisPipelineIncrementOperation(TypedDict):
+    """
+    TypeDict for 1 Redis Pipeline Increment Operation
+    """
+
+    key: str
+    increment_value: float
+    ttl: Optional[int]
+
+
+DynamicCacheControl = TypedDict(
+    "DynamicCacheControl",
+    {
+        # Will cache the response for the user-defined amount of time (in seconds).
+        "ttl": Optional[int],
+        # Namespace to use for caching
+        "namespace": Optional[str],
+        # Max Age to use for caching
+        "s-maxage": Optional[int],
+        "s-max-age": Optional[int],
+        # Will not return a cached response, but instead call the actual endpoint.
+        "no-cache": Optional[bool],
+        # Will not store the response in the cache.
+        "no-store": Optional[bool],
+    },
+)
+
+
+class CachePingResponse(BaseModel):
+    status: str
+    cache_type: str
+    ping_response: Optional[bool] = None
+    set_cache_response: Optional[str] = None
+    litellm_cache_params: Optional[str] = None
+
+    # intentionally a dict, since we run masker.mask_dict() on HealthCheckCacheParams
+    health_check_cache_params: Optional[dict] = None
+
+
+class HealthCheckCacheParams(BaseModel):
+    """
+    Cache Params returned on /cache/ping call
+    """
+
+    host: Optional[str] = None
+    port: Optional[Union[str, int]] = None
+    redis_kwargs: Optional[Dict[str, Any]] = None
+    namespace: Optional[str] = None
+    redis_version: Optional[Union[str, int, float]] = None
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/completion.py b/.venv/lib/python3.12/site-packages/litellm/types/completion.py
new file mode 100644
index 00000000..b06bb733
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/completion.py
@@ -0,0 +1,193 @@
+from typing import Iterable, List, Optional, Union
+
+from pydantic import BaseModel, ConfigDict
+from typing_extensions import Literal, Required, TypedDict
+
+
+class ChatCompletionSystemMessageParam(TypedDict, total=False):
+    content: Required[str]
+    """The contents of the system message."""
+
+    role: Required[Literal["system"]]
+    """The role of the messages author, in this case `system`."""
+
+    name: str
+    """An optional name for the participant.
+
+    Provides the model information to differentiate between participants of the same
+    role.
+    """
+
+
+class ChatCompletionContentPartTextParam(TypedDict, total=False):
+    text: Required[str]
+    """The text content."""
+
+    type: Required[Literal["text"]]
+    """The type of the content part."""
+
+
+class ImageURL(TypedDict, total=False):
+    url: Required[str]
+    """Either a URL of the image or the base64 encoded image data."""
+
+    detail: Literal["auto", "low", "high"]
+    """Specifies the detail level of the image.
+
+    Learn more in the
+    [Vision guide](https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding).
+    """
+
+
+class ChatCompletionContentPartImageParam(TypedDict, total=False):
+    image_url: Required[ImageURL]
+
+    type: Required[Literal["image_url"]]
+    """The type of the content part."""
+
+
+ChatCompletionContentPartParam = Union[
+    ChatCompletionContentPartTextParam, ChatCompletionContentPartImageParam
+]
+
+
+class ChatCompletionUserMessageParam(TypedDict, total=False):
+    content: Required[Union[str, Iterable[ChatCompletionContentPartParam]]]
+    """The contents of the user message."""
+
+    role: Required[Literal["user"]]
+    """The role of the messages author, in this case `user`."""
+
+    name: str
+    """An optional name for the participant.
+
+    Provides the model information to differentiate between participants of the same
+    role.
+    """
+
+
+class FunctionCall(TypedDict, total=False):
+    arguments: Required[str]
+    """
+    The arguments to call the function with, as generated by the model in JSON
+    format. Note that the model does not always generate valid JSON, and may
+    hallucinate parameters not defined by your function schema. Validate the
+    arguments in your code before calling your function.
+    """
+
+    name: Required[str]
+    """The name of the function to call."""
+
+
+class Function(TypedDict, total=False):
+    arguments: Required[str]
+    """
+    The arguments to call the function with, as generated by the model in JSON
+    format. Note that the model does not always generate valid JSON, and may
+    hallucinate parameters not defined by your function schema. Validate the
+    arguments in your code before calling your function.
+    """
+
+    name: Required[str]
+    """The name of the function to call."""
+
+
+class ChatCompletionToolMessageParam(TypedDict, total=False):
+    content: Required[Union[str, Iterable[ChatCompletionContentPartParam]]]
+    """The contents of the tool message."""
+
+    role: Required[Literal["tool"]]
+    """The role of the messages author, in this case `tool`."""
+
+    tool_call_id: Required[str]
+    """Tool call that this message is responding to."""
+
+
+class ChatCompletionFunctionMessageParam(TypedDict, total=False):
+    content: Required[Union[str, Iterable[ChatCompletionContentPartParam]]]
+    """The contents of the function message."""
+
+    name: Required[str]
+    """The name of the function to call."""
+
+    role: Required[Literal["function"]]
+    """The role of the messages author, in this case `function`."""
+
+
+class ChatCompletionMessageToolCallParam(TypedDict, total=False):
+    id: Required[str]
+    """The ID of the tool call."""
+
+    function: Required[Function]
+    """The function that the model called."""
+
+    type: Required[Literal["function"]]
+    """The type of the tool. Currently, only `function` is supported."""
+
+
+class ChatCompletionAssistantMessageParam(TypedDict, total=False):
+    role: Required[Literal["assistant"]]
+    """The role of the messages author, in this case `assistant`."""
+
+    content: Optional[str]
+    """The contents of the assistant message.
+
+    Required unless `tool_calls` or `function_call` is specified.
+    """
+
+    function_call: FunctionCall
+    """Deprecated and replaced by `tool_calls`.
+
+    The name and arguments of a function that should be called, as generated by the
+    model.
+    """
+
+    name: str
+    """An optional name for the participant.
+
+    Provides the model information to differentiate between participants of the same
+    role.
+    """
+
+    tool_calls: Iterable[ChatCompletionMessageToolCallParam]
+    """The tool calls generated by the model, such as function calls."""
+
+
+ChatCompletionMessageParam = Union[
+    ChatCompletionSystemMessageParam,
+    ChatCompletionUserMessageParam,
+    ChatCompletionAssistantMessageParam,
+    ChatCompletionFunctionMessageParam,
+    ChatCompletionToolMessageParam,
+]
+
+
+class CompletionRequest(BaseModel):
+    model: str
+    messages: List[str] = []
+    timeout: Optional[Union[float, int]] = None
+    temperature: Optional[float] = None
+    top_p: Optional[float] = None
+    n: Optional[int] = None
+    stream: Optional[bool] = None
+    stop: Optional[dict] = None
+    max_tokens: Optional[int] = None
+    presence_penalty: Optional[float] = None
+    frequency_penalty: Optional[float] = None
+    logit_bias: Optional[dict] = None
+    user: Optional[str] = None
+    response_format: Optional[dict] = None
+    seed: Optional[int] = None
+    tools: Optional[List[str]] = None
+    tool_choice: Optional[str] = None
+    logprobs: Optional[bool] = None
+    top_logprobs: Optional[int] = None
+    deployment_id: Optional[str] = None
+    functions: Optional[List[str]] = None
+    function_call: Optional[str] = None
+    base_url: Optional[str] = None
+    api_version: Optional[str] = None
+    api_key: Optional[str] = None
+    model_list: Optional[List[str]] = None
+
+    model_config = ConfigDict(protected_namespaces=(), extra="allow")
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/embedding.py b/.venv/lib/python3.12/site-packages/litellm/types/embedding.py
new file mode 100644
index 00000000..f8fdebc5
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/embedding.py
@@ -0,0 +1,21 @@
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, ConfigDict
+
+
+class EmbeddingRequest(BaseModel):
+    model: str
+    input: List[str] = []
+    timeout: int = 600
+    api_base: Optional[str] = None
+    api_version: Optional[str] = None
+    api_key: Optional[str] = None
+    api_type: Optional[str] = None
+    caching: bool = False
+    user: Optional[str] = None
+    custom_llm_provider: Optional[Union[str, dict]] = None
+    litellm_call_id: Optional[str] = None
+    litellm_logging_obj: Optional[dict] = None
+    logger_fn: Optional[str] = None
+
+    model_config = ConfigDict(extra="allow")
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/files.py b/.venv/lib/python3.12/site-packages/litellm/types/files.py
new file mode 100644
index 00000000..600ad806
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/files.py
@@ -0,0 +1,283 @@
+from enum import Enum
+from types import MappingProxyType
+from typing import List, Set, Mapping
+
+"""
+Base Enums/Consts
+"""
+
+
+class FileType(Enum):
+    AAC = "AAC"
+    CSV = "CSV"
+    DOC = "DOC"
+    DOCX = "DOCX"
+    FLAC = "FLAC"
+    FLV = "FLV"
+    GIF = "GIF"
+    GOOGLE_DOC = "GOOGLE_DOC"
+    GOOGLE_DRAWINGS = "GOOGLE_DRAWINGS"
+    GOOGLE_SHEETS = "GOOGLE_SHEETS"
+    GOOGLE_SLIDES = "GOOGLE_SLIDES"
+    HEIC = "HEIC"
+    HEIF = "HEIF"
+    HTML = "HTML"
+    JPEG = "JPEG"
+    JSON = "JSON"
+    M4A = "M4A"
+    M4V = "M4V"
+    MOV = "MOV"
+    MP3 = "MP3"
+    MP4 = "MP4"
+    MPEG = "MPEG"
+    MPEGPS = "MPEGPS"
+    MPG = "MPG"
+    MPA = "MPA"
+    MPGA = "MPGA"
+    OGG = "OGG"
+    OPUS = "OPUS"
+    PDF = "PDF"
+    PCM = "PCM"
+    PNG = "PNG"
+    PPT = "PPT"
+    PPTX = "PPTX"
+    RTF = "RTF"
+    THREE_GPP = "3GPP"
+    TXT = "TXT"
+    WAV = "WAV"
+    WEBM = "WEBM"
+    WEBP = "WEBP"
+    WMV = "WMV"
+    XLS = "XLS"
+    XLSX = "XLSX"
+
+
+FILE_EXTENSIONS: Mapping[FileType, List[str]] = MappingProxyType(
+    {
+        FileType.AAC: ["aac"],
+        FileType.CSV: ["csv"],
+        FileType.DOC: ["doc"],
+        FileType.DOCX: ["docx"],
+        FileType.FLAC: ["flac"],
+        FileType.FLV: ["flv"],
+        FileType.GIF: ["gif"],
+        FileType.GOOGLE_DOC: ["gdoc"],
+        FileType.GOOGLE_DRAWINGS: ["gdraw"],
+        FileType.GOOGLE_SHEETS: ["gsheet"],
+        FileType.GOOGLE_SLIDES: ["gslides"],
+        FileType.HEIC: ["heic"],
+        FileType.HEIF: ["heif"],
+        FileType.HTML: ["html", "htm"],
+        FileType.JPEG: ["jpeg", "jpg"],
+        FileType.JSON: ["json"],
+        FileType.M4A: ["m4a"],
+        FileType.M4V: ["m4v"],
+        FileType.MOV: ["mov"],
+        FileType.MP3: ["mp3"],
+        FileType.MP4: ["mp4"],
+        FileType.MPEG: ["mpeg"],
+        FileType.MPEGPS: ["mpegps"],
+        FileType.MPG: ["mpg"],
+        FileType.MPA: ["mpa"],
+        FileType.MPGA: ["mpga"],
+        FileType.OGG: ["ogg"],
+        FileType.OPUS: ["opus"],
+        FileType.PDF: ["pdf"],
+        FileType.PCM: ["pcm"],
+        FileType.PNG: ["png"],
+        FileType.PPT: ["ppt"],
+        FileType.PPTX: ["pptx"],
+        FileType.RTF: ["rtf"],
+        FileType.THREE_GPP: ["3gpp"],
+        FileType.TXT: ["txt"],
+        FileType.WAV: ["wav"],
+        FileType.WEBM: ["webm"],
+        FileType.WEBP: ["webp"],
+        FileType.WMV: ["wmv"],
+        FileType.XLS: ["xls"],
+        FileType.XLSX: ["xlsx"],
+    }
+)
+
+FILE_MIME_TYPES: Mapping[FileType, str] = MappingProxyType(
+    {
+        FileType.AAC: "audio/aac",
+        FileType.CSV: "text/csv",
+        FileType.DOC: "application/msword",
+        FileType.DOCX: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        FileType.FLAC: "audio/flac",
+        FileType.FLV: "video/x-flv",
+        FileType.GIF: "image/gif",
+        FileType.GOOGLE_DOC: "application/vnd.google-apps.document",
+        FileType.GOOGLE_DRAWINGS: "application/vnd.google-apps.drawing",
+        FileType.GOOGLE_SHEETS: "application/vnd.google-apps.spreadsheet",
+        FileType.GOOGLE_SLIDES: "application/vnd.google-apps.presentation",
+        FileType.HEIC: "image/heic",
+        FileType.HEIF: "image/heif",
+        FileType.HTML: "text/html",
+        FileType.JPEG: "image/jpeg",
+        FileType.JSON: "application/json",
+        FileType.M4A: "audio/x-m4a",
+        FileType.M4V: "video/x-m4v",
+        FileType.MOV: "video/quicktime",
+        FileType.MP3: "audio/mpeg",
+        FileType.MP4: "video/mp4",
+        FileType.MPEG: "video/mpeg",
+        FileType.MPEGPS: "video/mpegps",
+        FileType.MPG: "video/mpg",
+        FileType.MPA: "audio/m4a",
+        FileType.MPGA: "audio/mpga",
+        FileType.OGG: "audio/ogg",
+        FileType.OPUS: "audio/opus",
+        FileType.PDF: "application/pdf",
+        FileType.PCM: "audio/pcm",
+        FileType.PNG: "image/png",
+        FileType.PPT: "application/vnd.ms-powerpoint",
+        FileType.PPTX: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        FileType.RTF: "application/rtf",
+        FileType.THREE_GPP: "video/3gpp",
+        FileType.TXT: "text/plain",
+        FileType.WAV: "audio/wav",
+        FileType.WEBM: "video/webm",
+        FileType.WEBP: "image/webp",
+        FileType.WMV: "video/wmv",
+        FileType.XLS: "application/vnd.ms-excel",
+        FileType.XLSX: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    }
+)
+
+"""
+Util Functions
+"""
+
+
+def get_file_extension_from_mime_type(mime_type: str) -> str:
+    for file_type, mime in FILE_MIME_TYPES.items():
+        if mime.lower() == mime_type.lower():
+            return FILE_EXTENSIONS[file_type][0]
+    raise ValueError(f"Unknown extension for mime type: {mime_type}")
+
+
+def get_file_type_from_extension(extension: str) -> FileType:
+    for file_type, extensions in FILE_EXTENSIONS.items():
+        if extension.lower() in extensions:
+            return file_type
+
+    raise ValueError(f"Unknown file type for extension: {extension}")
+
+
+def get_file_extension_for_file_type(file_type: FileType) -> str:
+    return FILE_EXTENSIONS[file_type][0]
+
+
+def get_file_mime_type_for_file_type(file_type: FileType) -> str:
+    return FILE_MIME_TYPES[file_type]
+
+
+def get_file_mime_type_from_extension(extension: str) -> str:
+    file_type = get_file_type_from_extension(extension)
+    return get_file_mime_type_for_file_type(file_type)
+
+
+"""
+FileType Type Groupings (Videos, Images, etc)
+"""
+
+# Images
+IMAGE_FILE_TYPES = {
+    FileType.PNG,
+    FileType.JPEG,
+    FileType.GIF,
+    FileType.WEBP,
+    FileType.HEIC,
+    FileType.HEIF,
+}
+
+
+def is_image_file_type(file_type):
+    return file_type in IMAGE_FILE_TYPES
+
+
+# Videos
+VIDEO_FILE_TYPES = {
+    FileType.MOV,
+    FileType.MP4,
+    FileType.MPEG,
+    FileType.M4V,
+    FileType.FLV,
+    FileType.MPEGPS,
+    FileType.MPG,
+    FileType.WEBM,
+    FileType.WMV,
+    FileType.THREE_GPP,
+}
+
+
+def is_video_file_type(file_type):
+    return file_type in VIDEO_FILE_TYPES
+
+
+# Audio
+AUDIO_FILE_TYPES = {
+    FileType.AAC,
+    FileType.FLAC,
+    FileType.MP3,
+    FileType.MPA,
+    FileType.MPGA,
+    FileType.OPUS,
+    FileType.PCM,
+    FileType.WAV,
+}
+
+
+def is_audio_file_type(file_type):
+    return file_type in AUDIO_FILE_TYPES
+
+
+# Text
+TEXT_FILE_TYPES = {FileType.CSV, FileType.HTML, FileType.RTF, FileType.TXT}
+
+
+def is_text_file_type(file_type):
+    return file_type in TEXT_FILE_TYPES
+
+
+"""
+Other FileType Groupings
+"""
+# Accepted file types for GEMINI 1.5 through Vertex AI
+# https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-multimodal-prompts#gemini-send-multimodal-samples-images-nodejs
+GEMINI_1_5_ACCEPTED_FILE_TYPES: Set[FileType] = {
+    # Image
+    FileType.PNG,
+    FileType.JPEG,
+    FileType.WEBP,
+    # Audio
+    FileType.AAC,
+    FileType.FLAC,
+    FileType.MP3,
+    FileType.MPA,
+    FileType.MPEG,
+    FileType.MPGA,
+    FileType.OPUS,
+    FileType.PCM,
+    FileType.WAV,
+    FileType.WEBM,
+    # Video
+    FileType.FLV,
+    FileType.MOV,
+    FileType.MPEG,
+    FileType.MPEGPS,
+    FileType.MPG,
+    FileType.MP4,
+    FileType.WEBM,
+    FileType.WMV,
+    FileType.THREE_GPP,
+    # PDF
+    FileType.PDF,
+    FileType.TXT,
+}
+
+
+def is_gemini_1_5_accepted_file_type(file_type: FileType) -> bool:
+    return file_type in GEMINI_1_5_ACCEPTED_FILE_TYPES
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/fine_tuning.py b/.venv/lib/python3.12/site-packages/litellm/types/fine_tuning.py
new file mode 100644
index 00000000..af99d88c
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/fine_tuning.py
@@ -0,0 +1,5 @@
+from openai.types.fine_tuning.fine_tuning_job import Hyperparameters
+
+
+class OpenAIFineTuningHyperparameters(Hyperparameters):
+    model_config = {"extra": "allow"}
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/guardrails.py b/.venv/lib/python3.12/site-packages/litellm/types/guardrails.py
new file mode 100644
index 00000000..b7018fe2
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/guardrails.py
@@ -0,0 +1,168 @@
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
+
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
+from typing_extensions import Required, TypedDict
+
+"""
+Pydantic object defining how to set guardrails on litellm proxy
+
+guardrails:
+  - guardrail_name: "bedrock-pre-guard"
+    litellm_params:
+      guardrail: bedrock  # supported values: "aporia", "bedrock", "lakera"
+      mode: "during_call"
+      guardrailIdentifier: ff6ujrregl1q
+      guardrailVersion: "DRAFT"
+      default_on: true
+"""
+
+
+class SupportedGuardrailIntegrations(Enum):
+    APORIA = "aporia"
+    BEDROCK = "bedrock"
+    GURDRAILS_AI = "guardrails_ai"
+    LAKERA = "lakera"
+    PRESIDIO = "presidio"
+    HIDE_SECRETS = "hide-secrets"
+    AIM = "aim"
+
+
+class Role(Enum):
+    SYSTEM = "system"
+    ASSISTANT = "assistant"
+    USER = "user"
+
+
+default_roles = [Role.SYSTEM, Role.ASSISTANT, Role.USER]
+
+
+class GuardrailItemSpec(TypedDict, total=False):
+    callbacks: Required[List[str]]
+    default_on: bool
+    logging_only: Optional[bool]
+    enabled_roles: Optional[List[Role]]
+    callback_args: Dict[str, Dict]
+
+
+class GuardrailItem(BaseModel):
+    callbacks: List[str]
+    default_on: bool
+    logging_only: Optional[bool]
+    guardrail_name: str
+    callback_args: Dict[str, Dict]
+    enabled_roles: Optional[List[Role]]
+
+    model_config = ConfigDict(use_enum_values=True)
+
+    def __init__(
+        self,
+        callbacks: List[str],
+        guardrail_name: str,
+        default_on: bool = False,
+        logging_only: Optional[bool] = None,
+        enabled_roles: Optional[List[Role]] = default_roles,
+        callback_args: Dict[str, Dict] = {},
+    ):
+        super().__init__(
+            callbacks=callbacks,
+            default_on=default_on,
+            logging_only=logging_only,
+            guardrail_name=guardrail_name,
+            enabled_roles=enabled_roles,
+            callback_args=callback_args,
+        )
+
+
+# Define the TypedDicts
+class LakeraCategoryThresholds(TypedDict, total=False):
+    prompt_injection: float
+    jailbreak: float
+
+
+class LitellmParams(TypedDict):
+    guardrail: str
+    mode: str
+    api_key: Optional[str]
+    api_base: Optional[str]
+
+    # Lakera specific params
+    category_thresholds: Optional[LakeraCategoryThresholds]
+
+    # Bedrock specific params
+    guardrailIdentifier: Optional[str]
+    guardrailVersion: Optional[str]
+
+    # Presidio params
+    output_parse_pii: Optional[bool]
+    presidio_ad_hoc_recognizers: Optional[str]
+    mock_redacted_text: Optional[dict]
+
+    # hide secrets params
+    detect_secrets_config: Optional[dict]
+
+    # guardrails ai params
+    guard_name: Optional[str]
+    default_on: Optional[bool]
+
+
+class Guardrail(TypedDict, total=False):
+    guardrail_name: str
+    litellm_params: LitellmParams
+    guardrail_info: Optional[Dict]
+
+
+class guardrailConfig(TypedDict):
+    guardrails: List[Guardrail]
+
+
+class GuardrailEventHooks(str, Enum):
+    pre_call = "pre_call"
+    post_call = "post_call"
+    during_call = "during_call"
+    logging_only = "logging_only"
+
+
+class BedrockTextContent(TypedDict, total=False):
+    text: str
+
+
+class BedrockContentItem(TypedDict, total=False):
+    text: BedrockTextContent
+
+
+class BedrockRequest(TypedDict, total=False):
+    source: Literal["INPUT", "OUTPUT"]
+    content: List[BedrockContentItem]
+
+
+class DynamicGuardrailParams(TypedDict):
+    extra_body: Dict[str, Any]
+
+
+class GuardrailLiteLLMParamsResponse(BaseModel):
+    """The returned LiteLLM Params object for /guardrails/list"""
+
+    guardrail: str
+    mode: Union[str, List[str]]
+    default_on: bool = Field(default=False)
+
+    def __init__(self, **kwargs):
+        default_on = kwargs.get("default_on")
+        if default_on is None:
+            default_on = False
+
+        super().__init__(**kwargs)
+
+
+class GuardrailInfoResponse(BaseModel):
+    guardrail_name: str
+    litellm_params: GuardrailLiteLLMParamsResponse
+    guardrail_info: Optional[Dict]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+
+class ListGuardrailsResponse(BaseModel):
+    guardrails: List[GuardrailInfoResponse]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/argilla.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/argilla.py
new file mode 100644
index 00000000..6c0de762
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/argilla.py
@@ -0,0 +1,21 @@
+import os
+from datetime import datetime as dt
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional, Set, TypedDict
+
+
+class ArgillaItem(TypedDict):
+    fields: Dict[str, Any]
+
+
+class ArgillaPayload(TypedDict):
+    items: List[ArgillaItem]
+
+
+class ArgillaCredentialsObject(TypedDict):
+    ARGILLA_API_KEY: str
+    ARGILLA_DATASET_NAME: str
+    ARGILLA_BASE_URL: str
+
+
+SUPPORTED_PAYLOAD_FIELDS = ["messages", "response"]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/arize.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/arize.py
new file mode 100644
index 00000000..e1ec1755
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/arize.py
@@ -0,0 +1,15 @@
+from typing import TYPE_CHECKING, Any, Literal, Optional
+
+from pydantic import BaseModel
+
+if TYPE_CHECKING:
+    Protocol = Literal["otlp_grpc", "otlp_http"]
+else:
+    Protocol = Any
+
+
+class ArizeConfig(BaseModel):
+    space_key: Optional[str] = None
+    api_key: Optional[str] = None
+    protocol: Protocol
+    endpoint: str
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/arize_phoenix.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/arize_phoenix.py
new file mode 100644
index 00000000..4566022d
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/arize_phoenix.py
@@ -0,0 +1,9 @@
+from typing import TYPE_CHECKING, Literal, Optional
+
+from pydantic import BaseModel
+from .arize import Protocol
+
+class ArizePhoenixConfig(BaseModel):
+    otlp_auth_headers: Optional[str] = None
+    protocol: Protocol
+    endpoint: str 
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/base_health_check.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/base_health_check.py
new file mode 100644
index 00000000..b69529d1
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/base_health_check.py
@@ -0,0 +1,6 @@
+from typing import Literal, Optional, TypedDict
+
+
+class IntegrationHealthCheckStatus(TypedDict):
+    status: Literal["healthy", "unhealthy"]
+    error_message: Optional[str]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/datadog.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/datadog.py
new file mode 100644
index 00000000..79d4eded
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/datadog.py
@@ -0,0 +1,29 @@
+from enum import Enum
+from typing import Optional, TypedDict
+
+
+class DataDogStatus(str, Enum):
+    INFO = "info"
+    WARN = "warning"
+    ERROR = "error"
+
+
+class DatadogPayload(TypedDict, total=False):
+    ddsource: str
+    ddtags: str
+    hostname: str
+    message: str
+    service: str
+    status: str
+
+
+class DD_ERRORS(Enum):
+    DATADOG_413_ERROR = "Datadog API Error - Payload too large (batch is above 5MB uncompressed). If you want this logged either disable request/response logging or set `DD_BATCH_SIZE=50`"
+
+
+class DatadogProxyFailureHookJsonMessage(TypedDict, total=False):
+    exception: str
+    error_class: str
+    status_code: Optional[int]
+    traceback: str
+    user_api_key_dict: dict
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/datadog_llm_obs.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/datadog_llm_obs.py
new file mode 100644
index 00000000..9298b157
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/datadog_llm_obs.py
@@ -0,0 +1,54 @@
+"""
+Payloads for Datadog LLM Observability Service (LLMObs)
+
+API Reference: https://docs.datadoghq.com/llm_observability/setup/api/?tab=example#api-standards
+"""
+
+from typing import Any, Dict, List, Literal, Optional, TypedDict
+
+
+class InputMeta(TypedDict):
+    messages: List[Any]
+
+
+class OutputMeta(TypedDict):
+    messages: List[Any]
+
+
+class Meta(TypedDict):
+    # The span kind: "agent", "workflow", "llm", "tool", "task", "embedding", or "retrieval".
+    kind: Literal["llm", "tool", "task", "embedding", "retrieval"]
+    input: InputMeta  # The span’s input information.
+    output: OutputMeta  # The span’s output information.
+    metadata: Dict[str, Any]
+
+
+class LLMMetrics(TypedDict, total=False):
+    input_tokens: float
+    output_tokens: float
+    total_tokens: float
+    time_to_first_token: float
+    time_per_output_token: float
+
+
+class LLMObsPayload(TypedDict):
+    parent_id: str
+    trace_id: str
+    span_id: str
+    name: str
+    meta: Meta
+    start_ns: int
+    duration: int
+    metrics: LLMMetrics
+    tags: List
+
+
+class DDSpanAttributes(TypedDict):
+    ml_app: str
+    tags: List[str]
+    spans: List[LLMObsPayload]
+
+
+class DDIntakePayload(TypedDict):
+    type: str
+    attributes: DDSpanAttributes
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/gcs_bucket.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/gcs_bucket.py
new file mode 100644
index 00000000..a4fd8a6a
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/gcs_bucket.py
@@ -0,0 +1,28 @@
+from typing import TYPE_CHECKING, Any, Dict, Optional, TypedDict
+
+from litellm.types.utils import StandardLoggingPayload
+
+if TYPE_CHECKING:
+    from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+else:
+    VertexBase = Any
+
+
+class GCSLoggingConfig(TypedDict):
+    """
+    Internal LiteLLM Config for GCS Bucket logging
+    """
+
+    bucket_name: str
+    vertex_instance: VertexBase
+    path_service_account: Optional[str]
+
+
+class GCSLogQueueItem(TypedDict):
+    """
+    Internal Type, used for queueing logs to be sent to GCS Bucket
+    """
+
+    payload: StandardLoggingPayload
+    kwargs: Dict[str, Any]
+    response_obj: Optional[Any]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/langfuse.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/langfuse.py
new file mode 100644
index 00000000..ecf42d8c
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/langfuse.py
@@ -0,0 +1,7 @@
+from typing import Optional, TypedDict
+
+
+class LangfuseLoggingConfig(TypedDict):
+    langfuse_secret: Optional[str]
+    langfuse_public_key: Optional[str]
+    langfuse_host: Optional[str]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/langsmith.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/langsmith.py
new file mode 100644
index 00000000..48c8e2e0
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/langsmith.py
@@ -0,0 +1,61 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Dict, List, NamedTuple, Optional, TypedDict
+
+from pydantic import BaseModel
+
+
+class LangsmithInputs(BaseModel):
+    model: Optional[str] = None
+    messages: Optional[List[Any]] = None
+    stream: Optional[bool] = None
+    call_type: Optional[str] = None
+    litellm_call_id: Optional[str] = None
+    completion_start_time: Optional[datetime] = None
+    temperature: Optional[float] = None
+    max_tokens: Optional[int] = None
+    custom_llm_provider: Optional[str] = None
+    input: Optional[List[Any]] = None
+    log_event_type: Optional[str] = None
+    original_response: Optional[Any] = None
+    response_cost: Optional[float] = None
+
+    # LiteLLM Virtual Key specific fields
+    user_api_key: Optional[str] = None
+    user_api_key_user_id: Optional[str] = None
+    user_api_key_team_alias: Optional[str] = None
+
+
+class LangsmithCredentialsObject(TypedDict):
+    LANGSMITH_API_KEY: str
+    LANGSMITH_PROJECT: str
+    LANGSMITH_BASE_URL: str
+
+
+class LangsmithQueueObject(TypedDict):
+    """
+    Langsmith Queue Object - this is what gets stored in the internal system queue before flushing to Langsmith
+
+    We need to store:
+        - data[Dict] - data that should get logged on langsmith
+        - credentials[LangsmithCredentialsObject] - credentials to use for logging to langsmith
+    """
+
+    data: Dict
+    credentials: LangsmithCredentialsObject
+
+
+class CredentialsKey(NamedTuple):
+    """Immutable key for grouping credentials"""
+
+    api_key: str
+    project: str
+    base_url: str
+
+
+@dataclass
+class BatchGroup:
+    """Groups credentials with their associated queue objects"""
+
+    credentials: LangsmithCredentialsObject
+    queue_objects: List[LangsmithQueueObject]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/pagerduty.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/pagerduty.py
new file mode 100644
index 00000000..22fd1665
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/pagerduty.py
@@ -0,0 +1,62 @@
+from datetime import datetime
+from typing import List, Literal, Optional, TypedDict, Union
+
+from litellm.types.utils import StandardLoggingUserAPIKeyMetadata
+
+
+class LinkDict(TypedDict, total=False):
+    href: str
+    text: Optional[str]
+
+
+class ImageDict(TypedDict, total=False):
+    src: str
+    href: Optional[str]
+    alt: Optional[str]
+
+
+class PagerDutyPayload(TypedDict, total=False):
+    summary: str
+    timestamp: Optional[str]  # ISO 8601 date-time format
+    severity: Literal["critical", "warning", "error", "info"]
+    source: str
+    component: Optional[str]
+    group: Optional[str]
+    class_: Optional[str]  # Using class_ since 'class' is a reserved keyword
+    custom_details: Optional[dict]
+
+
+class PagerDutyRequestBody(TypedDict, total=False):
+    payload: PagerDutyPayload
+    routing_key: str
+    event_action: Literal["trigger", "acknowledge", "resolve"]
+    dedup_key: Optional[str]
+    client: Optional[str]
+    client_url: Optional[str]
+    links: Optional[List[LinkDict]]
+    images: Optional[List[ImageDict]]
+
+
+class AlertingConfig(TypedDict, total=False):
+    """
+    Config for alerting thresholds
+    """
+
+    # Requests failing threshold
+    failure_threshold: int  # Number of requests failing in a window
+    failure_threshold_window_seconds: int  # Window in seconds
+
+    # Requests hanging threshold
+    hanging_threshold_seconds: float  # Number of seconds of waiting for a response before a request is considered hanging
+    hanging_threshold_fails: int  # Number of requests hanging in a window
+    hanging_threshold_window_seconds: int  # Window in seconds
+
+
+class PagerDutyInternalEvent(StandardLoggingUserAPIKeyMetadata, total=False):
+    """Simple structure to hold timestamp and error info."""
+
+    failure_event_type: Literal["failed_response", "hanging_response"]
+    timestamp: datetime
+    error_class: Optional[str]
+    error_code: Optional[str]
+    error_llm_provider: Optional[str]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/prometheus.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/prometheus.py
new file mode 100644
index 00000000..8fdcce4c
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/prometheus.py
@@ -0,0 +1,294 @@
+from enum import Enum
+from typing import Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+from typing_extensions import Annotated
+
+import litellm
+
+REQUESTED_MODEL = "requested_model"
+EXCEPTION_STATUS = "exception_status"
+EXCEPTION_CLASS = "exception_class"
+STATUS_CODE = "status_code"
+EXCEPTION_LABELS = [EXCEPTION_STATUS, EXCEPTION_CLASS]
+LATENCY_BUCKETS = (
+    0.005,
+    0.00625,
+    0.0125,
+    0.025,
+    0.05,
+    0.1,
+    0.5,
+    1.0,
+    1.5,
+    2.0,
+    2.5,
+    3.0,
+    3.5,
+    4.0,
+    4.5,
+    5.0,
+    5.5,
+    6.0,
+    6.5,
+    7.0,
+    7.5,
+    8.0,
+    8.5,
+    9.0,
+    9.5,
+    10.0,
+    15.0,
+    20.0,
+    25.0,
+    30.0,
+    60.0,
+    120.0,
+    180.0,
+    240.0,
+    300.0,
+    float("inf"),
+)
+
+
+class UserAPIKeyLabelNames(Enum):
+    END_USER = "end_user"
+    USER = "user"
+    USER_EMAIL = "user_email"
+    API_KEY_HASH = "hashed_api_key"
+    API_KEY_ALIAS = "api_key_alias"
+    TEAM = "team"
+    TEAM_ALIAS = "team_alias"
+    REQUESTED_MODEL = REQUESTED_MODEL
+    v1_LITELLM_MODEL_NAME = "model"
+    v2_LITELLM_MODEL_NAME = "litellm_model_name"
+    TAG = "tag"
+    MODEL_ID = "model_id"
+    API_BASE = "api_base"
+    API_PROVIDER = "api_provider"
+    EXCEPTION_STATUS = EXCEPTION_STATUS
+    EXCEPTION_CLASS = EXCEPTION_CLASS
+    STATUS_CODE = "status_code"
+    FALLBACK_MODEL = "fallback_model"
+
+
+DEFINED_PROMETHEUS_METRICS = Literal[
+    "litellm_llm_api_latency_metric",
+    "litellm_request_total_latency_metric",
+    "litellm_proxy_total_requests_metric",
+    "litellm_proxy_failed_requests_metric",
+    "litellm_deployment_latency_per_output_token",
+    "litellm_requests_metric",
+    "litellm_input_tokens_metric",
+    "litellm_output_tokens_metric",
+    "litellm_deployment_successful_fallbacks",
+    "litellm_deployment_failed_fallbacks",
+    "litellm_remaining_team_budget_metric",
+    "litellm_team_max_budget_metric",
+    "litellm_team_budget_remaining_hours_metric",
+    "litellm_remaining_api_key_budget_metric",
+    "litellm_api_key_max_budget_metric",
+    "litellm_api_key_budget_remaining_hours_metric",
+]
+
+
+class PrometheusMetricLabels:
+    litellm_llm_api_latency_metric = [
+        UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+        UserAPIKeyLabelNames.REQUESTED_MODEL.value,
+        UserAPIKeyLabelNames.END_USER.value,
+        UserAPIKeyLabelNames.USER.value,
+    ]
+
+    litellm_request_total_latency_metric = [
+        UserAPIKeyLabelNames.END_USER.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.REQUESTED_MODEL.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+        UserAPIKeyLabelNames.USER.value,
+        UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
+    ]
+
+    litellm_proxy_total_requests_metric = [
+        UserAPIKeyLabelNames.END_USER.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.REQUESTED_MODEL.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+        UserAPIKeyLabelNames.USER.value,
+        UserAPIKeyLabelNames.STATUS_CODE.value,
+        UserAPIKeyLabelNames.USER_EMAIL.value,
+    ]
+
+    litellm_proxy_failed_requests_metric = [
+        UserAPIKeyLabelNames.END_USER.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.REQUESTED_MODEL.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+        UserAPIKeyLabelNames.USER.value,
+        UserAPIKeyLabelNames.EXCEPTION_STATUS.value,
+        UserAPIKeyLabelNames.EXCEPTION_CLASS.value,
+    ]
+
+    litellm_deployment_latency_per_output_token = [
+        UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
+        UserAPIKeyLabelNames.MODEL_ID.value,
+        UserAPIKeyLabelNames.API_BASE.value,
+        UserAPIKeyLabelNames.API_PROVIDER.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+    ]
+
+    litellm_requests_metric = [
+        UserAPIKeyLabelNames.END_USER.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+        UserAPIKeyLabelNames.USER.value,
+        UserAPIKeyLabelNames.USER_EMAIL.value,
+    ]
+
+    litellm_input_tokens_metric = [
+        UserAPIKeyLabelNames.END_USER.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+        UserAPIKeyLabelNames.USER.value,
+        UserAPIKeyLabelNames.REQUESTED_MODEL.value,
+    ]
+
+    litellm_output_tokens_metric = [
+        UserAPIKeyLabelNames.END_USER.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+        UserAPIKeyLabelNames.USER.value,
+        UserAPIKeyLabelNames.REQUESTED_MODEL.value,
+    ]
+
+    litellm_deployment_successful_fallbacks = [
+        UserAPIKeyLabelNames.REQUESTED_MODEL.value,
+        UserAPIKeyLabelNames.FALLBACK_MODEL.value,
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+        UserAPIKeyLabelNames.EXCEPTION_STATUS.value,
+        UserAPIKeyLabelNames.EXCEPTION_CLASS.value,
+    ]
+
+    litellm_deployment_failed_fallbacks = litellm_deployment_successful_fallbacks
+
+    litellm_remaining_team_budget_metric = [
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+    ]
+
+    litellm_team_max_budget_metric = [
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+    ]
+
+    litellm_team_budget_remaining_hours_metric = [
+        UserAPIKeyLabelNames.TEAM.value,
+        UserAPIKeyLabelNames.TEAM_ALIAS.value,
+    ]
+
+    litellm_remaining_api_key_budget_metric = [
+        UserAPIKeyLabelNames.API_KEY_HASH.value,
+        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
+    ]
+
+    litellm_api_key_max_budget_metric = litellm_remaining_api_key_budget_metric
+
+    litellm_api_key_budget_remaining_hours_metric = (
+        litellm_remaining_api_key_budget_metric
+    )
+
+    @staticmethod
+    def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]:
+        default_labels = getattr(PrometheusMetricLabels, label_name)
+        return default_labels + [
+            metric.replace(".", "_")
+            for metric in litellm.custom_prometheus_metadata_labels
+        ]
+
+
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class UserAPIKeyLabelValues(BaseModel):
+    end_user: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.END_USER.value)
+    ] = None
+    user: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.USER.value)
+    ] = None
+    user_email: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.USER_EMAIL.value)
+    ] = None
+    hashed_api_key: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.API_KEY_HASH.value)
+    ] = None
+    api_key_alias: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.API_KEY_ALIAS.value)
+    ] = None
+    team: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.TEAM.value)
+    ] = None
+    team_alias: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.TEAM_ALIAS.value)
+    ] = None
+    requested_model: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.REQUESTED_MODEL.value)
+    ] = None
+    model: Annotated[
+        Optional[str],
+        Field(..., alias=UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value),
+    ] = None
+    litellm_model_name: Annotated[
+        Optional[str],
+        Field(..., alias=UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value),
+    ] = None
+    tags: List[str] = []
+    custom_metadata_labels: Dict[str, str] = {}
+    model_id: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.MODEL_ID.value)
+    ] = None
+    api_base: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.API_BASE.value)
+    ] = None
+    api_provider: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.API_PROVIDER.value)
+    ] = None
+    exception_status: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.EXCEPTION_STATUS.value)
+    ] = None
+    exception_class: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.EXCEPTION_CLASS.value)
+    ] = None
+    status_code: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.STATUS_CODE.value)
+    ] = None
+    fallback_model: Annotated[
+        Optional[str], Field(..., alias=UserAPIKeyLabelNames.FALLBACK_MODEL.value)
+    ] = None
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/integrations/slack_alerting.py b/.venv/lib/python3.12/site-packages/litellm/types/integrations/slack_alerting.py
new file mode 100644
index 00000000..9019b098
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/integrations/slack_alerting.py
@@ -0,0 +1,186 @@
+import os
+from datetime import datetime as dt
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional, Set, TypedDict
+
+from pydantic import BaseModel, Field
+
+from litellm.types.utils import LiteLLMPydanticObjectBase
+
+
+class BaseOutageModel(TypedDict):
+    alerts: List[int]
+    minor_alert_sent: bool
+    major_alert_sent: bool
+    last_updated_at: float
+
+
+class OutageModel(BaseOutageModel):
+    model_id: str
+
+
+class ProviderRegionOutageModel(BaseOutageModel):
+    provider_region_id: str
+    deployment_ids: Set[str]
+
+
+# we use this for the email header, please send a test email if you change this. verify it looks good on email
+LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
+LITELLM_SUPPORT_CONTACT = "support@berri.ai"
+
+
+class SlackAlertingArgsEnum(Enum):
+    daily_report_frequency = 12 * 60 * 60
+    report_check_interval = 5 * 60
+    budget_alert_ttl = 24 * 60 * 60
+    outage_alert_ttl = 1 * 60
+    region_outage_alert_ttl = 1 * 60
+    minor_outage_alert_threshold = 1 * 5
+    major_outage_alert_threshold = 1 * 10
+    max_outage_alert_list_size = 1 * 10
+
+
+class SlackAlertingArgs(LiteLLMPydanticObjectBase):
+    daily_report_frequency: int = Field(
+        default=int(
+            os.getenv(
+                "SLACK_DAILY_REPORT_FREQUENCY",
+                int(SlackAlertingArgsEnum.daily_report_frequency.value),
+            )
+        ),
+        description="Frequency of receiving deployment latency/failure reports. Default is 12hours. Value is in seconds.",
+    )
+    report_check_interval: int = Field(
+        default=SlackAlertingArgsEnum.report_check_interval.value,
+        description="Frequency of checking cache if report should be sent. Background process. Default is once per hour. Value is in seconds.",
+    )  # 5 minutes
+    budget_alert_ttl: int = Field(
+        default=SlackAlertingArgsEnum.budget_alert_ttl.value,
+        description="Cache ttl for budgets alerts. Prevents spamming same alert, each time budget is crossed. Value is in seconds.",
+    )  # 24 hours
+    outage_alert_ttl: int = Field(
+        default=SlackAlertingArgsEnum.outage_alert_ttl.value,
+        description="Cache ttl for model outage alerts. Sets time-window for errors. Default is 1 minute. Value is in seconds.",
+    )  # 1 minute ttl
+    region_outage_alert_ttl: int = Field(
+        default=SlackAlertingArgsEnum.region_outage_alert_ttl.value,
+        description="Cache ttl for provider-region based outage alerts. Alert sent if 2+ models in same region report errors. Sets time-window for errors. Default is 1 minute. Value is in seconds.",
+    )  # 1 minute ttl
+    minor_outage_alert_threshold: int = Field(
+        default=SlackAlertingArgsEnum.minor_outage_alert_threshold.value,
+        description="The number of errors that count as a model/region minor outage. ('400' error code is not counted).",
+    )
+    major_outage_alert_threshold: int = Field(
+        default=SlackAlertingArgsEnum.major_outage_alert_threshold.value,
+        description="The number of errors that countas a model/region major outage. ('400' error code is not counted).",
+    )
+    max_outage_alert_list_size: int = Field(
+        default=SlackAlertingArgsEnum.max_outage_alert_list_size.value,
+        description="Maximum number of errors to store in cache. For a given model/region. Prevents memory leaks.",
+    )  # prevent memory leak
+    log_to_console: bool = Field(
+        default=False,
+        description="If true, the alerting payload will be printed to the console.",
+    )
+
+
+class DeploymentMetrics(LiteLLMPydanticObjectBase):
+    """
+    Metrics per deployment, stored in cache
+
+    Used for daily reporting
+    """
+
+    id: str
+    """id of deployment in router model list"""
+
+    failed_request: bool
+    """did it fail the request?"""
+
+    latency_per_output_token: Optional[float]
+    """latency/output token of deployment"""
+
+    updated_at: dt
+    """Current time of deployment being updated"""
+
+
+class SlackAlertingCacheKeys(Enum):
+    """
+    Enum for deployment daily metrics keys - {deployment_id}:{enum}
+    """
+
+    failed_requests_key = "failed_requests_daily_metrics"
+    latency_key = "latency_daily_metrics"
+    report_sent_key = "daily_metrics_report_sent"
+
+
+class AlertType(str, Enum):
+    """
+    Enum for alert types and management event types
+    """
+
+    # LLM-related alerts
+    llm_exceptions = "llm_exceptions"
+    llm_too_slow = "llm_too_slow"
+    llm_requests_hanging = "llm_requests_hanging"
+
+    # Budget and spend alerts
+    budget_alerts = "budget_alerts"
+    spend_reports = "spend_reports"
+    failed_tracking_spend = "failed_tracking_spend"
+
+    # Database alerts
+    db_exceptions = "db_exceptions"
+
+    # Report alerts
+    daily_reports = "daily_reports"
+
+    # Deployment alerts
+    cooldown_deployment = "cooldown_deployment"
+    new_model_added = "new_model_added"
+
+    # Outage alerts
+    outage_alerts = "outage_alerts"
+    region_outage_alerts = "region_outage_alerts"
+
+    # Fallback alerts
+    fallback_reports = "fallback_reports"
+
+    # Virtual Key Events
+    new_virtual_key_created = "new_virtual_key_created"
+    virtual_key_updated = "virtual_key_updated"
+    virtual_key_deleted = "virtual_key_deleted"
+
+    # Team Events
+    new_team_created = "new_team_created"
+    team_updated = "team_updated"
+    team_deleted = "team_deleted"
+
+    # Internal User Events
+    new_internal_user_created = "new_internal_user_created"
+    internal_user_updated = "internal_user_updated"
+    internal_user_deleted = "internal_user_deleted"
+
+
+DEFAULT_ALERT_TYPES: List[AlertType] = [
+    # LLM related alerts
+    AlertType.llm_exceptions,
+    AlertType.llm_too_slow,
+    AlertType.llm_requests_hanging,
+    # Budget and spend alerts
+    AlertType.budget_alerts,
+    AlertType.spend_reports,
+    AlertType.failed_tracking_spend,
+    # Database alerts
+    AlertType.db_exceptions,
+    # Report alerts
+    AlertType.daily_reports,
+    # Deployment alerts
+    AlertType.cooldown_deployment,
+    AlertType.new_model_added,
+    # Outage alerts
+    AlertType.outage_alerts,
+    AlertType.region_outage_alerts,
+    # Fallback alerts
+    AlertType.fallback_reports,
+]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/anthropic.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/anthropic.py
new file mode 100644
index 00000000..367b2421
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/anthropic.py
@@ -0,0 +1,366 @@
+from typing import Any, Dict, Iterable, List, Optional, Union
+
+from pydantic import BaseModel, validator
+from typing_extensions import Literal, Required, TypedDict
+
+from .openai import ChatCompletionCachedContent, ChatCompletionThinkingBlock
+
+
+class AnthropicMessagesToolChoice(TypedDict, total=False):
+    type: Required[Literal["auto", "any", "tool"]]
+    name: str
+    disable_parallel_tool_use: bool  # default is false
+
+
+class AnthropicInputSchema(TypedDict, total=False):
+    type: Optional[str]
+    properties: Optional[dict]
+    additionalProperties: Optional[bool]
+
+
+class AnthropicMessagesTool(TypedDict, total=False):
+    name: Required[str]
+    description: str
+    input_schema: Optional[AnthropicInputSchema]
+    type: Literal["custom"]
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+
+
+class AnthropicComputerTool(TypedDict, total=False):
+    display_width_px: Required[int]
+    display_height_px: Required[int]
+    display_number: int
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+    type: Required[str]
+    name: Required[str]
+
+
+class AnthropicHostedTools(TypedDict, total=False):  # for bash_tool and text_editor
+    type: Required[str]
+    name: Required[str]
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+
+
+AllAnthropicToolsValues = Union[
+    AnthropicComputerTool, AnthropicHostedTools, AnthropicMessagesTool
+]
+
+
+class AnthropicMessagesTextParam(TypedDict, total=False):
+    type: Required[Literal["text"]]
+    text: Required[str]
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+
+
+class AnthropicMessagesToolUseParam(TypedDict):
+    type: Required[Literal["tool_use"]]
+    id: str
+    name: str
+    input: dict
+
+
+AnthropicMessagesAssistantMessageValues = Union[
+    AnthropicMessagesTextParam,
+    AnthropicMessagesToolUseParam,
+    ChatCompletionThinkingBlock,
+]
+
+
+class AnthopicMessagesAssistantMessageParam(TypedDict, total=False):
+    content: Required[Union[str, Iterable[AnthropicMessagesAssistantMessageValues]]]
+    """The contents of the system message."""
+
+    role: Required[Literal["assistant"]]
+    """The role of the messages author, in this case `author`."""
+
+    name: str
+    """An optional name for the participant.
+
+    Provides the model information to differentiate between participants of the same
+    role.
+    """
+
+
+class AnthropicContentParamSource(TypedDict):
+    type: Literal["base64"]
+    media_type: str
+    data: str
+
+
+class AnthropicMessagesImageParam(TypedDict, total=False):
+    type: Required[Literal["image"]]
+    source: Required[AnthropicContentParamSource]
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+
+
+class CitationsObject(TypedDict):
+    enabled: bool
+
+
+class AnthropicMessagesDocumentParam(TypedDict, total=False):
+    type: Required[Literal["document"]]
+    source: Required[AnthropicContentParamSource]
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+    title: str
+    context: str
+    citations: Optional[CitationsObject]
+
+
+class AnthropicMessagesToolResultContent(TypedDict):
+    type: Literal["text"]
+    text: str
+
+
+class AnthropicMessagesToolResultParam(TypedDict, total=False):
+    type: Required[Literal["tool_result"]]
+    tool_use_id: Required[str]
+    is_error: bool
+    content: Union[
+        str,
+        Iterable[
+            Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]
+        ],
+    ]
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+
+
+AnthropicMessagesUserMessageValues = Union[
+    AnthropicMessagesTextParam,
+    AnthropicMessagesImageParam,
+    AnthropicMessagesToolResultParam,
+    AnthropicMessagesDocumentParam,
+]
+
+
+class AnthropicMessagesUserMessageParam(TypedDict, total=False):
+    role: Required[Literal["user"]]
+    content: Required[Union[str, Iterable[AnthropicMessagesUserMessageValues]]]
+
+
+class AnthropicMetadata(TypedDict, total=False):
+    user_id: str
+
+
+class AnthropicSystemMessageContent(TypedDict, total=False):
+    type: str
+    text: str
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+
+
+AllAnthropicMessageValues = Union[
+    AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam
+]
+
+
+class AnthropicMessageRequestBase(TypedDict, total=False):
+    messages: Required[List[AllAnthropicMessageValues]]
+    max_tokens: Required[int]
+    metadata: AnthropicMetadata
+    stop_sequences: List[str]
+    stream: bool
+    system: Union[str, List]
+    temperature: float
+    tool_choice: AnthropicMessagesToolChoice
+    tools: List[AllAnthropicToolsValues]
+    top_k: int
+    top_p: float
+
+
+class AnthropicMessagesRequest(AnthropicMessageRequestBase, total=False):
+    model: Required[str]
+    # litellm param - used for tracking litellm proxy metadata in the request
+    litellm_metadata: dict
+
+
+class ContentTextBlockDelta(TypedDict):
+    """
+    'delta': {'type': 'text_delta', 'text': 'Hello'}
+    """
+
+    type: str
+    text: str
+
+
+class ContentCitationsBlockDelta(TypedDict):
+    type: Literal["citations"]
+    citation: dict
+
+
+class ContentJsonBlockDelta(TypedDict):
+    """
+    "delta": {"type": "input_json_delta","partial_json": "{\"location\": \"San Fra"}}
+    """
+
+    type: str
+    partial_json: str
+
+
+class ContentBlockDelta(TypedDict):
+    type: Literal["content_block_delta"]
+    index: int
+    delta: Union[
+        ContentTextBlockDelta, ContentJsonBlockDelta, ContentCitationsBlockDelta
+    ]
+
+
+class ContentBlockStop(TypedDict):
+    type: Literal["content_block_stop"]
+    index: int
+
+
+class ToolUseBlock(TypedDict):
+    """
+    "content_block":{"type":"tool_use","id":"toolu_01T1x1fJ34qAmk2tNTrN7Up6","name":"get_weather","input":{}}
+    """
+
+    id: str
+
+    input: dict
+
+    name: str
+
+    type: Literal["tool_use"]
+
+
+class TextBlock(TypedDict):
+    text: str
+
+    type: Literal["text"]
+
+
+class ContentBlockStart(TypedDict):
+    """
+    event: content_block_start
+    data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01T1x1fJ34qAmk2tNTrN7Up6","name":"get_weather","input":{}}}
+    """
+
+    type: str
+    index: int
+    content_block: Union[ToolUseBlock, TextBlock]
+
+
+class MessageDelta(TypedDict, total=False):
+    stop_reason: Optional[str]
+
+
+class UsageDelta(TypedDict, total=False):
+    input_tokens: int
+    output_tokens: int
+
+
+class MessageBlockDelta(TypedDict):
+    """
+    Anthropic
+    chunk = {'type': 'message_delta', 'delta': {'stop_reason': 'max_tokens', 'stop_sequence': None}, 'usage': {'output_tokens': 10}}
+    """
+
+    type: Literal["message_delta"]
+    delta: MessageDelta
+    usage: UsageDelta
+
+
+class MessageChunk(TypedDict, total=False):
+    id: str
+    type: str
+    role: str
+    model: str
+    content: List
+    stop_reason: Optional[str]
+    stop_sequence: Optional[str]
+    usage: UsageDelta
+
+
+class MessageStartBlock(TypedDict):
+    """
+        Anthropic
+        chunk = {
+        "type": "message_start",
+        "message": {
+            "id": "msg_vrtx_011PqREFEMzd3REdCoUFAmdG",
+            "type": "message",
+            "role": "assistant",
+            "model": "claude-3-sonnet-20240229",
+            "content": [],
+            "stop_reason": null,
+            "stop_sequence": null,
+            "usage": {
+                "input_tokens": 270,
+                "output_tokens": 1
+            }
+        }
+    }
+    """
+
+    type: Literal["message_start"]
+    message: MessageChunk
+
+
+class AnthropicResponseContentBlockText(BaseModel):
+    type: Literal["text"]
+    text: str
+
+
+class AnthropicResponseContentBlockToolUse(BaseModel):
+    type: Literal["tool_use"]
+    id: str
+    name: str
+    input: dict
+
+
+class AnthropicResponseUsageBlock(BaseModel):
+    input_tokens: int
+    output_tokens: int
+
+
+AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"]
+
+
+class AnthropicResponse(BaseModel):
+    id: str
+    """Unique object identifier."""
+
+    type: Literal["message"]
+    """For Messages, this is always "message"."""
+
+    role: Literal["assistant"]
+    """Conversational role of the generated message. This will always be "assistant"."""
+
+    content: List[
+        Union[AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse]
+    ]
+    """Content generated by the model."""
+
+    model: str
+    """The model that handled the request."""
+
+    stop_reason: Optional[AnthropicFinishReason]
+    """The reason that we stopped."""
+
+    stop_sequence: Optional[str]
+    """Which custom stop sequence was generated, if any."""
+
+    usage: AnthropicResponseUsageBlock
+    """Billing and rate-limit usage."""
+
+
+from .openai import ChatCompletionUsageBlock
+
+
+class AnthropicChatCompletionUsageBlock(ChatCompletionUsageBlock, total=False):
+    cache_creation_input_tokens: int
+    cache_read_input_tokens: int
+
+
+ANTHROPIC_API_HEADERS = {
+    "anthropic-version",
+    "anthropic-beta",
+}
+
+ANTHROPIC_API_ONLY_HEADERS = {  # fails if calling anthropic on vertex ai / bedrock
+    "anthropic-beta",
+}
+
+
+class AnthropicThinkingParam(TypedDict, total=False):
+    type: Literal["enabled"]
+    budget_tokens: int
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/azure_ai.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/azure_ai.py
new file mode 100644
index 00000000..2d597aef
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/azure_ai.py
@@ -0,0 +1,17 @@
+from typing import Any, Dict, Iterable, List, Literal, Optional, Union
+
+from typing_extensions import Required, TypedDict
+
+
+class ImageEmbeddingInput(TypedDict, total=False):
+    image: Required[str]
+    text: str
+
+
+EncodingFormat = Literal["base64", "binary", "float", "int8", "ubinary", "uint8"]
+
+
+class ImageEmbeddingRequest(TypedDict, total=False):
+    input: Required[List[ImageEmbeddingInput]]
+    dimensions: int
+    encoding_format: EncodingFormat
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/bedrock.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/bedrock.py
new file mode 100644
index 00000000..57fb04c8
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/bedrock.py
@@ -0,0 +1,503 @@
+import json
+from typing import Any, List, Literal, Optional, TypedDict, Union
+
+from typing_extensions import (
+    TYPE_CHECKING,
+    Protocol,
+    Required,
+    Self,
+    TypeGuard,
+    get_origin,
+    override,
+    runtime_checkable,
+)
+
+from .openai import ChatCompletionToolCallChunk
+
+
+class CachePointBlock(TypedDict, total=False):
+    type: Literal["default"]
+
+
+class SystemContentBlock(TypedDict, total=False):
+    text: str
+    cachePoint: CachePointBlock
+
+
+class SourceBlock(TypedDict):
+    bytes: Optional[str]  # base 64 encoded string
+
+
+BedrockImageTypes = Literal["png", "jpeg", "gif", "webp"]
+
+
+class ImageBlock(TypedDict):
+    format: Union[BedrockImageTypes, str]
+    source: SourceBlock
+
+
+BedrockDocumentTypes = Literal[
+    "pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"
+]
+
+
+class DocumentBlock(TypedDict):
+    format: Union[BedrockDocumentTypes, str]
+    source: SourceBlock
+    name: str
+
+
+class ToolResultContentBlock(TypedDict, total=False):
+    image: ImageBlock
+    document: DocumentBlock
+    json: dict
+    text: str
+
+
+class ToolResultBlock(TypedDict, total=False):
+    content: Required[List[ToolResultContentBlock]]
+    toolUseId: Required[str]
+    status: Literal["success", "error"]
+
+
+class ToolUseBlock(TypedDict):
+    input: dict
+    name: str
+    toolUseId: str
+
+
+class BedrockConverseReasoningTextBlock(TypedDict, total=False):
+    text: Required[str]
+    signature: str
+
+
+class BedrockConverseReasoningContentBlock(TypedDict, total=False):
+    reasoningText: BedrockConverseReasoningTextBlock
+    redactedContent: str
+
+
+class BedrockConverseReasoningContentBlockDelta(TypedDict, total=False):
+    signature: str
+    redactedContent: str
+    text: str
+
+
+class ContentBlock(TypedDict, total=False):
+    text: str
+    image: ImageBlock
+    document: DocumentBlock
+    toolResult: ToolResultBlock
+    toolUse: ToolUseBlock
+    cachePoint: CachePointBlock
+    reasoningContent: BedrockConverseReasoningContentBlock
+
+
+class MessageBlock(TypedDict):
+    content: List[ContentBlock]
+    role: Literal["user", "assistant"]
+
+
+class ConverseMetricsBlock(TypedDict):
+    latencyMs: float  # time in ms
+
+
+class ConverseResponseOutputBlock(TypedDict):
+    message: Optional[MessageBlock]
+
+
+class ConverseTokenUsageBlock(TypedDict):
+    inputTokens: int
+    outputTokens: int
+    totalTokens: int
+    cacheReadInputTokenCount: int
+    cacheReadInputTokens: int
+    cacheWriteInputTokenCount: int
+    cacheWriteInputTokens: int
+
+
+class ConverseResponseBlock(TypedDict):
+    additionalModelResponseFields: dict
+    metrics: ConverseMetricsBlock
+    output: ConverseResponseOutputBlock
+    stopReason: (
+        str  # end_turn | tool_use | max_tokens | stop_sequence | content_filtered
+    )
+    usage: ConverseTokenUsageBlock
+
+
+class ToolInputSchemaBlock(TypedDict):
+    json: Optional[dict]
+
+
+class ToolSpecBlock(TypedDict, total=False):
+    inputSchema: Required[ToolInputSchemaBlock]
+    name: Required[str]
+    description: str
+
+
+class ToolBlock(TypedDict):
+    toolSpec: Optional[ToolSpecBlock]
+
+
+class SpecificToolChoiceBlock(TypedDict):
+    name: str
+
+
+class ToolChoiceValuesBlock(TypedDict, total=False):
+    any: dict
+    auto: dict
+    tool: SpecificToolChoiceBlock
+
+
+class ToolConfigBlock(TypedDict, total=False):
+    tools: Required[List[ToolBlock]]
+    toolChoice: Union[str, ToolChoiceValuesBlock]
+
+
+class GuardrailConfigBlock(TypedDict, total=False):
+    guardrailIdentifier: str
+    guardrailVersion: str
+    trace: Literal["enabled", "disabled"]
+
+
+class InferenceConfig(TypedDict, total=False):
+    maxTokens: int
+    stopSequences: List[str]
+    temperature: float
+    topP: float
+    topK: int
+
+
+class ToolBlockDeltaEvent(TypedDict):
+    input: str
+
+
+class ToolUseBlockStartEvent(TypedDict):
+    name: str
+    toolUseId: str
+
+
+class ContentBlockStartEvent(TypedDict, total=False):
+    toolUse: Optional[ToolUseBlockStartEvent]
+
+
+class ContentBlockDeltaEvent(TypedDict, total=False):
+    """
+    Either 'text' or 'toolUse' will be specified for Converse API streaming response.
+    """
+
+    text: str
+    toolUse: ToolBlockDeltaEvent
+    reasoningContent: BedrockConverseReasoningContentBlockDelta
+
+
+class CommonRequestObject(
+    TypedDict, total=False
+):  # common request object across sync + async flows
+    additionalModelRequestFields: dict
+    additionalModelResponseFieldPaths: List[str]
+    inferenceConfig: InferenceConfig
+    system: List[SystemContentBlock]
+    toolConfig: ToolConfigBlock
+    guardrailConfig: Optional[GuardrailConfigBlock]
+
+
+class RequestObject(CommonRequestObject, total=False):
+    messages: Required[List[MessageBlock]]
+
+
+class BedrockInvokeNovaRequest(TypedDict, total=False):
+    """
+    Request object for sending `nova` requests to `/bedrock/invoke/`
+    """
+
+    messages: List[MessageBlock]
+    inferenceConfig: InferenceConfig
+    system: List[SystemContentBlock]
+    toolConfig: ToolConfigBlock
+    guardrailConfig: Optional[GuardrailConfigBlock]
+
+
+class GenericStreamingChunk(TypedDict):
+    text: Required[str]
+    tool_use: Optional[ChatCompletionToolCallChunk]
+    is_finished: Required[bool]
+    finish_reason: Required[str]
+    usage: Optional[ConverseTokenUsageBlock]
+    index: int
+
+
+class Document(TypedDict):
+    title: str
+    snippet: str
+
+
+class ServerSentEvent:
+    def __init__(
+        self,
+        *,
+        event: Optional[str] = None,
+        data: Optional[str] = None,
+        id: Optional[str] = None,
+        retry: Optional[int] = None,
+    ) -> None:
+        if data is None:
+            data = ""
+
+        self._id = id
+        self._data = data
+        self._event = event or None
+        self._retry = retry
+
+    @property
+    def event(self) -> Optional[str]:
+        return self._event
+
+    @property
+    def id(self) -> Optional[str]:
+        return self._id
+
+    @property
+    def retry(self) -> Optional[int]:
+        return self._retry
+
+    @property
+    def data(self) -> str:
+        return self._data
+
+    def json(self) -> Any:
+        return json.loads(self.data)
+
+    @override
+    def __repr__(self) -> str:
+        return f"ServerSentEvent(event={self.event}, data={self.data}, id={self.id}, retry={self.retry})"
+
+
+COHERE_EMBEDDING_INPUT_TYPES = Literal[
+    "search_document", "search_query", "classification", "clustering", "image"
+]
+
+
+class CohereEmbeddingRequest(TypedDict, total=False):
+    texts: List[str]
+    images: List[str]
+    input_type: Required[COHERE_EMBEDDING_INPUT_TYPES]
+    truncate: Literal["NONE", "START", "END"]
+    embedding_types: Literal["float", "int8", "uint8", "binary", "ubinary"]
+
+
+class CohereEmbeddingRequestWithModel(CohereEmbeddingRequest):
+    model: Required[str]
+
+
+class CohereEmbeddingResponse(TypedDict):
+    embeddings: List[List[float]]
+    id: str
+    response_type: Literal["embedding_floats"]
+    texts: List[str]
+
+
+class AmazonTitanV2EmbeddingRequest(TypedDict):
+    inputText: str
+    dimensions: int
+    normalize: bool
+
+
+class AmazonTitanV2EmbeddingResponse(TypedDict):
+    embedding: List[float]
+    inputTextTokenCount: int
+
+
+class AmazonTitanG1EmbeddingRequest(TypedDict):
+    inputText: str
+
+
+class AmazonTitanG1EmbeddingResponse(TypedDict):
+    embedding: List[float]
+    inputTextTokenCount: int
+
+
+class AmazonTitanMultimodalEmbeddingConfig(TypedDict):
+    outputEmbeddingLength: Literal[256, 384, 1024]
+
+
+class AmazonTitanMultimodalEmbeddingRequest(TypedDict, total=False):
+    inputText: str
+    inputImage: str
+    embeddingConfig: AmazonTitanMultimodalEmbeddingConfig
+
+
+class AmazonTitanMultimodalEmbeddingResponse(TypedDict):
+    embedding: List[float]
+    inputTextTokenCount: int
+    message: str  # Specifies any errors that occur during generation.
+
+
+AmazonEmbeddingRequest = Union[
+    AmazonTitanMultimodalEmbeddingRequest,
+    AmazonTitanV2EmbeddingRequest,
+    AmazonTitanG1EmbeddingRequest,
+]
+
+
+class AmazonStability3TextToImageRequest(TypedDict, total=False):
+    """
+    Request for Amazon Stability 3 Text to Image API
+
+    Ref here: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-diffusion-3-text-image.html
+    """
+
+    prompt: str
+    aspect_ratio: Literal[
+        "16:9", "1:1", "21:9", "2:3", "3:2", "4:5", "5:4", "9:16", "9:21"
+    ]
+    mode: Literal["image-to-image", "text-to-image"]
+    output_format: Literal["JPEG", "PNG"]
+    seed: int
+    negative_prompt: str
+
+
+class AmazonStability3TextToImageResponse(TypedDict, total=False):
+    """
+    Response for Amazon Stability 3 Text to Image API
+
+    Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-diffusion-3-text-image.html
+    """
+
+    images: List[str]
+    seeds: List[str]
+    finish_reasons: List[str]
+
+
+class AmazonNovaCanvasRequestBase(TypedDict, total=False):
+    """
+    Base class for Amazon Nova Canvas API requests
+    """
+
+    pass
+
+
+class AmazonNovaCanvasImageGenerationConfig(TypedDict, total=False):
+    """
+    Config for Amazon Nova Canvas Text to Image API
+
+    Ref: https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html
+    """
+
+    cfgScale: int
+    seed: int
+    quality: Literal["standard", "premium"]
+    width: int
+    height: int
+    numberOfImages: int
+
+
+class AmazonNovaCanvasTextToImageParams(TypedDict, total=False):
+    """
+    Params for Amazon Nova Canvas Text to Image API
+    """
+
+    text: str
+    negativeText: str
+    controlStrength: float
+    controlMode: Literal["CANNY_EDIT", "SEGMENTATION"]
+    conditionImage: str
+
+
+class AmazonNovaCanvasTextToImageRequest(
+    AmazonNovaCanvasRequestBase, TypedDict, total=False
+):
+    """
+    Request for Amazon Nova Canvas Text to Image API
+
+    Ref: https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html
+    """
+
+    textToImageParams: AmazonNovaCanvasTextToImageParams
+    taskType: Literal["TEXT_IMAGE"]
+    imageGenerationConfig: AmazonNovaCanvasImageGenerationConfig
+
+
+class AmazonNovaCanvasTextToImageResponse(TypedDict, total=False):
+    """
+    Response for Amazon Nova Canvas Text to Image API
+
+    Ref: https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html
+    """
+
+    images: List[str]
+
+
+if TYPE_CHECKING:
+    from botocore.awsrequest import AWSPreparedRequest
+else:
+    AWSPreparedRequest = Any
+
+from pydantic import BaseModel
+
+
+class BedrockPreparedRequest(TypedDict):
+    """
+    Internal/Helper class for preparing the request for bedrock image generation
+    """
+
+    endpoint_url: str
+    prepped: AWSPreparedRequest
+    body: bytes
+    data: dict
+
+
+class BedrockRerankTextQuery(TypedDict):
+    text: str
+
+
+class BedrockRerankQuery(TypedDict):
+    textQuery: BedrockRerankTextQuery
+    type: Literal["TEXT"]
+
+
+class BedrockRerankModelConfiguration(TypedDict, total=False):
+    modelArn: Required[str]
+    modelConfiguration: dict
+
+
+class BedrockRerankBedrockRerankingConfiguration(TypedDict):
+    modelConfiguration: BedrockRerankModelConfiguration
+    numberOfResults: int
+
+
+class BedrockRerankConfiguration(TypedDict):
+    bedrockRerankingConfiguration: BedrockRerankBedrockRerankingConfiguration
+    type: Literal["BEDROCK_RERANKING_MODEL"]
+
+
+class BedrockRerankTextDocument(TypedDict, total=False):
+    text: str
+
+
+class BedrockRerankInlineDocumentSource(TypedDict, total=False):
+    jsonDocument: dict
+    textDocument: BedrockRerankTextDocument
+    type: Literal["TEXT", "JSON"]
+
+
+class BedrockRerankSource(TypedDict):
+    inlineDocumentSource: BedrockRerankInlineDocumentSource
+    type: Literal["INLINE"]
+
+
+class BedrockRerankRequest(TypedDict):
+    """
+    Request for Bedrock Rerank API
+    """
+
+    queries: List[BedrockRerankQuery]
+    rerankingConfiguration: BedrockRerankConfiguration
+    sources: List[BedrockRerankSource]
+
+
+class AmazonDeepSeekR1StreamingResponse(TypedDict):
+    generation: str
+    generation_token_count: int
+    stop_reason: Optional[str]
+    prompt_token_count: int
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/cohere.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/cohere.py
new file mode 100644
index 00000000..7112a242
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/cohere.py
@@ -0,0 +1,46 @@
+from typing import Iterable, List, Optional, Union
+
+from typing_extensions import Literal, Required, TypedDict
+
+
+class CallObject(TypedDict):
+    name: str
+    parameters: dict
+
+
+class ToolResultObject(TypedDict):
+    call: CallObject
+    outputs: List[dict]
+
+
+class ChatHistoryToolResult(TypedDict, total=False):
+    role: Required[Literal["TOOL"]]
+    tool_results: List[ToolResultObject]
+
+
+class ToolCallObject(TypedDict):
+    name: str
+    parameters: dict
+
+
+class ChatHistoryUser(TypedDict, total=False):
+    role: Required[Literal["USER"]]
+    message: str
+    tool_calls: List[ToolCallObject]
+
+
+class ChatHistorySystem(TypedDict, total=False):
+    role: Required[Literal["SYSTEM"]]
+    message: str
+    tool_calls: List[ToolCallObject]
+
+
+class ChatHistoryChatBot(TypedDict, total=False):
+    role: Required[Literal["CHATBOT"]]
+    message: str
+    tool_calls: List[ToolCallObject]
+
+
+ChatHistory = List[
+    Union[ChatHistorySystem, ChatHistoryChatBot, ChatHistoryUser, ChatHistoryToolResult]
+]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/custom_http.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/custom_http.py
new file mode 100644
index 00000000..5eec187d
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/custom_http.py
@@ -0,0 +1,24 @@
+import ssl
+from enum import Enum
+from typing import Union
+
+
+class httpxSpecialProvider(str, Enum):
+    """
+    Httpx Clients can be created for these litellm internal providers
+
+    Example:
+    - langsmith logging would need a custom async httpx client
+    - pass through endpoint would need a custom async httpx client
+    """
+
+    LoggingCallback = "logging_callback"
+    GuardrailCallback = "guardrail_callback"
+    Caching = "caching"
+    Oauth2Check = "oauth2_check"
+    SecretManager = "secret_manager"
+    PassThroughEndpoint = "pass_through_endpoint"
+    PromptFactory = "prompt_factory"
+
+
+VerifyTypes = Union[str, bool, ssl.SSLContext]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/custom_llm.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/custom_llm.py
new file mode 100644
index 00000000..d5499a41
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/custom_llm.py
@@ -0,0 +1,10 @@
+from typing import List
+
+from typing_extensions import Dict, Required, TypedDict, override
+
+from litellm.llms.custom_llm import CustomLLM
+
+
+class CustomLLMItem(TypedDict):
+    provider: str
+    custom_handler: CustomLLM
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/databricks.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/databricks.py
new file mode 100644
index 00000000..770e05fe
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/databricks.py
@@ -0,0 +1,21 @@
+from typing import TypedDict, Any, Union, Optional
+import json
+from typing_extensions import (
+    Self,
+    Protocol,
+    TypeGuard,
+    override,
+    get_origin,
+    runtime_checkable,
+    Required,
+)
+from pydantic import BaseModel
+
+
+class GenericStreamingChunk(TypedDict, total=False):
+    text: Required[str]
+    is_finished: Required[bool]
+    finish_reason: Required[Optional[str]]
+    logprobs: Optional[BaseModel]
+    original_chunk: Optional[BaseModel]
+    usage: Optional[BaseModel]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/mistral.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/mistral.py
new file mode 100644
index 00000000..e9563a9a
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/mistral.py
@@ -0,0 +1,12 @@
+from typing import List, Literal, Optional, TypedDict, Union
+
+
+class FunctionCall(TypedDict):
+    name: Optional[str]
+    arguments: Optional[Union[str, dict]]
+
+
+class MistralToolCallMessage(TypedDict):
+    id: Optional[str]
+    type: Literal["function"]
+    function: Optional[FunctionCall]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/ollama.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/ollama.py
new file mode 100644
index 00000000..9d71904c
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/ollama.py
@@ -0,0 +1,29 @@
+import json
+from typing import Any, List, Optional, TypedDict, Union
+
+from pydantic import BaseModel
+from typing_extensions import (
+    Protocol,
+    Required,
+    Self,
+    TypeGuard,
+    get_origin,
+    override,
+    runtime_checkable,
+)
+
+
+class OllamaToolCallFunction(
+    TypedDict
+):  # follows - https://github.com/ollama/ollama/blob/6bd8a4b0a1ac15d5718f52bbe1cd56f827beb694/api/types.go#L148
+    name: str
+    arguments: dict
+
+
+class OllamaToolCall(TypedDict):
+    function: OllamaToolCallFunction
+
+
+class OllamaVisionModelObject(TypedDict):
+    prompt: str
+    images: List[str]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/openai.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/openai.py
new file mode 100644
index 00000000..4b0be9d5
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/openai.py
@@ -0,0 +1,1040 @@
+from enum import Enum
+from os import PathLike
+from typing import IO, Any, Iterable, List, Literal, Mapping, Optional, Tuple, Union
+
+import httpx
+from openai._legacy_response import (
+    HttpxBinaryResponseContent as _HttpxBinaryResponseContent,
+)
+from openai.lib.streaming._assistants import (
+    AssistantEventHandler,
+    AssistantStreamManager,
+    AsyncAssistantEventHandler,
+    AsyncAssistantStreamManager,
+)
+from openai.pagination import AsyncCursorPage, SyncCursorPage
+from openai.types import Batch, EmbeddingCreateParams, FileObject
+from openai.types.beta.assistant import Assistant
+from openai.types.beta.assistant_tool_param import AssistantToolParam
+from openai.types.beta.thread_create_params import (
+    Message as OpenAICreateThreadParamsMessage,
+)
+from openai.types.beta.threads.message import Message as OpenAIMessage
+from openai.types.beta.threads.message_content import MessageContent
+from openai.types.beta.threads.run import Run
+from openai.types.chat import ChatCompletionChunk
+from openai.types.chat.chat_completion_audio_param import ChatCompletionAudioParam
+from openai.types.chat.chat_completion_content_part_input_audio_param import (
+    ChatCompletionContentPartInputAudioParam,
+)
+from openai.types.chat.chat_completion_modality import ChatCompletionModality
+from openai.types.chat.chat_completion_prediction_content_param import (
+    ChatCompletionPredictionContentParam,
+)
+from openai.types.embedding import Embedding as OpenAIEmbedding
+from openai.types.fine_tuning.fine_tuning_job import FineTuningJob
+from openai.types.responses.response import (
+    IncompleteDetails,
+    Response,
+    ResponseOutputItem,
+    ResponseTextConfig,
+    Tool,
+    ToolChoice,
+)
+from openai.types.responses.response_create_params import (
+    Reasoning,
+    ResponseIncludable,
+    ResponseInputParam,
+    ResponseTextConfigParam,
+    ToolChoice,
+    ToolParam,
+)
+from pydantic import BaseModel, Discriminator, Field, PrivateAttr
+from typing_extensions import Annotated, Dict, Required, TypedDict, override
+
+FileContent = Union[IO[bytes], bytes, PathLike]
+
+FileTypes = Union[
+    # file (or bytes)
+    FileContent,
+    # (filename, file (or bytes))
+    Tuple[Optional[str], FileContent],
+    # (filename, file (or bytes), content_type)
+    Tuple[Optional[str], FileContent, Optional[str]],
+    # (filename, file (or bytes), content_type, headers)
+    Tuple[Optional[str], FileContent, Optional[str], Mapping[str, str]],
+]
+
+
+EmbeddingInput = Union[str, List[str]]
+
+
+class HttpxBinaryResponseContent(_HttpxBinaryResponseContent):
+    _hidden_params: dict = {}
+    pass
+
+
+class NotGiven:
+    """
+    A sentinel singleton class used to distinguish omitted keyword arguments
+    from those passed in with the value None (which may have different behavior).
+
+    For example:
+
+    ```py
+    def get(timeout: Union[int, NotGiven, None] = NotGiven()) -> Response:
+        ...
+
+
+    get(timeout=1)  # 1s timeout
+    get(timeout=None)  # No timeout
+    get()  # Default timeout behavior, which may not be statically known at the method definition.
+    ```
+    """
+
+    def __bool__(self) -> Literal[False]:
+        return False
+
+    @override
+    def __repr__(self) -> str:
+        return "NOT_GIVEN"
+
+
+NOT_GIVEN = NotGiven()
+
+
+class ToolResourcesCodeInterpreter(TypedDict, total=False):
+    file_ids: List[str]
+    """
+    A list of [file](https://platform.openai.com/docs/api-reference/files) IDs made
+    available to the `code_interpreter` tool. There can be a maximum of 20 files
+    associated with the tool.
+    """
+
+
+class ToolResourcesFileSearchVectorStore(TypedDict, total=False):
+    file_ids: List[str]
+    """
+    A list of [file](https://platform.openai.com/docs/api-reference/files) IDs to
+    add to the vector store. There can be a maximum of 10000 files in a vector
+    store.
+    """
+
+    metadata: object
+    """Set of 16 key-value pairs that can be attached to a vector store.
+
+    This can be useful for storing additional information about the vector store in
+    a structured format. Keys can be a maximum of 64 characters long and values can
+    be a maxium of 512 characters long.
+    """
+
+
+class ToolResourcesFileSearch(TypedDict, total=False):
+    vector_store_ids: List[str]
+    """
+    The
+    [vector store](https://platform.openai.com/docs/api-reference/vector-stores/object)
+    attached to this thread. There can be a maximum of 1 vector store attached to
+    the thread.
+    """
+
+    vector_stores: Iterable[ToolResourcesFileSearchVectorStore]
+    """
+    A helper to create a
+    [vector store](https://platform.openai.com/docs/api-reference/vector-stores/object)
+    with file_ids and attach it to this thread. There can be a maximum of 1 vector
+    store attached to the thread.
+    """
+
+
+class OpenAICreateThreadParamsToolResources(TypedDict, total=False):
+    code_interpreter: ToolResourcesCodeInterpreter
+
+    file_search: ToolResourcesFileSearch
+
+
+class FileSearchToolParam(TypedDict, total=False):
+    type: Required[Literal["file_search"]]
+    """The type of tool being defined: `file_search`"""
+
+
+class CodeInterpreterToolParam(TypedDict, total=False):
+    type: Required[Literal["code_interpreter"]]
+    """The type of tool being defined: `code_interpreter`"""
+
+
+AttachmentTool = Union[CodeInterpreterToolParam, FileSearchToolParam]
+
+
+class Attachment(TypedDict, total=False):
+    file_id: str
+    """The ID of the file to attach to the message."""
+
+    tools: Iterable[AttachmentTool]
+    """The tools to add this file to."""
+
+
+class ImageFileObject(TypedDict):
+    file_id: Required[str]
+    detail: Optional[str]
+
+
+class ImageURLObject(TypedDict):
+    url: Required[str]
+    detail: Optional[str]
+
+
+class MessageContentTextObject(TypedDict):
+    type: Required[Literal["text"]]
+    text: str
+
+
+class MessageContentImageFileObject(TypedDict):
+    type: Literal["image_file"]
+    image_file: ImageFileObject
+
+
+class MessageContentImageURLObject(TypedDict):
+    type: Required[str]
+    image_url: ImageURLObject
+
+
+class MessageData(TypedDict):
+    role: Literal["user", "assistant"]
+    content: Union[
+        str,
+        List[
+            Union[
+                MessageContentTextObject,
+                MessageContentImageFileObject,
+                MessageContentImageURLObject,
+            ]
+        ],
+    ]
+    attachments: Optional[List[Attachment]]
+    metadata: Optional[dict]
+
+
+class Thread(BaseModel):
+    id: str
+    """The identifier, which can be referenced in API endpoints."""
+
+    created_at: int
+    """The Unix timestamp (in seconds) for when the thread was created."""
+
+    metadata: Optional[object] = None
+    """Set of 16 key-value pairs that can be attached to an object.
+
+    This can be useful for storing additional information about the object in a
+    structured format. Keys can be a maximum of 64 characters long and values can be
+    a maxium of 512 characters long.
+    """
+
+    object: Literal["thread"]
+    """The object type, which is always `thread`."""
+
+
+# OpenAI Files Types
+class CreateFileRequest(TypedDict, total=False):
+    """
+    CreateFileRequest
+    Used by Assistants API, Batches API, and Fine-Tunes API
+
+    Required Params:
+        file: FileTypes
+        purpose: Literal['assistants', 'batch', 'fine-tune']
+
+    Optional Params:
+        extra_headers: Optional[Dict[str, str]]
+        extra_body: Optional[Dict[str, str]] = None
+        timeout: Optional[float] = None
+    """
+
+    file: FileTypes
+    purpose: Literal["assistants", "batch", "fine-tune"]
+    extra_headers: Optional[Dict[str, str]]
+    extra_body: Optional[Dict[str, str]]
+    timeout: Optional[float]
+
+
+class FileContentRequest(TypedDict, total=False):
+    """
+    FileContentRequest
+    Used by Assistants API, Batches API, and Fine-Tunes API
+
+    Required Params:
+        file_id: str
+
+    Optional Params:
+        extra_headers: Optional[Dict[str, str]]
+        extra_body: Optional[Dict[str, str]] = None
+        timeout: Optional[float] = None
+    """
+
+    file_id: str
+    extra_headers: Optional[Dict[str, str]]
+    extra_body: Optional[Dict[str, str]]
+    timeout: Optional[float]
+
+
+# OpenAI Batches Types
+class CreateBatchRequest(TypedDict, total=False):
+    """
+    CreateBatchRequest
+    """
+
+    completion_window: Literal["24h"]
+    endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"]
+    input_file_id: str
+    metadata: Optional[Dict[str, str]]
+    extra_headers: Optional[Dict[str, str]]
+    extra_body: Optional[Dict[str, str]]
+    timeout: Optional[float]
+
+
+class RetrieveBatchRequest(TypedDict, total=False):
+    """
+    RetrieveBatchRequest
+    """
+
+    batch_id: str
+    extra_headers: Optional[Dict[str, str]]
+    extra_body: Optional[Dict[str, str]]
+    timeout: Optional[float]
+
+
+class CancelBatchRequest(TypedDict, total=False):
+    """
+    CancelBatchRequest
+    """
+
+    batch_id: str
+    extra_headers: Optional[Dict[str, str]]
+    extra_body: Optional[Dict[str, str]]
+    timeout: Optional[float]
+
+
+class ListBatchRequest(TypedDict, total=False):
+    """
+    ListBatchRequest - List your organization's batches
+    Calls https://api.openai.com/v1/batches
+    """
+
+    after: Union[str, NotGiven]
+    limit: Union[int, NotGiven]
+    extra_headers: Optional[Dict[str, str]]
+    extra_body: Optional[Dict[str, str]]
+    timeout: Optional[float]
+
+
+BatchJobStatus = Literal[
+    "validating",
+    "failed",
+    "in_progress",
+    "finalizing",
+    "completed",
+    "expired",
+    "cancelling",
+    "cancelled",
+]
+
+
+class ChatCompletionAudioDelta(TypedDict, total=False):
+    data: str
+    transcript: str
+    expires_at: int
+    id: str
+
+
+class ChatCompletionToolCallFunctionChunk(TypedDict, total=False):
+    name: Optional[str]
+    arguments: str
+
+
+class ChatCompletionAssistantToolCall(TypedDict):
+    id: Optional[str]
+    type: Literal["function"]
+    function: ChatCompletionToolCallFunctionChunk
+
+
+class ChatCompletionToolCallChunk(TypedDict):  # result of /chat/completions call
+    id: Optional[str]
+    type: Literal["function"]
+    function: ChatCompletionToolCallFunctionChunk
+    index: int
+
+
+class ChatCompletionDeltaToolCallChunk(TypedDict, total=False):
+    id: str
+    type: Literal["function"]
+    function: ChatCompletionToolCallFunctionChunk
+    index: int
+
+
+class ChatCompletionCachedContent(TypedDict):
+    type: Literal["ephemeral"]
+
+
+class ChatCompletionThinkingBlock(TypedDict, total=False):
+    type: Required[Literal["thinking"]]
+    thinking: str
+    signature: str
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
+
+
+class OpenAIChatCompletionTextObject(TypedDict):
+    type: Literal["text"]
+    text: str
+
+
+class ChatCompletionTextObject(
+    OpenAIChatCompletionTextObject, total=False
+):  # litellm wrapper on top of openai object for handling cached content
+    cache_control: ChatCompletionCachedContent
+
+
+class ChatCompletionImageUrlObject(TypedDict, total=False):
+    url: Required[str]
+    detail: str
+    format: str
+
+
+class ChatCompletionImageObject(TypedDict):
+    type: Literal["image_url"]
+    image_url: Union[str, ChatCompletionImageUrlObject]
+
+
+class ChatCompletionVideoUrlObject(TypedDict, total=False):
+    url: Required[str]
+    detail: str
+
+
+class ChatCompletionVideoObject(TypedDict):
+    type: Literal["video_url"]
+    video_url: Union[str, ChatCompletionVideoUrlObject]
+
+
+class ChatCompletionAudioObject(ChatCompletionContentPartInputAudioParam):
+    pass
+
+
+class DocumentObject(TypedDict):
+    type: Literal["text"]
+    media_type: str
+    data: str
+
+
+class CitationsObject(TypedDict):
+    enabled: bool
+
+
+class ChatCompletionDocumentObject(TypedDict):
+    type: Literal["document"]
+    source: DocumentObject
+    title: str
+    context: str
+    citations: Optional[CitationsObject]
+
+
+class ChatCompletionFileObjectFile(TypedDict):
+    file_data: Optional[str]
+    file_id: Optional[str]
+    filename: Optional[str]
+
+
+class ChatCompletionFileObject(TypedDict):
+    type: Literal["file"]
+    file: ChatCompletionFileObjectFile
+
+
+OpenAIMessageContentListBlock = Union[
+    ChatCompletionTextObject,
+    ChatCompletionImageObject,
+    ChatCompletionAudioObject,
+    ChatCompletionDocumentObject,
+    ChatCompletionVideoObject,
+    ChatCompletionFileObject,
+]
+
+OpenAIMessageContent = Union[
+    str,
+    Iterable[OpenAIMessageContentListBlock],
+]
+
+# The prompt(s) to generate completions for, encoded as a string, array of strings, array of tokens, or array of token arrays.
+AllPromptValues = Union[str, List[str], Iterable[int], Iterable[Iterable[int]], None]
+
+
+class OpenAIChatCompletionUserMessage(TypedDict):
+    role: Literal["user"]
+    content: OpenAIMessageContent
+
+
+class OpenAITextCompletionUserMessage(TypedDict):
+    role: Literal["user"]
+    content: AllPromptValues
+
+
+class ChatCompletionUserMessage(OpenAIChatCompletionUserMessage, total=False):
+    cache_control: ChatCompletionCachedContent
+
+
+class OpenAIChatCompletionAssistantMessage(TypedDict, total=False):
+    role: Required[Literal["assistant"]]
+    content: Optional[
+        Union[
+            str, Iterable[Union[ChatCompletionTextObject, ChatCompletionThinkingBlock]]
+        ]
+    ]
+    name: Optional[str]
+    tool_calls: Optional[List[ChatCompletionAssistantToolCall]]
+    function_call: Optional[ChatCompletionToolCallFunctionChunk]
+
+
+class ChatCompletionAssistantMessage(OpenAIChatCompletionAssistantMessage, total=False):
+    cache_control: ChatCompletionCachedContent
+    thinking_blocks: Optional[List[ChatCompletionThinkingBlock]]
+
+
+class ChatCompletionToolMessage(TypedDict):
+    role: Literal["tool"]
+    content: Union[str, Iterable[ChatCompletionTextObject]]
+    tool_call_id: str
+
+
+class ChatCompletionFunctionMessage(TypedDict):
+    role: Literal["function"]
+    content: Optional[Union[str, Iterable[ChatCompletionTextObject]]]
+    name: str
+    tool_call_id: Optional[str]
+
+
+class OpenAIChatCompletionSystemMessage(TypedDict, total=False):
+    role: Required[Literal["system"]]
+    content: Required[Union[str, List]]
+    name: str
+
+
+class OpenAIChatCompletionDeveloperMessage(TypedDict, total=False):
+    role: Required[Literal["developer"]]
+    content: Required[Union[str, List]]
+    name: str
+
+
+class ChatCompletionSystemMessage(OpenAIChatCompletionSystemMessage, total=False):
+    cache_control: ChatCompletionCachedContent
+
+
+class ChatCompletionDeveloperMessage(OpenAIChatCompletionDeveloperMessage, total=False):
+    cache_control: ChatCompletionCachedContent
+
+
+ValidUserMessageContentTypes = [
+    "text",
+    "image_url",
+    "input_audio",
+    "document",
+    "video_url",
+    "file",
+]  # used for validating user messages. Prevent users from accidentally sending anthropic messages.
+
+AllMessageValues = Union[
+    ChatCompletionUserMessage,
+    ChatCompletionAssistantMessage,
+    ChatCompletionToolMessage,
+    ChatCompletionSystemMessage,
+    ChatCompletionFunctionMessage,
+    ChatCompletionDeveloperMessage,
+]
+
+
+class ChatCompletionToolChoiceFunctionParam(TypedDict):
+    name: str
+
+
+class ChatCompletionToolChoiceObjectParam(TypedDict):
+    type: Literal["function"]
+    function: ChatCompletionToolChoiceFunctionParam
+
+
+ChatCompletionToolChoiceStringValues = Literal["none", "auto", "required"]
+
+ChatCompletionToolChoiceValues = Union[
+    ChatCompletionToolChoiceStringValues, ChatCompletionToolChoiceObjectParam
+]
+
+
+class ChatCompletionToolParamFunctionChunk(TypedDict, total=False):
+    name: Required[str]
+    description: str
+    parameters: dict
+
+
+class OpenAIChatCompletionToolParam(TypedDict):
+    type: Union[Literal["function"], str]
+    function: ChatCompletionToolParamFunctionChunk
+
+
+class ChatCompletionToolParam(OpenAIChatCompletionToolParam, total=False):
+    cache_control: ChatCompletionCachedContent
+
+
+class Function(TypedDict, total=False):
+    name: Required[str]
+    """The name of the function to call."""
+
+
+class ChatCompletionNamedToolChoiceParam(TypedDict, total=False):
+    function: Required[Function]
+
+    type: Required[Literal["function"]]
+    """The type of the tool. Currently, only `function` is supported."""
+
+
+class ChatCompletionRequest(TypedDict, total=False):
+    model: Required[str]
+    messages: Required[List[AllMessageValues]]
+    frequency_penalty: float
+    logit_bias: dict
+    logprobs: bool
+    top_logprobs: int
+    max_tokens: int
+    n: int
+    presence_penalty: float
+    response_format: dict
+    seed: int
+    service_tier: str
+    stop: Union[str, List[str]]
+    stream_options: dict
+    temperature: float
+    top_p: float
+    tools: List[ChatCompletionToolParam]
+    tool_choice: ChatCompletionToolChoiceValues
+    parallel_tool_calls: bool
+    function_call: Union[str, dict]
+    functions: List
+    user: str
+    metadata: dict  # litellm specific param
+
+
+class ChatCompletionDeltaChunk(TypedDict, total=False):
+    content: Optional[str]
+    tool_calls: List[ChatCompletionDeltaToolCallChunk]
+    role: str
+
+
+ChatCompletionAssistantContentValue = (
+    str  # keep as var, used in stream_chunk_builder as well
+)
+
+
+class ChatCompletionResponseMessage(TypedDict, total=False):
+    content: Optional[ChatCompletionAssistantContentValue]
+    tool_calls: Optional[List[ChatCompletionToolCallChunk]]
+    role: Literal["assistant"]
+    function_call: Optional[ChatCompletionToolCallFunctionChunk]
+    provider_specific_fields: Optional[dict]
+    reasoning_content: Optional[str]
+    thinking_blocks: Optional[List[ChatCompletionThinkingBlock]]
+
+
+class ChatCompletionUsageBlock(TypedDict):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class OpenAIChatCompletionChunk(ChatCompletionChunk):
+    def __init__(self, **kwargs):
+        # Set the 'object' kwarg to 'chat.completion.chunk'
+        kwargs["object"] = "chat.completion.chunk"
+        super().__init__(**kwargs)
+
+
+class Hyperparameters(BaseModel):
+    batch_size: Optional[Union[str, int]] = None  # "Number of examples in each batch."
+    learning_rate_multiplier: Optional[Union[str, float]] = (
+        None  # Scaling factor for the learning rate
+    )
+    n_epochs: Optional[Union[str, int]] = (
+        None  # "The number of epochs to train the model for"
+    )
+
+
+class FineTuningJobCreate(BaseModel):
+    """
+    FineTuningJobCreate - Create a fine-tuning job
+
+    Example Request
+    ```
+    {
+        "model": "gpt-3.5-turbo",
+        "training_file": "file-abc123",
+        "hyperparameters": {
+            "batch_size": "auto",
+            "learning_rate_multiplier": 0.1,
+            "n_epochs": 3
+        },
+        "suffix": "custom-model-name",
+        "validation_file": "file-xyz789",
+        "integrations": ["slack"],
+        "seed": 42
+    }
+    ```
+    """
+
+    model: str  # "The name of the model to fine-tune."
+    training_file: str  # "The ID of an uploaded file that contains training data."
+    hyperparameters: Optional[Hyperparameters] = (
+        None  # "The hyperparameters used for the fine-tuning job."
+    )
+    suffix: Optional[str] = (
+        None  # "A string of up to 18 characters that will be added to your fine-tuned model name."
+    )
+    validation_file: Optional[str] = (
+        None  # "The ID of an uploaded file that contains validation data."
+    )
+    integrations: Optional[List[str]] = (
+        None  # "A list of integrations to enable for your fine-tuning job."
+    )
+    seed: Optional[int] = None  # "The seed controls the reproducibility of the job."
+
+
+class LiteLLMFineTuningJobCreate(FineTuningJobCreate):
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai"]
+
+    class Config:
+        extra = "allow"  # This allows the model to accept additional fields
+
+
+AllEmbeddingInputValues = Union[str, List[str], List[int], List[List[int]]]
+
+OpenAIAudioTranscriptionOptionalParams = Literal[
+    "language", "prompt", "temperature", "response_format", "timestamp_granularities"
+]
+
+
+OpenAIImageVariationOptionalParams = Literal["n", "size", "response_format", "user"]
+
+
+class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
+    """TypedDict for Optional parameters supported by the responses API."""
+
+    include: Optional[List[ResponseIncludable]]
+    instructions: Optional[str]
+    max_output_tokens: Optional[int]
+    metadata: Optional[Dict[str, Any]]
+    parallel_tool_calls: Optional[bool]
+    previous_response_id: Optional[str]
+    reasoning: Optional[Reasoning]
+    store: Optional[bool]
+    stream: Optional[bool]
+    temperature: Optional[float]
+    text: Optional[ResponseTextConfigParam]
+    tool_choice: Optional[ToolChoice]
+    tools: Optional[Iterable[ToolParam]]
+    top_p: Optional[float]
+    truncation: Optional[Literal["auto", "disabled"]]
+    user: Optional[str]
+
+
+class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False):
+    """TypedDict for request parameters supported by the responses API."""
+
+    input: Union[str, ResponseInputParam]
+    model: str
+
+
+class BaseLiteLLMOpenAIResponseObject(BaseModel):
+    def __getitem__(self, key):
+        return self.__dict__[key]
+
+    def get(self, key, default=None):
+        return self.__dict__.get(key, default)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+    def items(self):
+        return self.__dict__.items()
+
+
+class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
+    reasoning_tokens: int
+
+    model_config = {"extra": "allow"}
+
+
+class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject):
+    input_tokens: int
+    """The number of input tokens."""
+
+    output_tokens: int
+    """The number of output tokens."""
+
+    output_tokens_details: Optional[OutputTokensDetails]
+    """A detailed breakdown of the output tokens."""
+
+    total_tokens: int
+    """The total number of tokens used."""
+
+    model_config = {"extra": "allow"}
+
+
+class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
+    id: str
+    created_at: float
+    error: Optional[dict]
+    incomplete_details: Optional[IncompleteDetails]
+    instructions: Optional[str]
+    metadata: Optional[Dict]
+    model: Optional[str]
+    object: Optional[str]
+    output: List[ResponseOutputItem]
+    parallel_tool_calls: bool
+    temperature: Optional[float]
+    tool_choice: ToolChoice
+    tools: List[Tool]
+    top_p: Optional[float]
+    max_output_tokens: Optional[int]
+    previous_response_id: Optional[str]
+    reasoning: Optional[Reasoning]
+    status: Optional[str]
+    text: Optional[ResponseTextConfig]
+    truncation: Optional[Literal["auto", "disabled"]]
+    usage: Optional[ResponseAPIUsage]
+    user: Optional[str]
+    # Define private attributes using PrivateAttr
+    _hidden_params: dict = PrivateAttr(default_factory=dict)
+
+
+class ResponsesAPIStreamEvents(str, Enum):
+    """
+    Enum representing all supported OpenAI stream event types for the Responses API.
+
+    Inherits from str to allow direct string comparison and usage as dictionary keys.
+    """
+
+    # Response lifecycle events
+    RESPONSE_CREATED = "response.created"
+    RESPONSE_IN_PROGRESS = "response.in_progress"
+    RESPONSE_COMPLETED = "response.completed"
+    RESPONSE_FAILED = "response.failed"
+    RESPONSE_INCOMPLETE = "response.incomplete"
+
+    # Output item events
+    OUTPUT_ITEM_ADDED = "response.output_item.added"
+    OUTPUT_ITEM_DONE = "response.output_item.done"
+
+    # Content part events
+    CONTENT_PART_ADDED = "response.content_part.added"
+    CONTENT_PART_DONE = "response.content_part.done"
+
+    # Output text events
+    OUTPUT_TEXT_DELTA = "response.output_text.delta"
+    OUTPUT_TEXT_ANNOTATION_ADDED = "response.output_text.annotation.added"
+    OUTPUT_TEXT_DONE = "response.output_text.done"
+
+    # Refusal events
+    REFUSAL_DELTA = "response.refusal.delta"
+    REFUSAL_DONE = "response.refusal.done"
+
+    # Function call events
+    FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta"
+    FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done"
+
+    # File search events
+    FILE_SEARCH_CALL_IN_PROGRESS = "response.file_search_call.in_progress"
+    FILE_SEARCH_CALL_SEARCHING = "response.file_search_call.searching"
+    FILE_SEARCH_CALL_COMPLETED = "response.file_search_call.completed"
+
+    # Web search events
+    WEB_SEARCH_CALL_IN_PROGRESS = "response.web_search_call.in_progress"
+    WEB_SEARCH_CALL_SEARCHING = "response.web_search_call.searching"
+    WEB_SEARCH_CALL_COMPLETED = "response.web_search_call.completed"
+
+    # Error event
+    ERROR = "error"
+
+
+class ResponseCreatedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.RESPONSE_CREATED]
+    response: ResponsesAPIResponse
+
+
+class ResponseInProgressEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS]
+    response: ResponsesAPIResponse
+
+
+class ResponseCompletedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.RESPONSE_COMPLETED]
+    response: ResponsesAPIResponse
+    _hidden_params: dict = PrivateAttr(default_factory=dict)
+
+
+class ResponseFailedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.RESPONSE_FAILED]
+    response: ResponsesAPIResponse
+
+
+class ResponseIncompleteEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE]
+    response: ResponsesAPIResponse
+
+
+class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED]
+    output_index: int
+    item: dict
+
+
+class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE]
+    output_index: int
+    item: dict
+
+
+class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED]
+    item_id: str
+    output_index: int
+    content_index: int
+    part: dict
+
+
+class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_DONE]
+    item_id: str
+    output_index: int
+    content_index: int
+    part: dict
+
+
+class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA]
+    item_id: str
+    output_index: int
+    content_index: int
+    delta: str
+
+
+class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED]
+    item_id: str
+    output_index: int
+    content_index: int
+    annotation_index: int
+    annotation: dict
+
+
+class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE]
+    item_id: str
+    output_index: int
+    content_index: int
+    text: str
+
+
+class RefusalDeltaEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.REFUSAL_DELTA]
+    item_id: str
+    output_index: int
+    content_index: int
+    delta: str
+
+
+class RefusalDoneEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.REFUSAL_DONE]
+    item_id: str
+    output_index: int
+    content_index: int
+    refusal: str
+
+
+class FunctionCallArgumentsDeltaEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA]
+    item_id: str
+    output_index: int
+    delta: str
+
+
+class FunctionCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE]
+    item_id: str
+    output_index: int
+    arguments: str
+
+
+class FileSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS]
+    output_index: int
+    item_id: str
+
+
+class FileSearchCallSearchingEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING]
+    output_index: int
+    item_id: str
+
+
+class FileSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED]
+    output_index: int
+    item_id: str
+
+
+class WebSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS]
+    output_index: int
+    item_id: str
+
+
+class WebSearchCallSearchingEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING]
+    output_index: int
+    item_id: str
+
+
+class WebSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED]
+    output_index: int
+    item_id: str
+
+
+class ErrorEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.ERROR]
+    code: Optional[str]
+    message: str
+    param: Optional[str]
+
+
+# Union type for all possible streaming responses
+ResponsesAPIStreamingResponse = Annotated[
+    Union[
+        ResponseCreatedEvent,
+        ResponseInProgressEvent,
+        ResponseCompletedEvent,
+        ResponseFailedEvent,
+        ResponseIncompleteEvent,
+        OutputItemAddedEvent,
+        OutputItemDoneEvent,
+        ContentPartAddedEvent,
+        ContentPartDoneEvent,
+        OutputTextDeltaEvent,
+        OutputTextAnnotationAddedEvent,
+        OutputTextDoneEvent,
+        RefusalDeltaEvent,
+        RefusalDoneEvent,
+        FunctionCallArgumentsDeltaEvent,
+        FunctionCallArgumentsDoneEvent,
+        FileSearchCallInProgressEvent,
+        FileSearchCallSearchingEvent,
+        FileSearchCallCompletedEvent,
+        WebSearchCallInProgressEvent,
+        WebSearchCallSearchingEvent,
+        WebSearchCallCompletedEvent,
+        ErrorEvent,
+    ],
+    Discriminator("type"),
+]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/rerank.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/rerank.py
new file mode 100644
index 00000000..f781af88
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/rerank.py
@@ -0,0 +1,19 @@
+import json
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
+
+from typing_extensions import (
+    Protocol,
+    Required,
+    Self,
+    TypeGuard,
+    get_origin,
+    override,
+    runtime_checkable,
+)
+
+
+class InfinityRerankResult(TypedDict):
+    index: int
+    relevance_score: float
+    document: Optional[str]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/vertex_ai.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/vertex_ai.py
new file mode 100644
index 00000000..7024909a
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/vertex_ai.py
@@ -0,0 +1,486 @@
+import json
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
+
+from typing_extensions import (
+    Protocol,
+    Required,
+    Self,
+    TypeGuard,
+    get_origin,
+    override,
+    runtime_checkable,
+)
+
+
+class FunctionResponse(TypedDict):
+    name: str
+    response: Optional[dict]
+
+
+class FunctionCall(TypedDict):
+    name: str
+    args: Optional[dict]
+
+
+class FileDataType(TypedDict):
+    mime_type: str
+    file_uri: str  # the cloud storage uri of storing this file
+
+
+class BlobType(TypedDict):
+    mime_type: Required[str]
+    data: Required[str]
+
+
+class PartType(TypedDict, total=False):
+    text: str
+    inline_data: BlobType
+    file_data: FileDataType
+    function_call: FunctionCall
+    function_response: FunctionResponse
+
+
+class HttpxFunctionCall(TypedDict):
+    name: str
+    args: dict
+
+
+class HttpxExecutableCode(TypedDict):
+    code: str
+    language: str
+
+
+class HttpxCodeExecutionResult(TypedDict):
+    outcome: str
+    output: str
+
+
+class HttpxPartType(TypedDict, total=False):
+    text: str
+    inline_data: BlobType
+    file_data: FileDataType
+    functionCall: HttpxFunctionCall
+    function_response: FunctionResponse
+    executableCode: HttpxExecutableCode
+    codeExecutionResult: HttpxCodeExecutionResult
+
+
+class HttpxContentType(TypedDict, total=False):
+    role: Literal["user", "model"]
+    parts: List[HttpxPartType]
+
+
+class ContentType(TypedDict, total=False):
+    role: Literal["user", "model"]
+    parts: Required[List[PartType]]
+
+
+class SystemInstructions(TypedDict):
+    parts: Required[List[PartType]]
+
+
+class Schema(TypedDict, total=False):
+    type: Literal["STRING", "INTEGER", "BOOLEAN", "NUMBER", "ARRAY", "OBJECT"]
+    description: str
+    enum: List[str]
+    items: List["Schema"]
+    properties: "Schema"
+    required: List[str]
+    nullable: bool
+
+
+class FunctionDeclaration(TypedDict, total=False):
+    name: Required[str]
+    description: str
+    parameters: Union[Schema, dict]
+    response: Schema
+
+
+class VertexAISearch(TypedDict, total=False):
+    datastore: Required[str]
+
+
+class Retrieval(TypedDict):
+    source: VertexAISearch
+
+
+class FunctionCallingConfig(TypedDict, total=False):
+    mode: Literal["ANY", "AUTO", "NONE"]
+    allowed_function_names: List[str]
+
+
+HarmCategory = Literal[
+    "HARM_CATEGORY_UNSPECIFIED",
+    "HARM_CATEGORY_HATE_SPEECH",
+    "HARM_CATEGORY_DANGEROUS_CONTENT",
+    "HARM_CATEGORY_HARASSMENT",
+    "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+]
+HarmBlockThreshold = Literal[
+    "HARM_BLOCK_THRESHOLD_UNSPECIFIED",
+    "BLOCK_LOW_AND_ABOVE",
+    "BLOCK_MEDIUM_AND_ABOVE",
+    "BLOCK_ONLY_HIGH",
+    "BLOCK_NONE",
+]
+HarmBlockMethod = Literal["HARM_BLOCK_METHOD_UNSPECIFIED", "SEVERITY", "PROBABILITY"]
+
+HarmProbability = Literal[
+    "HARM_PROBABILITY_UNSPECIFIED", "NEGLIGIBLE", "LOW", "MEDIUM", "HIGH"
+]
+
+HarmSeverity = Literal[
+    "HARM_SEVERITY_UNSPECIFIED",
+    "HARM_SEVERITY_NEGLIGIBLE",
+    "HARM_SEVERITY_LOW",
+    "HARM_SEVERITY_MEDIUM",
+    "HARM_SEVERITY_HIGH",
+]
+
+
+class SafetSettingsConfig(TypedDict, total=False):
+    category: HarmCategory
+    threshold: HarmBlockThreshold
+    max_influential_terms: int
+    method: HarmBlockMethod
+
+
+class GenerationConfig(TypedDict, total=False):
+    temperature: float
+    top_p: float
+    top_k: float
+    candidate_count: int
+    max_output_tokens: int
+    stop_sequences: List[str]
+    presence_penalty: float
+    frequency_penalty: float
+    response_mime_type: Literal["text/plain", "application/json"]
+    response_schema: dict
+    seed: int
+    responseLogprobs: bool
+    logprobs: int
+
+
+class Tools(TypedDict, total=False):
+    function_declarations: List[FunctionDeclaration]
+    googleSearch: dict
+    googleSearchRetrieval: dict
+    code_execution: dict
+    retrieval: Retrieval
+
+
+class ToolConfig(TypedDict):
+    functionCallingConfig: FunctionCallingConfig
+
+
+class TTL(TypedDict, total=False):
+    seconds: Required[float]
+    nano: float
+
+
+class UsageMetadata(TypedDict, total=False):
+    promptTokenCount: int
+    totalTokenCount: int
+    candidatesTokenCount: int
+    cachedContentTokenCount: int
+
+
+class CachedContent(TypedDict, total=False):
+    ttl: TTL
+    expire_time: str
+    contents: List[ContentType]
+    tools: List[Tools]
+    createTime: str  # "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z"
+    updateTime: str  # "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z"
+    usageMetadata: UsageMetadata
+    expireTime: str  # "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z"
+    name: str
+    displayName: str
+    model: str
+    systemInstruction: ContentType
+    toolConfig: ToolConfig
+
+
+class RequestBody(TypedDict, total=False):
+    contents: Required[List[ContentType]]
+    system_instruction: SystemInstructions
+    tools: Tools
+    toolConfig: ToolConfig
+    safetySettings: List[SafetSettingsConfig]
+    generationConfig: GenerationConfig
+    cachedContent: str
+
+
+class CachedContentRequestBody(TypedDict, total=False):
+    contents: Required[List[ContentType]]
+    system_instruction: SystemInstructions
+    tools: Tools
+    toolConfig: ToolConfig
+    model: Required[str]  # Format: models/{model}
+    ttl: str  # ending in 's' - Example: "3.5s".
+    displayName: str
+
+
+class CachedContentListAllResponseBody(TypedDict, total=False):
+    cachedContents: List[CachedContent]
+    nextPageToken: str
+
+
+class SafetyRatings(TypedDict):
+    category: HarmCategory
+    probability: HarmProbability
+    probabilityScore: int
+    severity: HarmSeverity
+    blocked: bool
+
+
+class Date(TypedDict):
+    year: int
+    month: int
+    date: int
+
+
+class Citation(TypedDict):
+    startIndex: int
+    endIndex: int
+    uri: str
+    title: str
+    license: str
+    publicationDate: Date
+
+
+class CitationMetadata(TypedDict):
+    citations: List[Citation]
+
+
+class SearchEntryPoint(TypedDict, total=False):
+    renderedContent: str
+    sdkBlob: str
+
+
+class GroundingMetadata(TypedDict, total=False):
+    webSearchQueries: List[str]
+    searchEntryPoint: SearchEntryPoint
+    groundingAttributions: List[dict]
+
+
+class LogprobsCandidate(TypedDict):
+    token: str
+    tokenId: int
+    logProbability: float
+
+
+class LogprobsTopCandidate(TypedDict):
+    candidates: List[LogprobsCandidate]
+
+
+class LogprobsResult(TypedDict, total=False):
+    topCandidates: List[LogprobsTopCandidate]
+    chosenCandidates: List[LogprobsCandidate]
+
+
+class Candidates(TypedDict, total=False):
+    index: int
+    content: HttpxContentType
+    finishReason: Literal[
+        "FINISH_REASON_UNSPECIFIED",
+        "STOP",
+        "MAX_TOKENS",
+        "SAFETY",
+        "RECITATION",
+        "OTHER",
+        "BLOCKLIST",
+        "PROHIBITED_CONTENT",
+        "SPII",
+    ]
+    safetyRatings: List[SafetyRatings]
+    citationMetadata: CitationMetadata
+    groundingMetadata: GroundingMetadata
+    finishMessage: str
+    logprobsResult: LogprobsResult
+
+
+class PromptFeedback(TypedDict):
+    blockReason: str
+    safetyRatings: List[SafetyRatings]
+    blockReasonMessage: str
+
+
+class GenerateContentResponseBody(TypedDict, total=False):
+    candidates: List[Candidates]
+    promptFeedback: PromptFeedback
+    usageMetadata: Required[UsageMetadata]
+
+
+class FineTuneHyperparameters(TypedDict, total=False):
+    epoch_count: Optional[int]
+    learning_rate_multiplier: Optional[float]
+    adapter_size: Optional[
+        Literal[
+            "ADAPTER_SIZE_UNSPECIFIED",
+            "ADAPTER_SIZE_ONE",
+            "ADAPTER_SIZE_FOUR",
+            "ADAPTER_SIZE_EIGHT",
+            "ADAPTER_SIZE_SIXTEEN",
+        ]
+    ]
+
+
+class FineTunesupervisedTuningSpec(TypedDict, total=False):
+    training_dataset_uri: str
+    validation_dataset: Optional[str]
+    tuned_model_display_name: Optional[str]
+    hyperParameters: Optional[FineTuneHyperparameters]
+
+
+class FineTuneJobCreate(TypedDict, total=False):
+    baseModel: str
+    supervisedTuningSpec: FineTunesupervisedTuningSpec
+    tunedModelDisplayName: Optional[str]
+
+
+class ResponseSupervisedTuningSpec(TypedDict, total=False):
+    trainingDatasetUri: Optional[str]
+    hyperParameters: Optional[FineTuneHyperparameters]
+
+
+class ResponseTuningJob(TypedDict):
+    name: Optional[str]
+    tunedModelDisplayName: Optional[str]
+    baseModel: Optional[str]
+    supervisedTuningSpec: Optional[ResponseSupervisedTuningSpec]
+    state: Optional[
+        Literal[
+            "JOB_STATE_PENDING",
+            "JOB_STATE_RUNNING",
+            "JOB_STATE_SUCCEEDED",
+            "JOB_STATE_FAILED",
+            "JOB_STATE_CANCELLED",
+        ]
+    ]
+    createTime: Optional[str]
+    updateTime: Optional[str]
+
+
+class InstanceVideo(TypedDict, total=False):
+    gcsUri: str
+    videoSegmentConfig: Tuple[float, float, float]
+
+
+class InstanceImage(TypedDict, total=False):
+    gcsUri: Optional[str]
+    bytesBase64Encoded: Optional[str]
+    mimeType: Optional[str]
+
+
+class Instance(TypedDict, total=False):
+    text: str
+    image: InstanceImage
+    video: InstanceVideo
+
+
+class VertexMultimodalEmbeddingRequest(TypedDict, total=False):
+    instances: List[Instance]
+
+
+class VideoEmbedding(TypedDict):
+    startOffsetSec: int
+    endOffsetSec: int
+    embedding: List[float]
+
+
+class MultimodalPrediction(TypedDict, total=False):
+    textEmbedding: List[float]
+    imageEmbedding: List[float]
+    videoEmbeddings: List[VideoEmbedding]
+
+
+class MultimodalPredictions(TypedDict, total=False):
+    predictions: List[MultimodalPrediction]
+
+
+class VertexAICachedContentResponseObject(TypedDict):
+    name: str
+    model: str
+
+
+class TaskTypeEnum(Enum):
+    TASK_TYPE_UNSPECIFIED = "TASK_TYPE_UNSPECIFIED"
+    RETRIEVAL_QUERY = "RETRIEVAL_QUERY"
+    RETRIEVAL_DOCUMENT = "RETRIEVAL_DOCUMENT"
+    SEMANTIC_SIMILARITY = "SEMANTIC_SIMILARITY"
+    CLASSIFICATION = "CLASSIFICATION"
+    CLUSTERING = "CLUSTERING"
+    QUESTION_ANSWERING = "QUESTION_ANSWERING"
+    FACT_VERIFICATION = "FACT_VERIFICATION"
+
+
+class VertexAITextEmbeddingsRequestBody(TypedDict, total=False):
+    content: Required[ContentType]
+    taskType: TaskTypeEnum
+    title: str
+    outputDimensionality: int
+
+
+class ContentEmbeddings(TypedDict):
+    values: List[int]
+
+
+class VertexAITextEmbeddingsResponseObject(TypedDict):
+    embedding: ContentEmbeddings
+
+
+class EmbedContentRequest(VertexAITextEmbeddingsRequestBody):
+    model: Required[str]
+
+
+class VertexAIBatchEmbeddingsRequestBody(TypedDict, total=False):
+    requests: List[EmbedContentRequest]
+
+
+class VertexAIBatchEmbeddingsResponseObject(TypedDict):
+    embeddings: List[ContentEmbeddings]
+
+
+# Vertex AI Batch Prediction
+
+
+class GcsSource(TypedDict):
+    uris: str
+
+
+class InputConfig(TypedDict):
+    instancesFormat: str
+    gcsSource: GcsSource
+
+
+class GcsDestination(TypedDict):
+    outputUriPrefix: str
+
+
+class OutputConfig(TypedDict, total=False):
+    predictionsFormat: str
+    gcsDestination: GcsDestination
+
+
+class VertexAIBatchPredictionJob(TypedDict):
+    displayName: str
+    model: str
+    inputConfig: InputConfig
+    outputConfig: OutputConfig
+
+
+class VertexBatchPredictionResponse(TypedDict, total=False):
+    name: str
+    displayName: str
+    model: str
+    inputConfig: InputConfig
+    outputConfig: OutputConfig
+    state: str
+    createTime: str
+    updateTime: str
+    modelVersionId: str
+
+
+VERTEX_CREDENTIALS_TYPES = Union[str, Dict[str, str]]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/llms/watsonx.py b/.venv/lib/python3.12/site-packages/litellm/types/llms/watsonx.py
new file mode 100644
index 00000000..7dee2836
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/llms/watsonx.py
@@ -0,0 +1,33 @@
+import json
+from enum import Enum
+from typing import Any, List, Optional, TypedDict, Union
+
+from pydantic import BaseModel
+
+
+class WatsonXAPIParams(TypedDict):
+    project_id: str
+    space_id: Optional[str]
+    region_name: Optional[str]
+
+
+class WatsonXCredentials(TypedDict):
+    api_key: str
+    api_base: str
+    token: Optional[str]
+
+
+class WatsonXAIEndpoint(str, Enum):
+    TEXT_GENERATION = "/ml/v1/text/generation"
+    TEXT_GENERATION_STREAM = "/ml/v1/text/generation_stream"
+    CHAT = "/ml/v1/text/chat"
+    CHAT_STREAM = "/ml/v1/text/chat_stream"
+    DEPLOYMENT_TEXT_GENERATION = "/ml/v1/deployments/{deployment_id}/text/generation"
+    DEPLOYMENT_TEXT_GENERATION_STREAM = (
+        "/ml/v1/deployments/{deployment_id}/text/generation_stream"
+    )
+    DEPLOYMENT_CHAT = "/ml/v1/deployments/{deployment_id}/text/chat"
+    DEPLOYMENT_CHAT_STREAM = "/ml/v1/deployments/{deployment_id}/text/chat_stream"
+    EMBEDDINGS = "/ml/v1/text/embeddings"
+    PROMPTS = "/ml/v1/prompts"
+    AVAILABLE_MODELS = "/ml/v1/foundation_model_specs"
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/passthrough_endpoints/vertex_ai.py b/.venv/lib/python3.12/site-packages/litellm/types/passthrough_endpoints/vertex_ai.py
new file mode 100644
index 00000000..90871198
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/passthrough_endpoints/vertex_ai.py
@@ -0,0 +1,20 @@
+"""
+Used for /vertex_ai/ pass through endpoints
+"""
+
+from typing import Optional
+
+from pydantic import BaseModel
+
+from ..llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
+
+
+class VertexPassThroughCredentials(BaseModel):
+    # Example: vertex_project = "my-project-123"
+    vertex_project: Optional[str] = None
+
+    # Example: vertex_location = "us-central1"
+    vertex_location: Optional[str] = None
+
+    # Example: vertex_credentials = "/path/to/credentials.json" or "os.environ/GOOGLE_CREDS"
+    vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES] = None
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/rerank.py b/.venv/lib/python3.12/site-packages/litellm/types/rerank.py
new file mode 100644
index 00000000..8e2a8cc3
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/rerank.py
@@ -0,0 +1,78 @@
+"""
+LiteLLM Follows the cohere API format for the re rank API
+https://docs.cohere.com/reference/rerank
+
+"""
+
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, PrivateAttr
+from typing_extensions import Required, TypedDict
+
+
+class RerankRequest(BaseModel):
+    model: str
+    query: str
+    top_n: Optional[int] = None
+    documents: List[Union[str, dict]]
+    rank_fields: Optional[List[str]] = None
+    return_documents: Optional[bool] = None
+    max_chunks_per_doc: Optional[int] = None
+    max_tokens_per_doc: Optional[int] = None
+
+
+
+class OptionalRerankParams(TypedDict, total=False):
+    query: str
+    top_n: Optional[int]
+    documents: List[Union[str, dict]]
+    rank_fields: Optional[List[str]]
+    return_documents: Optional[bool]
+    max_chunks_per_doc: Optional[int]
+    max_tokens_per_doc: Optional[int]
+
+
+class RerankBilledUnits(TypedDict, total=False):
+    search_units: Optional[int]
+    total_tokens: Optional[int]
+
+
+class RerankTokens(TypedDict, total=False):
+    input_tokens: Optional[int]
+    output_tokens: Optional[int]
+
+
+class RerankResponseMeta(TypedDict, total=False):
+    api_version: Optional[dict]
+    billed_units: Optional[RerankBilledUnits]
+    tokens: Optional[RerankTokens]
+
+
+class RerankResponseDocument(TypedDict):
+    text: str
+
+
+class RerankResponseResult(TypedDict, total=False):
+    index: Required[int]
+    relevance_score: Required[float]
+    document: RerankResponseDocument
+
+
+class RerankResponse(BaseModel):
+    id: Optional[str] = None
+    results: Optional[List[RerankResponseResult]] = (
+        None  # Contains index and relevance_score
+    )
+    meta: Optional[RerankResponseMeta] = None  # Contains api_version and billed_units
+
+    # Define private attributes using PrivateAttr
+    _hidden_params: dict = PrivateAttr(default_factory=dict)
+
+    def __getitem__(self, key):
+        return self.__dict__[key]
+
+    def get(self, key, default=None):
+        return self.__dict__.get(key, default)
+
+    def __contains__(self, key):
+        return key in self.__dict__
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/router.py b/.venv/lib/python3.12/site-packages/litellm/types/router.py
new file mode 100644
index 00000000..e34366aa
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/router.py
@@ -0,0 +1,707 @@
+"""
+litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
+"""
+
+import datetime
+import enum
+import uuid
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union, get_type_hints
+
+import httpx
+from httpx import AsyncClient, Client
+from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+from pydantic import BaseModel, ConfigDict, Field
+from typing_extensions import Required, TypedDict
+
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+
+from ..exceptions import RateLimitError
+from .completion import CompletionRequest
+from .embedding import EmbeddingRequest
+from .llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
+from .utils import ModelResponse, ProviderSpecificModelInfo
+
+
+class ConfigurableClientsideParamsCustomAuth(TypedDict):
+    api_base: str
+
+
+CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = Optional[
+    List[Union[str, ConfigurableClientsideParamsCustomAuth]]
+]
+
+
+class ModelConfig(BaseModel):
+    model_name: str
+    litellm_params: Union[CompletionRequest, EmbeddingRequest]
+    tpm: int
+    rpm: int
+
+    model_config = ConfigDict(protected_namespaces=())
+
+
+class RouterConfig(BaseModel):
+    model_list: List[ModelConfig]
+
+    redis_url: Optional[str] = None
+    redis_host: Optional[str] = None
+    redis_port: Optional[int] = None
+    redis_password: Optional[str] = None
+
+    cache_responses: Optional[bool] = False
+    cache_kwargs: Optional[Dict] = {}
+    caching_groups: Optional[List[Tuple[str, List[str]]]] = None
+    client_ttl: Optional[int] = 3600
+    num_retries: Optional[int] = 0
+    timeout: Optional[float] = None
+    default_litellm_params: Optional[Dict[str, str]] = {}
+    set_verbose: Optional[bool] = False
+    fallbacks: Optional[List] = []
+    allowed_fails: Optional[int] = None
+    context_window_fallbacks: Optional[List] = []
+    model_group_alias: Optional[Dict[str, List[str]]] = {}
+    retry_after: Optional[int] = 0
+    routing_strategy: Literal[
+        "simple-shuffle",
+        "least-busy",
+        "usage-based-routing",
+        "latency-based-routing",
+    ] = "simple-shuffle"
+
+    model_config = ConfigDict(protected_namespaces=())
+
+
+class UpdateRouterConfig(BaseModel):
+    """
+    Set of params that you can modify via `router.update_settings()`.
+    """
+
+    routing_strategy_args: Optional[dict] = None
+    routing_strategy: Optional[str] = None
+    model_group_retry_policy: Optional[dict] = None
+    allowed_fails: Optional[int] = None
+    cooldown_time: Optional[float] = None
+    num_retries: Optional[int] = None
+    timeout: Optional[float] = None
+    max_retries: Optional[int] = None
+    retry_after: Optional[float] = None
+    fallbacks: Optional[List[dict]] = None
+    context_window_fallbacks: Optional[List[dict]] = None
+
+    model_config = ConfigDict(protected_namespaces=())
+
+
+class ModelInfo(BaseModel):
+    id: Optional[
+        str
+    ]  # Allow id to be optional on input, but it will always be present as a str in the model instance
+    db_model: bool = (
+        False  # used for proxy - to separate models which are stored in the db vs. config.
+    )
+    updated_at: Optional[datetime.datetime] = None
+    updated_by: Optional[str] = None
+
+    created_at: Optional[datetime.datetime] = None
+    created_by: Optional[str] = None
+
+    base_model: Optional[str] = (
+        None  # specify if the base model is azure/gpt-3.5-turbo etc for accurate cost tracking
+    )
+    tier: Optional[Literal["free", "paid"]] = None
+
+    """
+    Team Model Specific Fields
+    """
+    # the team id that this model belongs to
+    team_id: Optional[str] = None
+
+    # the model_name that can be used by the team when making LLM calls
+    team_public_model_name: Optional[str] = None
+
+    def __init__(self, id: Optional[Union[str, int]] = None, **params):
+        if id is None:
+            id = str(uuid.uuid4())  # Generate a UUID if id is None or not provided
+        elif isinstance(id, int):
+            id = str(id)
+        super().__init__(id=id, **params)
+
+    model_config = ConfigDict(extra="allow")
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class CredentialLiteLLMParams(BaseModel):
+    api_key: Optional[str] = None
+    api_base: Optional[str] = None
+    api_version: Optional[str] = None
+    ## VERTEX AI ##
+    vertex_project: Optional[str] = None
+    vertex_location: Optional[str] = None
+    vertex_credentials: Optional[Union[str, dict]] = None
+    ## UNIFIED PROJECT/REGION ##
+    region_name: Optional[str] = None
+
+    ## AWS BEDROCK / SAGEMAKER ##
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    aws_region_name: Optional[str] = None
+    ## IBM WATSONX ##
+    watsonx_region_name: Optional[str] = None
+
+
+class GenericLiteLLMParams(CredentialLiteLLMParams):
+    """
+    LiteLLM Params without 'model' arg (used across completion / assistants api)
+    """
+
+    custom_llm_provider: Optional[str] = None
+    tpm: Optional[int] = None
+    rpm: Optional[int] = None
+    timeout: Optional[Union[float, str, httpx.Timeout]] = (
+        None  # if str, pass in as os.environ/
+    )
+    stream_timeout: Optional[Union[float, str]] = (
+        None  # timeout when making stream=True calls, if str, pass in as os.environ/
+    )
+    max_retries: Optional[int] = None
+    organization: Optional[str] = None  # for openai orgs
+    configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
+
+    ## LOGGING PARAMS ##
+    litellm_trace_id: Optional[str] = None
+
+    ## CUSTOM PRICING ##
+    input_cost_per_token: Optional[float] = None
+    output_cost_per_token: Optional[float] = None
+    input_cost_per_second: Optional[float] = None
+    output_cost_per_second: Optional[float] = None
+
+    max_file_size_mb: Optional[float] = None
+
+    # Deployment budgets
+    max_budget: Optional[float] = None
+    budget_duration: Optional[str] = None
+    use_in_pass_through: Optional[bool] = False
+    model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
+    merge_reasoning_content_in_choices: Optional[bool] = False
+    model_info: Optional[Dict] = None
+
+    def __init__(
+        self,
+        custom_llm_provider: Optional[str] = None,
+        max_retries: Optional[Union[int, str]] = None,
+        tpm: Optional[int] = None,
+        rpm: Optional[int] = None,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+        api_version: Optional[str] = None,
+        timeout: Optional[Union[float, str]] = None,  # if str, pass in as os.environ/
+        stream_timeout: Optional[Union[float, str]] = (
+            None  # timeout when making stream=True calls, if str, pass in as os.environ/
+        ),
+        organization: Optional[str] = None,  # for openai orgs
+        ## LOGGING PARAMS ##
+        litellm_trace_id: Optional[str] = None,
+        ## UNIFIED PROJECT/REGION ##
+        region_name: Optional[str] = None,
+        ## VERTEX AI ##
+        vertex_project: Optional[str] = None,
+        vertex_location: Optional[str] = None,
+        vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES] = None,
+        ## AWS BEDROCK / SAGEMAKER ##
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+        aws_region_name: Optional[str] = None,
+        ## IBM WATSONX ##
+        watsonx_region_name: Optional[str] = None,
+        input_cost_per_token: Optional[float] = None,
+        output_cost_per_token: Optional[float] = None,
+        input_cost_per_second: Optional[float] = None,
+        output_cost_per_second: Optional[float] = None,
+        max_file_size_mb: Optional[float] = None,
+        # Deployment budgets
+        max_budget: Optional[float] = None,
+        budget_duration: Optional[str] = None,
+        # Pass through params
+        use_in_pass_through: Optional[bool] = False,
+        # This will merge the reasoning content in the choices
+        merge_reasoning_content_in_choices: Optional[bool] = False,
+        model_info: Optional[Dict] = None,
+        **params,
+    ):
+        args = locals()
+        args.pop("max_retries", None)
+        args.pop("self", None)
+        args.pop("params", None)
+        args.pop("__class__", None)
+        if max_retries is not None and isinstance(max_retries, str):
+            max_retries = int(max_retries)  # cast to int
+        # We need to keep max_retries in args since it's a parameter of GenericLiteLLMParams
+        args["max_retries"] = (
+            max_retries  # Put max_retries back in args after popping it
+        )
+        super().__init__(**args, **params)
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class LiteLLM_Params(GenericLiteLLMParams):
+    """
+    LiteLLM Params with 'model' requirement - used for completions
+    """
+
+    model: str
+    model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
+
+    def __init__(
+        self,
+        model: str,
+        custom_llm_provider: Optional[str] = None,
+        max_retries: Optional[Union[int, str]] = None,
+        tpm: Optional[int] = None,
+        rpm: Optional[int] = None,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+        api_version: Optional[str] = None,
+        timeout: Optional[Union[float, str]] = None,  # if str, pass in as os.environ/
+        stream_timeout: Optional[Union[float, str]] = (
+            None  # timeout when making stream=True calls, if str, pass in as os.environ/
+        ),
+        organization: Optional[str] = None,  # for openai orgs
+        ## VERTEX AI ##
+        vertex_project: Optional[str] = None,
+        vertex_location: Optional[str] = None,
+        ## AWS BEDROCK / SAGEMAKER ##
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+        aws_region_name: Optional[str] = None,
+        # OpenAI / Azure Whisper
+        # set a max-size of file that can be passed to litellm proxy
+        max_file_size_mb: Optional[float] = None,
+        # will use deployment on pass-through endpoints if True
+        use_in_pass_through: Optional[bool] = False,
+        **params,
+    ):
+        args = locals()
+        args.pop("max_retries", None)
+        args.pop("self", None)
+        args.pop("params", None)
+        args.pop("__class__", None)
+        if max_retries is not None and isinstance(max_retries, str):
+            max_retries = int(max_retries)  # cast to int
+        super().__init__(max_retries=max_retries, **args, **params)
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class updateLiteLLMParams(GenericLiteLLMParams):
+    # This class is used to update the LiteLLM_Params
+    # only differece is model is optional
+    model: Optional[str] = None
+
+
+class updateDeployment(BaseModel):
+    model_name: Optional[str] = None
+    litellm_params: Optional[updateLiteLLMParams] = None
+    model_info: Optional[ModelInfo] = None
+
+    model_config = ConfigDict(protected_namespaces=())
+
+
+class LiteLLMParamsTypedDict(TypedDict, total=False):
+    model: str
+    custom_llm_provider: Optional[str]
+    tpm: Optional[int]
+    rpm: Optional[int]
+    order: Optional[int]
+    weight: Optional[int]
+    max_parallel_requests: Optional[int]
+    api_key: Optional[str]
+    api_base: Optional[str]
+    api_version: Optional[str]
+    timeout: Optional[Union[float, str, httpx.Timeout]]
+    stream_timeout: Optional[Union[float, str]]
+    max_retries: Optional[int]
+    organization: Optional[Union[List, str]]  # for openai orgs
+    configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS  # for allowing api base switching on finetuned models
+    ## DROP PARAMS ##
+    drop_params: Optional[bool]
+    ## UNIFIED PROJECT/REGION ##
+    region_name: Optional[str]
+    ## VERTEX AI ##
+    vertex_project: Optional[str]
+    vertex_location: Optional[str]
+    ## AWS BEDROCK / SAGEMAKER ##
+    aws_access_key_id: Optional[str]
+    aws_secret_access_key: Optional[str]
+    aws_region_name: Optional[str]
+    ## IBM WATSONX ##
+    watsonx_region_name: Optional[str]
+    ## CUSTOM PRICING ##
+    input_cost_per_token: Optional[float]
+    output_cost_per_token: Optional[float]
+    input_cost_per_second: Optional[float]
+    output_cost_per_second: Optional[float]
+    num_retries: Optional[int]
+    ## MOCK RESPONSES ##
+    mock_response: Optional[Union[str, ModelResponse, Exception]]
+
+    # routing params
+    # use this for tag-based routing
+    tags: Optional[List[str]]
+
+    # deployment budgets
+    max_budget: Optional[float]
+    budget_duration: Optional[str]
+
+
+class DeploymentTypedDict(TypedDict, total=False):
+    model_name: Required[str]
+    litellm_params: Required[LiteLLMParamsTypedDict]
+    model_info: dict
+
+
+SPECIAL_MODEL_INFO_PARAMS = [
+    "input_cost_per_token",
+    "output_cost_per_token",
+    "input_cost_per_character",
+    "output_cost_per_character",
+]
+
+
+class Deployment(BaseModel):
+    model_name: str
+    litellm_params: LiteLLM_Params
+    model_info: ModelInfo
+
+    model_config = ConfigDict(extra="allow", protected_namespaces=())
+
+    def __init__(
+        self,
+        model_name: str,
+        litellm_params: LiteLLM_Params,
+        model_info: Optional[Union[ModelInfo, dict]] = None,
+        **params,
+    ):
+        if model_info is None:
+            model_info = ModelInfo()
+        elif isinstance(model_info, dict):
+            model_info = ModelInfo(**model_info)
+
+        for (
+            key
+        ) in (
+            SPECIAL_MODEL_INFO_PARAMS
+        ):  # ensures custom pricing info is consistently in 'model_info'
+            field = getattr(litellm_params, key, None)
+            if field is not None:
+                setattr(model_info, key, field)
+
+        super().__init__(
+            model_info=model_info,
+            model_name=model_name,
+            litellm_params=litellm_params,
+            **params,
+        )
+
+    def to_json(self, **kwargs):
+        try:
+            return self.model_dump(**kwargs)  # noqa
+        except Exception as e:
+            # if using pydantic v1
+            return self.dict(**kwargs)
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class RouterErrors(enum.Enum):
+    """
+    Enum for router specific errors with common codes
+    """
+
+    user_defined_ratelimit_error = "Deployment over user-defined ratelimit."
+    no_deployments_available = "No deployments available for selected model"
+    no_deployments_with_tag_routing = (
+        "Not allowed to access model due to tags configuration"
+    )
+    no_deployments_with_provider_budget_routing = (
+        "No deployments available - crossed budget"
+    )
+
+
+class AllowedFailsPolicy(BaseModel):
+    """
+    Use this to set a custom number of allowed fails/minute before cooling down a deployment
+    If `AuthenticationErrorAllowedFails = 1000`, then 1000 AuthenticationError will be allowed before cooling down a deployment
+
+    Mapping of Exception type to allowed_fails for each exception
+    https://docs.litellm.ai/docs/exception_mapping
+    """
+
+    BadRequestErrorAllowedFails: Optional[int] = None
+    AuthenticationErrorAllowedFails: Optional[int] = None
+    TimeoutErrorAllowedFails: Optional[int] = None
+    RateLimitErrorAllowedFails: Optional[int] = None
+    ContentPolicyViolationErrorAllowedFails: Optional[int] = None
+    InternalServerErrorAllowedFails: Optional[int] = None
+
+
+class RetryPolicy(BaseModel):
+    """
+    Use this to set a custom number of retries per exception type
+    If RateLimitErrorRetries = 3, then 3 retries will be made for RateLimitError
+    Mapping of Exception type to number of retries
+    https://docs.litellm.ai/docs/exception_mapping
+    """
+
+    BadRequestErrorRetries: Optional[int] = None
+    AuthenticationErrorRetries: Optional[int] = None
+    TimeoutErrorRetries: Optional[int] = None
+    RateLimitErrorRetries: Optional[int] = None
+    ContentPolicyViolationErrorRetries: Optional[int] = None
+    InternalServerErrorRetries: Optional[int] = None
+
+
+class AlertingConfig(BaseModel):
+    """
+    Use this configure alerting for the router. Receive alerts on the following events
+    - LLM API Exceptions
+    - LLM Responses Too Slow
+    - LLM Requests Hanging
+
+    Args:
+        webhook_url: str            - webhook url for alerting, slack provides a webhook url to send alerts to
+        alerting_threshold: Optional[float] = None - threshold for slow / hanging llm responses (in seconds)
+    """
+
+    webhook_url: str
+    alerting_threshold: Optional[float] = 300
+
+
+class ModelGroupInfo(BaseModel):
+    model_group: str
+    providers: List[str]
+    max_input_tokens: Optional[float] = None
+    max_output_tokens: Optional[float] = None
+    input_cost_per_token: Optional[float] = None
+    output_cost_per_token: Optional[float] = None
+    mode: Optional[
+        Union[
+            str,
+            Literal[
+                "chat",
+                "embedding",
+                "completion",
+                "image_generation",
+                "audio_transcription",
+                "rerank",
+                "moderations",
+            ],
+        ]
+    ] = Field(default="chat")
+    tpm: Optional[int] = None
+    rpm: Optional[int] = None
+    supports_parallel_function_calling: bool = Field(default=False)
+    supports_vision: bool = Field(default=False)
+    supports_function_calling: bool = Field(default=False)
+    supported_openai_params: Optional[List[str]] = Field(default=[])
+    configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
+
+    def __init__(self, **data):
+        for field_name, field_type in get_type_hints(self.__class__).items():
+            if field_type == bool and data.get(field_name) is None:
+                data[field_name] = False
+        super().__init__(**data)
+
+
+class AssistantsTypedDict(TypedDict):
+    custom_llm_provider: Literal["azure", "openai"]
+    litellm_params: LiteLLMParamsTypedDict
+
+
+class FineTuningConfig(BaseModel):
+
+    custom_llm_provider: Literal["azure", "openai"]
+
+
+class CustomRoutingStrategyBase:
+    async def async_get_available_deployment(
+        self,
+        model: str,
+        messages: Optional[List[Dict[str, str]]] = None,
+        input: Optional[Union[str, List]] = None,
+        specific_deployment: Optional[bool] = False,
+        request_kwargs: Optional[Dict] = None,
+    ):
+        """
+        Asynchronously retrieves the available deployment based on the given parameters.
+
+        Args:
+            model (str): The name of the model.
+            messages (Optional[List[Dict[str, str]]], optional): The list of messages for a given request. Defaults to None.
+            input (Optional[Union[str, List]], optional): The input for a given embedding request. Defaults to None.
+            specific_deployment (Optional[bool], optional): Whether to retrieve a specific deployment. Defaults to False.
+            request_kwargs (Optional[Dict], optional): Additional request keyword arguments. Defaults to None.
+
+        Returns:
+            Returns an element from litellm.router.model_list
+
+        """
+        pass
+
+    def get_available_deployment(
+        self,
+        model: str,
+        messages: Optional[List[Dict[str, str]]] = None,
+        input: Optional[Union[str, List]] = None,
+        specific_deployment: Optional[bool] = False,
+        request_kwargs: Optional[Dict] = None,
+    ):
+        """
+        Synchronously retrieves the available deployment based on the given parameters.
+
+        Args:
+            model (str): The name of the model.
+            messages (Optional[List[Dict[str, str]]], optional): The list of messages for a given request. Defaults to None.
+            input (Optional[Union[str, List]], optional): The input for a given embedding request. Defaults to None.
+            specific_deployment (Optional[bool], optional): Whether to retrieve a specific deployment. Defaults to False.
+            request_kwargs (Optional[Dict], optional): Additional request keyword arguments. Defaults to None.
+
+        Returns:
+            Returns an element from litellm.router.model_list
+
+        """
+        pass
+
+
+class RouterGeneralSettings(BaseModel):
+    async_only_mode: bool = Field(
+        default=False
+    )  # this will only initialize async clients. Good for memory utils
+    pass_through_all_models: bool = Field(
+        default=False
+    )  # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding
+
+
+class RouterRateLimitErrorBasic(ValueError):
+    """
+    Raise a basic error inside helper functions.
+    """
+
+    def __init__(
+        self,
+        model: str,
+    ):
+        self.model = model
+        _message = f"{RouterErrors.no_deployments_available.value}."
+        super().__init__(_message)
+
+
+class RouterRateLimitError(ValueError):
+    def __init__(
+        self,
+        model: str,
+        cooldown_time: float,
+        enable_pre_call_checks: bool,
+        cooldown_list: List,
+    ):
+        self.model = model
+        self.cooldown_time = cooldown_time
+        self.enable_pre_call_checks = enable_pre_call_checks
+        self.cooldown_list = cooldown_list
+        _message = f"{RouterErrors.no_deployments_available.value}, Try again in {cooldown_time} seconds. Passed model={model}. pre-call-checks={enable_pre_call_checks}, cooldown_list={cooldown_list}"
+        super().__init__(_message)
+
+
+class RouterModelGroupAliasItem(TypedDict):
+    model: str
+    hidden: bool  # if 'True', don't return on `.get_model_list`
+
+
+VALID_LITELLM_ENVIRONMENTS = [
+    "development",
+    "staging",
+    "production",
+]
+
+
+class RoutingStrategy(enum.Enum):
+    LEAST_BUSY = "least-busy"
+    LATENCY_BASED = "latency-based-routing"
+    COST_BASED = "cost-based-routing"
+    USAGE_BASED_ROUTING_V2 = "usage-based-routing-v2"
+    USAGE_BASED_ROUTING = "usage-based-routing"
+    PROVIDER_BUDGET_LIMITING = "provider-budget-routing"
+
+
+class RouterCacheEnum(enum.Enum):
+    TPM = "global_router:{id}:{model}:tpm:{current_minute}"
+    RPM = "global_router:{id}:{model}:rpm:{current_minute}"
+
+
+class GenericBudgetWindowDetails(BaseModel):
+    """Details about a provider's budget window"""
+
+    budget_start: float
+    spend_key: str
+    start_time_key: str
+    ttl_seconds: int
+
+
+OptionalPreCallChecks = List[Literal["prompt_caching", "router_budget_limiting"]]
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/services.py b/.venv/lib/python3.12/site-packages/litellm/types/services.py
new file mode 100644
index 00000000..3eb283db
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/services.py
@@ -0,0 +1,39 @@
+import enum
+import uuid
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class ServiceTypes(str, enum.Enum):
+    """
+    Enum for litellm + litellm-adjacent services (redis/postgres/etc.)
+    """
+
+    REDIS = "redis"
+    DB = "postgres"
+    BATCH_WRITE_TO_DB = "batch_write_to_db"
+    RESET_BUDGET_JOB = "reset_budget_job"
+    LITELLM = "self"
+    ROUTER = "router"
+    AUTH = "auth"
+    PROXY_PRE_CALL = "proxy_pre_call"
+
+
+class ServiceLoggerPayload(BaseModel):
+    """
+    The payload logged during service success/failure
+    """
+
+    is_error: bool = Field(description="did an error occur")
+    error: Optional[str] = Field(None, description="what was the error")
+    service: ServiceTypes = Field(description="who is this for? - postgres/redis")
+    duration: float = Field(description="How long did the request take?")
+    call_type: str = Field(description="The call of the service, being made")
+
+    def to_json(self, **kwargs):
+        try:
+            return self.model_dump(**kwargs)  # noqa
+        except Exception as e:
+            # if using pydantic v1
+            return self.dict(**kwargs)
diff --git a/.venv/lib/python3.12/site-packages/litellm/types/utils.py b/.venv/lib/python3.12/site-packages/litellm/types/utils.py
new file mode 100644
index 00000000..a6654285
--- /dev/null
+++ b/.venv/lib/python3.12/site-packages/litellm/types/utils.py
@@ -0,0 +1,2081 @@
+import json
+import time
+import uuid
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+
+from aiohttp import FormData
+from openai._models import BaseModel as OpenAIObject
+from openai.types.audio.transcription_create_params import FileTypes  # type: ignore
+from openai.types.completion_usage import (
+    CompletionTokensDetails,
+    CompletionUsage,
+    PromptTokensDetails,
+)
+from openai.types.moderation import (
+    Categories,
+    CategoryAppliedInputTypes,
+    CategoryScores,
+)
+from openai.types.moderation_create_response import Moderation, ModerationCreateResponse
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
+from typing_extensions import Callable, Dict, Required, TypedDict, override
+
+import litellm
+
+from ..litellm_core_utils.core_helpers import map_finish_reason
+from .guardrails import GuardrailEventHooks
+from .llms.openai import (
+    Batch,
+    ChatCompletionThinkingBlock,
+    ChatCompletionToolCallChunk,
+    ChatCompletionUsageBlock,
+    OpenAIChatCompletionChunk,
+)
+from .rerank import RerankResponse
+
+
+def _generate_id():  # private helper function
+    return "chatcmpl-" + str(uuid.uuid4())
+
+
+class LiteLLMPydanticObjectBase(BaseModel):
+    """
+    Implements default functions, all pydantic objects should have.
+    """
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump(**kwargs)  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict(**kwargs)
+
+    def fields_set(self):
+        try:
+            return self.model_fields_set  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.__fields_set__
+
+    model_config = ConfigDict(protected_namespaces=())
+
+
+class LiteLLMCommonStrings(Enum):
+    redacted_by_litellm = "redacted by litellm. 'litellm.turn_off_message_logging=True'"
+    llm_provider_not_provided = "Unmapped LLM provider for this endpoint. You passed model={model}, custom_llm_provider={custom_llm_provider}. Check supported provider and route: https://docs.litellm.ai/docs/providers"
+
+
+SupportedCacheControls = ["ttl", "s-maxage", "no-cache", "no-store"]
+
+
+class CostPerToken(TypedDict):
+    input_cost_per_token: float
+    output_cost_per_token: float
+
+
+class ProviderField(TypedDict):
+    field_name: str
+    field_type: Literal["string"]
+    field_description: str
+    field_value: str
+
+
+class ProviderSpecificModelInfo(TypedDict, total=False):
+    supports_system_messages: Optional[bool]
+    supports_response_schema: Optional[bool]
+    supports_vision: Optional[bool]
+    supports_function_calling: Optional[bool]
+    supports_tool_choice: Optional[bool]
+    supports_assistant_prefill: Optional[bool]
+    supports_prompt_caching: Optional[bool]
+    supports_audio_input: Optional[bool]
+    supports_embedding_image_input: Optional[bool]
+    supports_audio_output: Optional[bool]
+    supports_pdf_input: Optional[bool]
+    supports_native_streaming: Optional[bool]
+    supports_parallel_function_calling: Optional[bool]
+
+
+class ModelInfoBase(ProviderSpecificModelInfo, total=False):
+    key: Required[str]  # the key in litellm.model_cost which is returned
+
+    max_tokens: Required[Optional[int]]
+    max_input_tokens: Required[Optional[int]]
+    max_output_tokens: Required[Optional[int]]
+    input_cost_per_token: Required[float]
+    cache_creation_input_token_cost: Optional[float]
+    cache_read_input_token_cost: Optional[float]
+    input_cost_per_character: Optional[float]  # only for vertex ai models
+    input_cost_per_audio_token: Optional[float]
+    input_cost_per_token_above_128k_tokens: Optional[float]  # only for vertex ai models
+    input_cost_per_character_above_128k_tokens: Optional[
+        float
+    ]  # only for vertex ai models
+    input_cost_per_query: Optional[float]  # only for rerank models
+    input_cost_per_image: Optional[float]  # only for vertex ai models
+    input_cost_per_audio_per_second: Optional[float]  # only for vertex ai models
+    input_cost_per_video_per_second: Optional[float]  # only for vertex ai models
+    input_cost_per_second: Optional[float]  # for OpenAI Speech models
+    input_cost_per_token_batches: Optional[float]
+    output_cost_per_token_batches: Optional[float]
+    output_cost_per_token: Required[float]
+    output_cost_per_character: Optional[float]  # only for vertex ai models
+    output_cost_per_audio_token: Optional[float]
+    output_cost_per_token_above_128k_tokens: Optional[
+        float
+    ]  # only for vertex ai models
+    output_cost_per_character_above_128k_tokens: Optional[
+        float
+    ]  # only for vertex ai models
+    output_cost_per_image: Optional[float]
+    output_vector_size: Optional[int]
+    output_cost_per_video_per_second: Optional[float]  # only for vertex ai models
+    output_cost_per_audio_per_second: Optional[float]  # only for vertex ai models
+    output_cost_per_second: Optional[float]  # for OpenAI Speech models
+
+    litellm_provider: Required[str]
+    mode: Required[
+        Literal[
+            "completion", "embedding", "image_generation", "chat", "audio_transcription"
+        ]
+    ]
+    tpm: Optional[int]
+    rpm: Optional[int]
+
+
+class ModelInfo(ModelInfoBase, total=False):
+    """
+    Model info for a given model, this is information found in litellm.model_prices_and_context_window.json
+    """
+
+    supported_openai_params: Required[Optional[List[str]]]
+
+
+class GenericStreamingChunk(TypedDict, total=False):
+    text: Required[str]
+    tool_use: Optional[ChatCompletionToolCallChunk]
+    is_finished: Required[bool]
+    finish_reason: Required[str]
+    usage: Required[Optional[ChatCompletionUsageBlock]]
+    index: int
+
+    # use this dict if you want to return any provider specific fields in the response
+    provider_specific_fields: Optional[Dict[str, Any]]
+
+
+from enum import Enum
+
+
+class CallTypes(Enum):
+    embedding = "embedding"
+    aembedding = "aembedding"
+    completion = "completion"
+    acompletion = "acompletion"
+    atext_completion = "atext_completion"
+    text_completion = "text_completion"
+    image_generation = "image_generation"
+    aimage_generation = "aimage_generation"
+    moderation = "moderation"
+    amoderation = "amoderation"
+    atranscription = "atranscription"
+    transcription = "transcription"
+    aspeech = "aspeech"
+    speech = "speech"
+    rerank = "rerank"
+    arerank = "arerank"
+    arealtime = "_arealtime"
+    create_batch = "create_batch"
+    acreate_batch = "acreate_batch"
+    aretrieve_batch = "aretrieve_batch"
+    retrieve_batch = "retrieve_batch"
+    pass_through = "pass_through_endpoint"
+    anthropic_messages = "anthropic_messages"
+    get_assistants = "get_assistants"
+    aget_assistants = "aget_assistants"
+    create_assistants = "create_assistants"
+    acreate_assistants = "acreate_assistants"
+    delete_assistant = "delete_assistant"
+    adelete_assistant = "adelete_assistant"
+    acreate_thread = "acreate_thread"
+    create_thread = "create_thread"
+    aget_thread = "aget_thread"
+    get_thread = "get_thread"
+    a_add_message = "a_add_message"
+    add_message = "add_message"
+    aget_messages = "aget_messages"
+    get_messages = "get_messages"
+    arun_thread = "arun_thread"
+    run_thread = "run_thread"
+    arun_thread_stream = "arun_thread_stream"
+    run_thread_stream = "run_thread_stream"
+    afile_retrieve = "afile_retrieve"
+    file_retrieve = "file_retrieve"
+    afile_delete = "afile_delete"
+    file_delete = "file_delete"
+    afile_list = "afile_list"
+    file_list = "file_list"
+    acreate_file = "acreate_file"
+    create_file = "create_file"
+    afile_content = "afile_content"
+    file_content = "file_content"
+    create_fine_tuning_job = "create_fine_tuning_job"
+    acreate_fine_tuning_job = "acreate_fine_tuning_job"
+    acancel_fine_tuning_job = "acancel_fine_tuning_job"
+    cancel_fine_tuning_job = "cancel_fine_tuning_job"
+    alist_fine_tuning_jobs = "alist_fine_tuning_jobs"
+    list_fine_tuning_jobs = "list_fine_tuning_jobs"
+    aretrieve_fine_tuning_job = "aretrieve_fine_tuning_job"
+    retrieve_fine_tuning_job = "retrieve_fine_tuning_job"
+    responses = "responses"
+    aresponses = "aresponses"
+
+
+CallTypesLiteral = Literal[
+    "embedding",
+    "aembedding",
+    "completion",
+    "acompletion",
+    "atext_completion",
+    "text_completion",
+    "image_generation",
+    "aimage_generation",
+    "moderation",
+    "amoderation",
+    "atranscription",
+    "transcription",
+    "aspeech",
+    "speech",
+    "rerank",
+    "arerank",
+    "_arealtime",
+    "create_batch",
+    "acreate_batch",
+    "pass_through_endpoint",
+    "anthropic_messages",
+    "aretrieve_batch",
+    "retrieve_batch",
+]
+
+
+class PassthroughCallTypes(Enum):
+    passthrough_image_generation = "passthrough-image-generation"
+
+
+class TopLogprob(OpenAIObject):
+    token: str
+    """The token."""
+
+    bytes: Optional[List[int]] = None
+    """A list of integers representing the UTF-8 bytes representation of the token.
+
+    Useful in instances where characters are represented by multiple tokens and
+    their byte representations must be combined to generate the correct text
+    representation. Can be `null` if there is no bytes representation for the token.
+    """
+
+    logprob: float
+    """The log probability of this token, if it is within the top 20 most likely
+    tokens.
+
+    Otherwise, the value `-9999.0` is used to signify that the token is very
+    unlikely.
+    """
+
+
+class ChatCompletionTokenLogprob(OpenAIObject):
+    token: str
+    """The token."""
+
+    bytes: Optional[List[int]] = None
+    """A list of integers representing the UTF-8 bytes representation of the token.
+
+    Useful in instances where characters are represented by multiple tokens and
+    their byte representations must be combined to generate the correct text
+    representation. Can be `null` if there is no bytes representation for the token.
+    """
+
+    logprob: float
+    """The log probability of this token, if it is within the top 20 most likely
+    tokens.
+
+    Otherwise, the value `-9999.0` is used to signify that the token is very
+    unlikely.
+    """
+
+    top_logprobs: List[TopLogprob]
+    """List of the most likely tokens and their log probability, at this token
+    position.
+
+    In rare cases, there may be fewer than the number of requested `top_logprobs`
+    returned.
+    """
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+
+class ChoiceLogprobs(OpenAIObject):
+    content: Optional[List[ChatCompletionTokenLogprob]] = None
+    """A list of message content tokens with log probability information."""
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+
+class FunctionCall(OpenAIObject):
+    arguments: str
+    name: Optional[str] = None
+
+
+class Function(OpenAIObject):
+    arguments: str
+    name: Optional[
+        str
+    ]  # can be None - openai e.g.: ChoiceDeltaToolCallFunction(arguments='{"', name=None), type=None)
+
+    def __init__(
+        self,
+        arguments: Optional[Union[Dict, str]],
+        name: Optional[str] = None,
+        **params,
+    ):
+        if arguments is None:
+            arguments = ""
+        elif isinstance(arguments, Dict):
+            arguments = json.dumps(arguments)
+        else:
+            arguments = arguments
+
+        name = name
+
+        # Build a dictionary with the structure your BaseModel expects
+        data = {"arguments": arguments, "name": name, **params}
+
+        super(Function, self).__init__(**data)
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class ChatCompletionDeltaToolCall(OpenAIObject):
+    id: Optional[str] = None
+    function: Function
+    type: Optional[str] = None
+    index: int
+
+
+class HiddenParams(OpenAIObject):
+    original_response: Optional[Union[str, Any]] = None
+    model_id: Optional[str] = None  # used in Router for individual deployments
+    api_base: Optional[str] = None  # returns api base used for making completion call
+
+    model_config = ConfigDict(extra="allow", protected_namespaces=())
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class ChatCompletionMessageToolCall(OpenAIObject):
+    def __init__(
+        self,
+        function: Union[Dict, Function],
+        id: Optional[str] = None,
+        type: Optional[str] = None,
+        **params,
+    ):
+        super(ChatCompletionMessageToolCall, self).__init__(**params)
+        if isinstance(function, Dict):
+            self.function = Function(**function)
+        else:
+            self.function = function
+
+        if id is not None:
+            self.id = id
+        else:
+            self.id = f"{uuid.uuid4()}"
+
+        if type is not None:
+            self.type = type
+        else:
+            self.type = "function"
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+from openai.types.chat.chat_completion_audio import ChatCompletionAudio
+
+
+class ChatCompletionAudioResponse(ChatCompletionAudio):
+
+    def __init__(
+        self,
+        data: str,
+        expires_at: int,
+        transcript: str,
+        id: Optional[str] = None,
+        **params,
+    ):
+        if id is not None:
+            id = id
+        else:
+            id = f"{uuid.uuid4()}"
+        super(ChatCompletionAudioResponse, self).__init__(
+            data=data, expires_at=expires_at, transcript=transcript, id=id, **params
+        )
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+"""
+Reference:
+ChatCompletionMessage(content='This is a test', role='assistant', function_call=None, tool_calls=None))
+"""
+
+
+def add_provider_specific_fields(
+    object: BaseModel, provider_specific_fields: Optional[Dict[str, Any]]
+):
+    if not provider_specific_fields:  # set if provider_specific_fields is not empty
+        return
+    setattr(object, "provider_specific_fields", provider_specific_fields)
+
+
+class Message(OpenAIObject):
+    content: Optional[str]
+    role: Literal["assistant", "user", "system", "tool", "function"]
+    tool_calls: Optional[List[ChatCompletionMessageToolCall]]
+    function_call: Optional[FunctionCall]
+    audio: Optional[ChatCompletionAudioResponse] = None
+    reasoning_content: Optional[str] = None
+    thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
+    provider_specific_fields: Optional[Dict[str, Any]] = Field(
+        default=None, exclude=True
+    )
+
+    def __init__(
+        self,
+        content: Optional[str] = None,
+        role: Literal["assistant"] = "assistant",
+        function_call=None,
+        tool_calls: Optional[list] = None,
+        audio: Optional[ChatCompletionAudioResponse] = None,
+        provider_specific_fields: Optional[Dict[str, Any]] = None,
+        reasoning_content: Optional[str] = None,
+        thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None,
+        **params,
+    ):
+        init_values: Dict[str, Any] = {
+            "content": content,
+            "role": role or "assistant",  # handle null input
+            "function_call": (
+                FunctionCall(**function_call) if function_call is not None else None
+            ),
+            "tool_calls": (
+                [
+                    (
+                        ChatCompletionMessageToolCall(**tool_call)
+                        if isinstance(tool_call, dict)
+                        else tool_call
+                    )
+                    for tool_call in tool_calls
+                ]
+                if tool_calls is not None and len(tool_calls) > 0
+                else None
+            ),
+        }
+
+        if audio is not None:
+            init_values["audio"] = audio
+
+        if thinking_blocks is not None:
+            init_values["thinking_blocks"] = thinking_blocks
+
+        if reasoning_content is not None:
+            init_values["reasoning_content"] = reasoning_content
+
+        super(Message, self).__init__(
+            **init_values,  # type: ignore
+            **params,
+        )
+
+        if audio is None:
+            # delete audio from self
+            # OpenAI compatible APIs like mistral API will raise an error if audio is passed in
+            del self.audio
+
+        if reasoning_content is None:
+            # ensure default response matches OpenAI spec
+            del self.reasoning_content
+
+        if thinking_blocks is None:
+            # ensure default response matches OpenAI spec
+            del self.thinking_blocks
+
+        add_provider_specific_fields(self, provider_specific_fields)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class Delta(OpenAIObject):
+    reasoning_content: Optional[str] = None
+    thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
+    provider_specific_fields: Optional[Dict[str, Any]] = Field(default=None)
+
+    def __init__(
+        self,
+        content=None,
+        role=None,
+        function_call=None,
+        tool_calls=None,
+        audio: Optional[ChatCompletionAudioResponse] = None,
+        reasoning_content: Optional[str] = None,
+        thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None,
+        **params,
+    ):
+        super(Delta, self).__init__(**params)
+        add_provider_specific_fields(self, params.get("provider_specific_fields", {}))
+        self.content = content
+        self.role = role
+        # Set default values and correct types
+        self.function_call: Optional[Union[FunctionCall, Any]] = None
+        self.tool_calls: Optional[List[Union[ChatCompletionDeltaToolCall, Any]]] = None
+        self.audio: Optional[ChatCompletionAudioResponse] = None
+
+        if reasoning_content is not None:
+            self.reasoning_content = reasoning_content
+        else:
+            # ensure default response matches OpenAI spec
+            del self.reasoning_content
+
+        if thinking_blocks is not None:
+            self.thinking_blocks = thinking_blocks
+        else:
+            # ensure default response matches OpenAI spec
+            del self.thinking_blocks
+
+        if function_call is not None and isinstance(function_call, dict):
+            self.function_call = FunctionCall(**function_call)
+        else:
+            self.function_call = function_call
+        if tool_calls is not None and isinstance(tool_calls, list):
+            self.tool_calls = []
+            for tool_call in tool_calls:
+                if isinstance(tool_call, dict):
+                    if tool_call.get("index", None) is None:
+                        tool_call["index"] = 0
+                    self.tool_calls.append(ChatCompletionDeltaToolCall(**tool_call))
+                elif isinstance(tool_call, ChatCompletionDeltaToolCall):
+                    self.tool_calls.append(tool_call)
+        else:
+            self.tool_calls = tool_calls
+
+        self.audio = audio
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class Choices(OpenAIObject):
+    def __init__(
+        self,
+        finish_reason=None,
+        index=0,
+        message: Optional[Union[Message, dict]] = None,
+        logprobs=None,
+        enhancements=None,
+        **params,
+    ):
+        super(Choices, self).__init__(**params)
+        if finish_reason is not None:
+            self.finish_reason = map_finish_reason(
+                finish_reason
+            )  # set finish_reason for all responses
+        else:
+            self.finish_reason = "stop"
+        self.index = index
+        if message is None:
+            self.message = Message()
+        else:
+            if isinstance(message, Message):
+                self.message = message
+            elif isinstance(message, dict):
+                self.message = Message(**message)
+        if logprobs is not None:
+            if isinstance(logprobs, dict):
+                self.logprobs = ChoiceLogprobs(**logprobs)
+            else:
+                self.logprobs = logprobs
+        if enhancements is not None:
+            self.enhancements = enhancements
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class CompletionTokensDetailsWrapper(
+    CompletionTokensDetails
+):  # wrapper for older openai versions
+    text_tokens: Optional[int] = None
+    """Text tokens generated by the model."""
+
+
+class PromptTokensDetailsWrapper(
+    PromptTokensDetails
+):  # wrapper for older openai versions
+    text_tokens: Optional[int] = None
+    """Text tokens sent to the model."""
+
+    image_tokens: Optional[int] = None
+    """Image tokens sent to the model."""
+
+
+class Usage(CompletionUsage):
+    _cache_creation_input_tokens: int = PrivateAttr(
+        0
+    )  # hidden param for prompt caching. Might change, once openai introduces their equivalent.
+    _cache_read_input_tokens: int = PrivateAttr(
+        0
+    )  # hidden param for prompt caching. Might change, once openai introduces their equivalent.
+
+    def __init__(
+        self,
+        prompt_tokens: Optional[int] = None,
+        completion_tokens: Optional[int] = None,
+        total_tokens: Optional[int] = None,
+        reasoning_tokens: Optional[int] = None,
+        prompt_tokens_details: Optional[Union[PromptTokensDetailsWrapper, dict]] = None,
+        completion_tokens_details: Optional[
+            Union[CompletionTokensDetailsWrapper, dict]
+        ] = None,
+        **params,
+    ):
+        # handle reasoning_tokens
+        _completion_tokens_details: Optional[CompletionTokensDetailsWrapper] = None
+        if reasoning_tokens:
+            completion_tokens_details = CompletionTokensDetailsWrapper(
+                reasoning_tokens=reasoning_tokens
+            )
+
+        # Ensure completion_tokens_details is properly handled
+        if completion_tokens_details:
+            if isinstance(completion_tokens_details, dict):
+                _completion_tokens_details = CompletionTokensDetailsWrapper(
+                    **completion_tokens_details
+                )
+            elif isinstance(completion_tokens_details, CompletionTokensDetails):
+                _completion_tokens_details = completion_tokens_details
+
+        ## DEEPSEEK MAPPING ##
+        if "prompt_cache_hit_tokens" in params and isinstance(
+            params["prompt_cache_hit_tokens"], int
+        ):
+            if prompt_tokens_details is None:
+                prompt_tokens_details = PromptTokensDetailsWrapper(
+                    cached_tokens=params["prompt_cache_hit_tokens"]
+                )
+
+        ## ANTHROPIC MAPPING ##
+        if "cache_read_input_tokens" in params and isinstance(
+            params["cache_read_input_tokens"], int
+        ):
+            if prompt_tokens_details is None:
+                prompt_tokens_details = PromptTokensDetailsWrapper(
+                    cached_tokens=params["cache_read_input_tokens"]
+                )
+
+        # handle prompt_tokens_details
+        _prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
+        if prompt_tokens_details:
+            if isinstance(prompt_tokens_details, dict):
+                _prompt_tokens_details = PromptTokensDetailsWrapper(
+                    **prompt_tokens_details
+                )
+            elif isinstance(prompt_tokens_details, PromptTokensDetails):
+                _prompt_tokens_details = prompt_tokens_details
+
+        super().__init__(
+            prompt_tokens=prompt_tokens or 0,
+            completion_tokens=completion_tokens or 0,
+            total_tokens=total_tokens or 0,
+            completion_tokens_details=_completion_tokens_details or None,
+            prompt_tokens_details=_prompt_tokens_details or None,
+        )
+
+        ## ANTHROPIC MAPPING ##
+        if "cache_creation_input_tokens" in params and isinstance(
+            params["cache_creation_input_tokens"], int
+        ):
+            self._cache_creation_input_tokens = params["cache_creation_input_tokens"]
+
+        if "cache_read_input_tokens" in params and isinstance(
+            params["cache_read_input_tokens"], int
+        ):
+            self._cache_read_input_tokens = params["cache_read_input_tokens"]
+
+        ## DEEPSEEK MAPPING ##
+        if "prompt_cache_hit_tokens" in params and isinstance(
+            params["prompt_cache_hit_tokens"], int
+        ):
+            self._cache_read_input_tokens = params["prompt_cache_hit_tokens"]
+
+        for k, v in params.items():
+            setattr(self, k, v)
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class StreamingChoices(OpenAIObject):
+    def __init__(
+        self,
+        finish_reason=None,
+        index=0,
+        delta: Optional[Delta] = None,
+        logprobs=None,
+        enhancements=None,
+        **params,
+    ):
+        # Fix Perplexity return both delta and message cause OpenWebUI repect text
+        # https://github.com/BerriAI/litellm/issues/8455
+        params.pop("message", None)
+        super(StreamingChoices, self).__init__(**params)
+        if finish_reason:
+            self.finish_reason = map_finish_reason(finish_reason)
+        else:
+            self.finish_reason = None
+        self.index = index
+        if delta is not None:
+
+            if isinstance(delta, Delta):
+                self.delta = delta
+            elif isinstance(delta, dict):
+                self.delta = Delta(**delta)
+        else:
+            self.delta = Delta()
+        if enhancements is not None:
+            self.enhancements = enhancements
+
+        if logprobs is not None and isinstance(logprobs, dict):
+            self.logprobs = ChoiceLogprobs(**logprobs)
+        else:
+            self.logprobs = logprobs  # type: ignore
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class StreamingChatCompletionChunk(OpenAIChatCompletionChunk):
+    def __init__(self, **kwargs):
+
+        new_choices = []
+        for choice in kwargs["choices"]:
+            new_choice = StreamingChoices(**choice).model_dump()
+            new_choices.append(new_choice)
+        kwargs["choices"] = new_choices
+
+        super().__init__(**kwargs)
+
+
+from openai.types.chat import ChatCompletionChunk
+
+
+class ModelResponseBase(OpenAIObject):
+    id: str
+    """A unique identifier for the completion."""
+
+    created: int
+    """The Unix timestamp (in seconds) of when the completion was created."""
+
+    model: Optional[str] = None
+    """The model used for completion."""
+
+    object: str
+    """The object type, which is always "text_completion" """
+
+    system_fingerprint: Optional[str] = None
+    """This fingerprint represents the backend configuration that the model runs with.
+
+    Can be used in conjunction with the `seed` request parameter to understand when
+    backend changes have been made that might impact determinism.
+    """
+
+    _hidden_params: dict = {}
+
+    _response_headers: Optional[dict] = None
+
+
+class ModelResponseStream(ModelResponseBase):
+    choices: List[StreamingChoices]
+    provider_specific_fields: Optional[Dict[str, Any]] = Field(default=None)
+
+    def __init__(
+        self,
+        choices: Optional[List[Union[StreamingChoices, dict, BaseModel]]] = None,
+        id: Optional[str] = None,
+        created: Optional[int] = None,
+        provider_specific_fields: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ):
+        if choices is not None and isinstance(choices, list):
+            new_choices = []
+            for choice in choices:
+                _new_choice = None
+                if isinstance(choice, StreamingChoices):
+                    _new_choice = choice
+                elif isinstance(choice, dict):
+                    _new_choice = StreamingChoices(**choice)
+                elif isinstance(choice, BaseModel):
+                    _new_choice = StreamingChoices(**choice.model_dump())
+                new_choices.append(_new_choice)
+            kwargs["choices"] = new_choices
+        else:
+            kwargs["choices"] = [StreamingChoices()]
+
+        if id is None:
+            id = _generate_id()
+        else:
+            id = id
+        if created is None:
+            created = int(time.time())
+        else:
+            created = created
+
+        if (
+            "usage" in kwargs
+            and kwargs["usage"] is not None
+            and isinstance(kwargs["usage"], dict)
+        ):
+            kwargs["usage"] = Usage(**kwargs["usage"])
+
+        kwargs["id"] = id
+        kwargs["created"] = created
+        kwargs["object"] = "chat.completion.chunk"
+        kwargs["provider_specific_fields"] = provider_specific_fields
+
+        super().__init__(**kwargs)
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class ModelResponse(ModelResponseBase):
+    choices: List[Union[Choices, StreamingChoices]]
+    """The list of completion choices the model generated for the input prompt."""
+
+    def __init__(
+        self,
+        id=None,
+        choices=None,
+        created=None,
+        model=None,
+        object=None,
+        system_fingerprint=None,
+        usage=None,
+        stream=None,
+        stream_options=None,
+        response_ms=None,
+        hidden_params=None,
+        _response_headers=None,
+        **params,
+    ) -> None:
+        if stream is not None and stream is True:
+            object = "chat.completion.chunk"
+            if choices is not None and isinstance(choices, list):
+                new_choices = []
+                for choice in choices:
+                    _new_choice = None
+                    if isinstance(choice, StreamingChoices):
+                        _new_choice = choice
+                    elif isinstance(choice, dict):
+                        _new_choice = StreamingChoices(**choice)
+                    elif isinstance(choice, BaseModel):
+                        _new_choice = StreamingChoices(**choice.model_dump())
+                    new_choices.append(_new_choice)
+                choices = new_choices
+            else:
+                choices = [StreamingChoices()]
+        else:
+            object = "chat.completion"
+            if choices is not None and isinstance(choices, list):
+                new_choices = []
+                for choice in choices:
+                    if isinstance(choice, Choices):
+                        _new_choice = choice  # type: ignore
+                    elif isinstance(choice, dict):
+                        _new_choice = Choices(**choice)  # type: ignore
+                    else:
+                        _new_choice = choice
+                    new_choices.append(_new_choice)
+                choices = new_choices
+            else:
+                choices = [Choices()]
+        if id is None:
+            id = _generate_id()
+        else:
+            id = id
+        if created is None:
+            created = int(time.time())
+        else:
+            created = created
+        model = model
+        if usage is not None:
+            if isinstance(usage, dict):
+                usage = Usage(**usage)
+            else:
+                usage = usage
+        elif stream is None or stream is False:
+            usage = Usage()
+        if hidden_params:
+            self._hidden_params = hidden_params
+
+        if _response_headers:
+            self._response_headers = _response_headers
+
+        init_values = {
+            "id": id,
+            "choices": choices,
+            "created": created,
+            "model": model,
+            "object": object,
+            "system_fingerprint": system_fingerprint,
+        }
+
+        if usage is not None:
+            init_values["usage"] = usage
+
+        super().__init__(
+            **init_values,
+            **params,
+        )
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class Embedding(OpenAIObject):
+    embedding: Union[list, str] = []
+    index: int
+    object: Literal["embedding"]
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+class EmbeddingResponse(OpenAIObject):
+    model: Optional[str] = None
+    """The model used for embedding."""
+
+    data: List
+    """The actual embedding value"""
+
+    object: Literal["list"]
+    """The object type, which is always "list" """
+
+    usage: Optional[Usage] = None
+    """Usage statistics for the embedding request."""
+
+    _hidden_params: dict = {}
+    _response_headers: Optional[Dict] = None
+    _response_ms: Optional[float] = None
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        usage: Optional[Usage] = None,
+        response_ms=None,
+        data: Optional[Union[List, List[Embedding]]] = None,
+        hidden_params=None,
+        _response_headers=None,
+        **params,
+    ):
+        object = "list"
+        if response_ms:
+            _response_ms = response_ms
+        else:
+            _response_ms = None
+        if data:
+            data = data
+        else:
+            data = []
+
+        if usage:
+            usage = usage
+        else:
+            usage = Usage()
+
+        if _response_headers:
+            self._response_headers = _response_headers
+
+        model = model
+        super().__init__(model=model, object=object, data=data, usage=usage)  # type: ignore
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class Logprobs(OpenAIObject):
+    text_offset: Optional[List[int]]
+    token_logprobs: Optional[List[Union[float, None]]]
+    tokens: Optional[List[str]]
+    top_logprobs: Optional[List[Union[Dict[str, float], None]]]
+
+
+class TextChoices(OpenAIObject):
+    def __init__(self, finish_reason=None, index=0, text=None, logprobs=None, **params):
+        super(TextChoices, self).__init__(**params)
+        if finish_reason:
+            self.finish_reason = map_finish_reason(finish_reason)
+        else:
+            self.finish_reason = None
+        self.index = index
+        if text is not None:
+            self.text = text
+        else:
+            self.text = None
+        if logprobs is None:
+            self.logprobs = None
+        else:
+            if isinstance(logprobs, dict):
+                self.logprobs = Logprobs(**logprobs)
+            else:
+                self.logprobs = logprobs
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class TextCompletionResponse(OpenAIObject):
+    """
+    {
+        "id": response["id"],
+        "object": "text_completion",
+        "created": response["created"],
+        "model": response["model"],
+        "choices": [
+        {
+            "text": response["choices"][0]["message"]["content"],
+            "index": response["choices"][0]["index"],
+            "logprobs": transformed_logprobs,
+            "finish_reason": response["choices"][0]["finish_reason"]
+        }
+        ],
+        "usage": response["usage"]
+    }
+    """
+
+    id: str
+    object: str
+    created: int
+    model: Optional[str]
+    choices: List[TextChoices]
+    usage: Optional[Usage]
+    _response_ms: Optional[int] = None
+    _hidden_params: HiddenParams
+
+    def __init__(
+        self,
+        id=None,
+        choices=None,
+        created=None,
+        model=None,
+        usage=None,
+        stream=False,
+        response_ms=None,
+        object=None,
+        **params,
+    ):
+        if stream:
+            object = "text_completion.chunk"
+            choices = [TextChoices()]
+        else:
+            object = "text_completion"
+            if choices is not None and isinstance(choices, list):
+                new_choices = []
+                for choice in choices:
+                    _new_choice = None
+                    if isinstance(choice, TextChoices):
+                        _new_choice = choice
+                    elif isinstance(choice, dict):
+                        _new_choice = TextChoices(**choice)
+                    new_choices.append(_new_choice)
+                choices = new_choices
+            else:
+                choices = [TextChoices()]
+        if object is not None:
+            object = object
+        if id is None:
+            id = _generate_id()
+        else:
+            id = id
+        if created is None:
+            created = int(time.time())
+        else:
+            created = created
+
+        model = model
+        if usage:
+            usage = usage
+        else:
+            usage = Usage()
+
+        super(TextCompletionResponse, self).__init__(
+            id=id,  # type: ignore
+            object=object,  # type: ignore
+            created=created,  # type: ignore
+            model=model,  # type: ignore
+            choices=choices,  # type: ignore
+            usage=usage,  # type: ignore
+            **params,
+        )
+
+        if response_ms:
+            self._response_ms = response_ms
+        else:
+            self._response_ms = None
+        self._hidden_params = HiddenParams()
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+
+from openai.types.images_response import Image as OpenAIImage
+
+
+class ImageObject(OpenAIImage):
+    """
+    Represents the url or the content of an image generated by the OpenAI API.
+
+    Attributes:
+    b64_json: The base64-encoded JSON of the generated image, if response_format is b64_json.
+    url: The URL of the generated image, if response_format is url (default).
+    revised_prompt: The prompt that was used to generate the image, if there was any revision to the prompt.
+
+    https://platform.openai.com/docs/api-reference/images/object
+    """
+
+    b64_json: Optional[str] = None
+    url: Optional[str] = None
+    revised_prompt: Optional[str] = None
+
+    def __init__(self, b64_json=None, url=None, revised_prompt=None, **kwargs):
+        super().__init__(b64_json=b64_json, url=url, revised_prompt=revised_prompt)  # type: ignore
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+from openai.types.images_response import ImagesResponse as OpenAIImageResponse
+
+
+class ImageResponse(OpenAIImageResponse):
+    _hidden_params: dict = {}
+    usage: Usage
+
+    def __init__(
+        self,
+        created: Optional[int] = None,
+        data: Optional[List[ImageObject]] = None,
+        response_ms=None,
+        usage: Optional[Usage] = None,
+        hidden_params: Optional[dict] = None,
+    ):
+        if response_ms:
+            _response_ms = response_ms
+        else:
+            _response_ms = None
+        if data:
+            data = data
+        else:
+            data = []
+
+        if created:
+            created = created
+        else:
+            created = int(time.time())
+
+        _data: List[OpenAIImage] = []
+        for d in data:
+            if isinstance(d, dict):
+                _data.append(ImageObject(**d))
+            elif isinstance(d, BaseModel):
+                _data.append(ImageObject(**d.model_dump()))
+        _usage = usage or Usage(
+            prompt_tokens=0,
+            completion_tokens=0,
+            total_tokens=0,
+        )
+        super().__init__(created=created, data=_data, usage=_usage)  # type: ignore
+        self._hidden_params = hidden_params or {}
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class TranscriptionResponse(OpenAIObject):
+    text: Optional[str] = None
+
+    _hidden_params: dict = {}
+    _response_headers: Optional[dict] = None
+
+    def __init__(self, text=None):
+        super().__init__(text=text)  # type: ignore
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        # Allow dictionary-style assignment of attributes
+        setattr(self, key, value)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class GenericImageParsingChunk(TypedDict):
+    type: str
+    media_type: str
+    data: str
+
+
+class ResponseFormatChunk(TypedDict, total=False):
+    type: Required[Literal["json_object", "text"]]
+    response_schema: dict
+
+
+class LoggedLiteLLMParams(TypedDict, total=False):
+    force_timeout: Optional[float]
+    custom_llm_provider: Optional[str]
+    api_base: Optional[str]
+    litellm_call_id: Optional[str]
+    model_alias_map: Optional[dict]
+    metadata: Optional[dict]
+    model_info: Optional[dict]
+    proxy_server_request: Optional[dict]
+    acompletion: Optional[bool]
+    preset_cache_key: Optional[str]
+    no_log: Optional[bool]
+    input_cost_per_second: Optional[float]
+    input_cost_per_token: Optional[float]
+    output_cost_per_token: Optional[float]
+    output_cost_per_second: Optional[float]
+    cooldown_time: Optional[float]
+
+
+class AdapterCompletionStreamWrapper:
+    def __init__(self, completion_stream):
+        self.completion_stream = completion_stream
+
+    def __iter__(self):
+        return self
+
+    def __aiter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            for chunk in self.completion_stream:
+                if chunk == "None" or chunk is None:
+                    raise Exception
+                return chunk
+            raise StopIteration
+        except StopIteration:
+            raise StopIteration
+        except Exception as e:
+            print(f"AdapterCompletionStreamWrapper - {e}")  # noqa
+
+    async def __anext__(self):
+        try:
+            async for chunk in self.completion_stream:
+                if chunk == "None" or chunk is None:
+                    raise Exception
+                return chunk
+            raise StopIteration
+        except StopIteration:
+            raise StopAsyncIteration
+
+
+class StandardLoggingUserAPIKeyMetadata(TypedDict):
+    user_api_key_hash: Optional[str]  # hash of the litellm virtual key used
+    user_api_key_alias: Optional[str]
+    user_api_key_org_id: Optional[str]
+    user_api_key_team_id: Optional[str]
+    user_api_key_user_id: Optional[str]
+    user_api_key_user_email: Optional[str]
+    user_api_key_team_alias: Optional[str]
+    user_api_key_end_user_id: Optional[str]
+
+
+class StandardLoggingPromptManagementMetadata(TypedDict):
+    prompt_id: str
+    prompt_variables: Optional[dict]
+    prompt_integration: str
+
+
+class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
+    """
+    Specific metadata k,v pairs logged to integration for easier cost tracking and prompt management
+    """
+
+    spend_logs_metadata: Optional[
+        dict
+    ]  # special param to log k,v pairs to spendlogs for a call
+    requester_ip_address: Optional[str]
+    requester_metadata: Optional[dict]
+    prompt_management_metadata: Optional[StandardLoggingPromptManagementMetadata]
+    applied_guardrails: Optional[List[str]]
+
+
+class StandardLoggingAdditionalHeaders(TypedDict, total=False):
+    x_ratelimit_limit_requests: int
+    x_ratelimit_limit_tokens: int
+    x_ratelimit_remaining_requests: int
+    x_ratelimit_remaining_tokens: int
+
+
+class StandardLoggingHiddenParams(TypedDict):
+    model_id: Optional[
+        str
+    ]  # id of the model in the router, separates multiple models with the same name but different credentials
+    cache_key: Optional[str]
+    api_base: Optional[str]
+    response_cost: Optional[str]
+    litellm_overhead_time_ms: Optional[float]
+    additional_headers: Optional[StandardLoggingAdditionalHeaders]
+    batch_models: Optional[List[str]]
+    litellm_model_name: Optional[str]  # the model name sent to the provider by litellm
+
+
+class StandardLoggingModelInformation(TypedDict):
+    model_map_key: str
+    model_map_value: Optional[ModelInfo]
+
+
+class StandardLoggingModelCostFailureDebugInformation(TypedDict, total=False):
+    """
+    Debug information, if cost tracking fails.
+
+    Avoid logging sensitive information like response or optional params
+    """
+
+    error_str: Required[str]
+    traceback_str: Required[str]
+    model: str
+    cache_hit: Optional[bool]
+    custom_llm_provider: Optional[str]
+    base_model: Optional[str]
+    call_type: str
+    custom_pricing: Optional[bool]
+
+
+class StandardLoggingPayloadErrorInformation(TypedDict, total=False):
+    error_code: Optional[str]
+    error_class: Optional[str]
+    llm_provider: Optional[str]
+    traceback: Optional[str]
+    error_message: Optional[str]
+
+
+class StandardLoggingGuardrailInformation(TypedDict, total=False):
+    guardrail_name: Optional[str]
+    guardrail_mode: Optional[Union[GuardrailEventHooks, List[GuardrailEventHooks]]]
+    guardrail_response: Optional[Union[dict, str]]
+    guardrail_status: Literal["success", "failure"]
+
+
+StandardLoggingPayloadStatus = Literal["success", "failure"]
+
+
+class StandardLoggingPayload(TypedDict):
+    id: str
+    trace_id: str  # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries)
+    call_type: str
+    stream: Optional[bool]
+    response_cost: float
+    response_cost_failure_debug_info: Optional[
+        StandardLoggingModelCostFailureDebugInformation
+    ]
+    status: StandardLoggingPayloadStatus
+    custom_llm_provider: Optional[str]
+    total_tokens: int
+    prompt_tokens: int
+    completion_tokens: int
+    startTime: float  # Note: making this camelCase was a mistake, everything should be snake case
+    endTime: float
+    completionStartTime: float
+    response_time: float
+    model_map_information: StandardLoggingModelInformation
+    model: str
+    model_id: Optional[str]
+    model_group: Optional[str]
+    api_base: str
+    metadata: StandardLoggingMetadata
+    cache_hit: Optional[bool]
+    cache_key: Optional[str]
+    saved_cache_cost: float
+    request_tags: list
+    end_user: Optional[str]
+    requester_ip_address: Optional[str]
+    messages: Optional[Union[str, list, dict]]
+    response: Optional[Union[str, list, dict]]
+    error_str: Optional[str]
+    error_information: Optional[StandardLoggingPayloadErrorInformation]
+    model_parameters: dict
+    hidden_params: StandardLoggingHiddenParams
+    guardrail_information: Optional[StandardLoggingGuardrailInformation]
+
+
+from typing import AsyncIterator, Iterator
+
+
+class CustomStreamingDecoder:
+    async def aiter_bytes(
+        self, iterator: AsyncIterator[bytes]
+    ) -> AsyncIterator[
+        Optional[Union[GenericStreamingChunk, StreamingChatCompletionChunk]]
+    ]:
+        raise NotImplementedError
+
+    def iter_bytes(
+        self, iterator: Iterator[bytes]
+    ) -> Iterator[Optional[Union[GenericStreamingChunk, StreamingChatCompletionChunk]]]:
+        raise NotImplementedError
+
+
+class StandardPassThroughResponseObject(TypedDict):
+    response: str
+
+
+OPENAI_RESPONSE_HEADERS = [
+    "x-ratelimit-remaining-requests",
+    "x-ratelimit-remaining-tokens",
+    "x-ratelimit-limit-requests",
+    "x-ratelimit-limit-tokens",
+    "x-ratelimit-reset-requests",
+    "x-ratelimit-reset-tokens",
+]
+
+
+class StandardCallbackDynamicParams(TypedDict, total=False):
+    # Langfuse dynamic params
+    langfuse_public_key: Optional[str]
+    langfuse_secret: Optional[str]
+    langfuse_secret_key: Optional[str]
+    langfuse_host: Optional[str]
+
+    # GCS dynamic params
+    gcs_bucket_name: Optional[str]
+    gcs_path_service_account: Optional[str]
+
+    # Langsmith dynamic params
+    langsmith_api_key: Optional[str]
+    langsmith_project: Optional[str]
+    langsmith_base_url: Optional[str]
+
+    # Humanloop dynamic params
+    humanloop_api_key: Optional[str]
+
+    # Arize dynamic params
+    arize_api_key: Optional[str]
+    arize_space_key: Optional[str]
+
+    # Logging settings
+    turn_off_message_logging: Optional[bool]  # when true will not log messages
+
+
+all_litellm_params = [
+    "metadata",
+    "litellm_metadata",
+    "litellm_trace_id",
+    "tags",
+    "acompletion",
+    "aimg_generation",
+    "atext_completion",
+    "text_completion",
+    "caching",
+    "mock_response",
+    "mock_timeout",
+    "disable_add_transform_inline_image_block",
+    "api_key",
+    "api_version",
+    "prompt_id",
+    "provider_specific_header",
+    "prompt_variables",
+    "api_base",
+    "force_timeout",
+    "logger_fn",
+    "verbose",
+    "custom_llm_provider",
+    "litellm_logging_obj",
+    "litellm_call_id",
+    "use_client",
+    "id",
+    "fallbacks",
+    "azure",
+    "headers",
+    "model_list",
+    "num_retries",
+    "context_window_fallback_dict",
+    "retry_policy",
+    "retry_strategy",
+    "roles",
+    "final_prompt_value",
+    "bos_token",
+    "eos_token",
+    "request_timeout",
+    "complete_response",
+    "self",
+    "client",
+    "rpm",
+    "tpm",
+    "max_parallel_requests",
+    "input_cost_per_token",
+    "output_cost_per_token",
+    "input_cost_per_second",
+    "output_cost_per_second",
+    "hf_model_name",
+    "model_info",
+    "proxy_server_request",
+    "preset_cache_key",
+    "caching_groups",
+    "ttl",
+    "cache",
+    "no-log",
+    "base_model",
+    "stream_timeout",
+    "supports_system_message",
+    "region_name",
+    "allowed_model_region",
+    "model_config",
+    "fastest_response",
+    "cooldown_time",
+    "cache_key",
+    "max_retries",
+    "azure_ad_token_provider",
+    "tenant_id",
+    "client_id",
+    "azure_username",
+    "azure_password",
+    "client_secret",
+    "user_continue_message",
+    "configurable_clientside_auth_params",
+    "weight",
+    "ensure_alternating_roles",
+    "assistant_continue_message",
+    "user_continue_message",
+    "fallback_depth",
+    "max_fallbacks",
+    "max_budget",
+    "budget_duration",
+    "use_in_pass_through",
+    "merge_reasoning_content_in_choices",
+    "litellm_credential_name",
+] + list(StandardCallbackDynamicParams.__annotations__.keys())
+
+
+class KeyGenerationConfig(TypedDict, total=False):
+    required_params: List[
+        str
+    ]  # specify params that must be present in the key generation request
+
+
+class TeamUIKeyGenerationConfig(KeyGenerationConfig):
+    allowed_team_member_roles: List[str]
+
+
+class PersonalUIKeyGenerationConfig(KeyGenerationConfig):
+    allowed_user_roles: List[str]
+
+
+class StandardKeyGenerationConfig(TypedDict, total=False):
+    team_key_generation: TeamUIKeyGenerationConfig
+    personal_key_generation: PersonalUIKeyGenerationConfig
+
+
+class BudgetConfig(BaseModel):
+    max_budget: Optional[float] = None
+    budget_duration: Optional[str] = None
+    tpm_limit: Optional[int] = None
+    rpm_limit: Optional[int] = None
+
+    def __init__(self, **data: Any) -> None:
+        # Map time_period to budget_duration if present
+        if "time_period" in data:
+            data["budget_duration"] = data.pop("time_period")
+
+        # Map budget_limit to max_budget if present
+        if "budget_limit" in data:
+            data["max_budget"] = data.pop("budget_limit")
+
+        super().__init__(**data)
+
+
+GenericBudgetConfigType = Dict[str, BudgetConfig]
+
+
+class LlmProviders(str, Enum):
+    OPENAI = "openai"
+    OPENAI_LIKE = "openai_like"  # embedding only
+    JINA_AI = "jina_ai"
+    XAI = "xai"
+    CUSTOM_OPENAI = "custom_openai"
+    TEXT_COMPLETION_OPENAI = "text-completion-openai"
+    COHERE = "cohere"
+    COHERE_CHAT = "cohere_chat"
+    CLARIFAI = "clarifai"
+    ANTHROPIC = "anthropic"
+    ANTHROPIC_TEXT = "anthropic_text"
+    REPLICATE = "replicate"
+    HUGGINGFACE = "huggingface"
+    TOGETHER_AI = "together_ai"
+    OPENROUTER = "openrouter"
+    VERTEX_AI = "vertex_ai"
+    VERTEX_AI_BETA = "vertex_ai_beta"
+    GEMINI = "gemini"
+    AI21 = "ai21"
+    BASETEN = "baseten"
+    AZURE = "azure"
+    AZURE_TEXT = "azure_text"
+    AZURE_AI = "azure_ai"
+    SAGEMAKER = "sagemaker"
+    SAGEMAKER_CHAT = "sagemaker_chat"
+    BEDROCK = "bedrock"
+    VLLM = "vllm"
+    NLP_CLOUD = "nlp_cloud"
+    PETALS = "petals"
+    OOBABOOGA = "oobabooga"
+    OLLAMA = "ollama"
+    OLLAMA_CHAT = "ollama_chat"
+    DEEPINFRA = "deepinfra"
+    PERPLEXITY = "perplexity"
+    MISTRAL = "mistral"
+    GROQ = "groq"
+    NVIDIA_NIM = "nvidia_nim"
+    CEREBRAS = "cerebras"
+    AI21_CHAT = "ai21_chat"
+    VOLCENGINE = "volcengine"
+    CODESTRAL = "codestral"
+    TEXT_COMPLETION_CODESTRAL = "text-completion-codestral"
+    DEEPSEEK = "deepseek"
+    SAMBANOVA = "sambanova"
+    MARITALK = "maritalk"
+    VOYAGE = "voyage"
+    CLOUDFLARE = "cloudflare"
+    XINFERENCE = "xinference"
+    FIREWORKS_AI = "fireworks_ai"
+    FRIENDLIAI = "friendliai"
+    WATSONX = "watsonx"
+    WATSONX_TEXT = "watsonx_text"
+    TRITON = "triton"
+    PREDIBASE = "predibase"
+    DATABRICKS = "databricks"
+    EMPOWER = "empower"
+    GITHUB = "github"
+    CUSTOM = "custom"
+    LITELLM_PROXY = "litellm_proxy"
+    HOSTED_VLLM = "hosted_vllm"
+    LM_STUDIO = "lm_studio"
+    GALADRIEL = "galadriel"
+    INFINITY = "infinity"
+    DEEPGRAM = "deepgram"
+    AIOHTTP_OPENAI = "aiohttp_openai"
+    LANGFUSE = "langfuse"
+    HUMANLOOP = "humanloop"
+    TOPAZ = "topaz"
+    ASSEMBLYAI = "assemblyai"
+    SNOWFLAKE = "snowflake"
+
+
+# Create a set of all provider values for quick lookup
+LlmProvidersSet = {provider.value for provider in LlmProviders}
+
+
+class LiteLLMLoggingBaseClass:
+    """
+    Base class for logging pre and post call
+
+    Meant to simplify type checking for logging obj.
+    """
+
+    def pre_call(self, input, api_key, model=None, additional_args={}):
+        pass
+
+    def post_call(
+        self, original_response, input=None, api_key=None, additional_args={}
+    ):
+        pass
+
+
+class CustomHuggingfaceTokenizer(TypedDict):
+    identifier: str
+    revision: str  # usually 'main'
+    auth_token: Optional[str]
+
+
+class LITELLM_IMAGE_VARIATION_PROVIDERS(Enum):
+    """
+    Try using an enum for endpoints. This should make it easier to track what provider is supported for what endpoint.
+    """
+
+    OPENAI = LlmProviders.OPENAI.value
+    TOPAZ = LlmProviders.TOPAZ.value
+
+
+class HttpHandlerRequestFields(TypedDict, total=False):
+    data: dict  # request body
+    params: dict  # query params
+    files: dict  # file uploads
+    content: Any  # raw content
+
+
+class ProviderSpecificHeader(TypedDict):
+    custom_llm_provider: str
+    extra_headers: dict
+
+
+class SelectTokenizerResponse(TypedDict):
+    type: Literal["openai_tokenizer", "huggingface_tokenizer"]
+    tokenizer: Any
+
+
+class LiteLLMBatch(Batch):
+    _hidden_params: dict = {}
+    usage: Optional[Usage] = None
+
+    def __contains__(self, key):
+        # Define custom behavior for the 'in' operator
+        return hasattr(self, key)
+
+    def get(self, key, default=None):
+        # Custom .get() method to access attributes with a default value if the attribute doesn't exist
+        return getattr(self, key, default)
+
+    def __getitem__(self, key):
+        # Allow dictionary-style access to attributes
+        return getattr(self, key)
+
+    def json(self, **kwargs):  # type: ignore
+        try:
+            return self.model_dump()  # noqa
+        except Exception:
+            # if using pydantic v1
+            return self.dict()
+
+
+class RawRequestTypedDict(TypedDict, total=False):
+    raw_request_api_base: Optional[str]
+    raw_request_body: Optional[dict]
+    raw_request_headers: Optional[dict]
+    error: Optional[str]
+
+
+class CredentialBase(BaseModel):
+    credential_name: str
+    credential_info: dict
+
+
+class CredentialItem(CredentialBase):
+    credential_values: dict
+
+
+class CreateCredentialItem(CredentialBase):
+    credential_values: Optional[dict] = None
+    model_id: Optional[str] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_credential_params(cls, values):
+        if not values.get("credential_values") and not values.get("model_id"):
+            raise ValueError("Either credential_values or model_id must be set")
+        return values