1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
"""
Helper functions to get/set num success and num failures per deployment
set_deployment_failures_for_current_minute
set_deployment_successes_for_current_minute
get_deployment_failures_for_current_minute
get_deployment_successes_for_current_minute
"""
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from litellm.router import Router as _Router
LitellmRouter = _Router
else:
LitellmRouter = Any
def increment_deployment_successes_for_current_minute(
litellm_router_instance: LitellmRouter,
deployment_id: str,
) -> str:
"""
In-Memory: Increments the number of successes for the current minute for a deployment_id
"""
key = f"{deployment_id}:successes"
litellm_router_instance.cache.increment_cache(
local_only=True,
key=key,
value=1,
ttl=60,
)
return key
def increment_deployment_failures_for_current_minute(
litellm_router_instance: LitellmRouter,
deployment_id: str,
):
"""
In-Memory: Increments the number of failures for the current minute for a deployment_id
"""
key = f"{deployment_id}:fails"
litellm_router_instance.cache.increment_cache(
local_only=True,
key=key,
value=1,
ttl=60,
)
def get_deployment_successes_for_current_minute(
litellm_router_instance: LitellmRouter,
deployment_id: str,
) -> int:
"""
Returns the number of successes for the current minute for a deployment_id
Returns 0 if no value found
"""
key = f"{deployment_id}:successes"
return (
litellm_router_instance.cache.get_cache(
local_only=True,
key=key,
)
or 0
)
def get_deployment_failures_for_current_minute(
litellm_router_instance: LitellmRouter,
deployment_id: str,
) -> int:
"""
Returns the number of fails for the current minute for a deployment_id
Returns 0 if no value found
"""
key = f"{deployment_id}:fails"
return (
litellm_router_instance.cache.get_cache(
local_only=True,
key=key,
)
or 0
)
|
