about summary refs log tree commit diff
path: root/aisearch/src/web/app.py
blob: 68d4a3dbfe99d796ab18fc887079417df72fd653 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
import asyncio
import concurrent.futures
import os
import uuid

import dspy
import quart
import quart_flask_patch  # noqa: F401
import torch
from flask_caching import Cache
from flask_limiter import Limiter
from flask_limiter.util import get_remote_address
from gnais.config import Config
from gnais.search.prompts import GN_FACT_EXTRACTION_PROMPT, GN_UPDATE_MEMORY_PROMPT
from gnais.search.ragent import hybrid_search
from markupsafe import escape
from mem0 import Memory
from mem0.configs.base import MemoryConfig
from quart import (
    Quart,
    Response,
    flash,
    jsonify,
    redirect,
    render_template,
    request,
    session,
    url_for,
)
from quart.typing import ResponseReturnValue
from quart_auth import (
    AuthUser,
    QuartAuth,
    Unauthorized,
    current_user,
    login_required,
    login_user,
    logout_user,
)

app = Quart(__name__)
app.config.from_object(Config)

# Basic Login
app.secret_key = Config.SECRET_KEY
QuartAuth(app)

# Set up template and static directories
template_dir = os.path.join(os.path.dirname(__file__), "templates")
app.template_folder = template_dir
static_dir = os.path.join(os.path.dirname(__file__), "static")
app.static_folder = static_dir


limiter = Limiter(
    get_remote_address,
    app=app,
    default_limits=["100 per minute"],
    storage_uri="redis://localhost:6379",
    strategy="fixed-window",
)

cache = Cache(config={"CACHE_TYPE": "RedisCache"})
cache.init_app(app)

#  Bootstrapping our model
#  NOTE: The backend serving localhost:7501 should also run on the GPU
#  (e.g. vLLM with --device cuda) for maximum inference throughput.
torch.manual_seed(app.config["SEED"])
if torch.cuda.is_available():
    torch.cuda.manual_seed_all(app.config["SEED"])


dspy.configure(lm=app.config["DEFAULT_LLM"])


@app.before_serving
async def _set_default_executor():
    loop = asyncio.get_running_loop()
    loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=64))


#  Shared mem0 memory instance (loaded once at server startup)
_MEMORY = Memory(
    config=MemoryConfig(
        custom_fact_extraction_prompt=GN_FACT_EXTRACTION_PROMPT,
        custom_update_extraction_prompt=GN_UPDATE_MEMORY_PROMPT,
        llm={
            "provider": "litellm",
            "config": {
                "model": Config.MEMORY_MODEL,
                "temperature": 0.5,
                "max_tokens": 100_000,
                "api_key": Config.API_KEY,
            },
        },
        embedder={
            "provider": "huggingface",
            "config": {
                "model": "Qwen/Qwen3-Embedding-0.6B",
                "embedding_dims": 1024,
                "model_kwargs": {
                    "trust_remote_code": True,
                    "device": "cuda" if torch.cuda.is_available() else "cpu",
                },
            },
        },
        vector_store={
            "provider": "chroma",
            "config": {
                "collection_name": "mem0",
                "host": "localhost",
                "port": 8001,
            },
        },
    )
)


def _format_sse(event: str, data: str) -> str:
    lines = data.splitlines() or [""]
    payload = [f"event: {event}"]
    payload.extend(f"data: {line}" for line in lines)
    payload.append("")
    return "\n".join(payload) + "\n"


def _stream_chunk_markup(content: str, error: bool = False) -> str:
    css_class = "stream-fragment stream-fragment-error" if error else "stream-fragment"
    return f"<span class='{css_class}'>{escape(content)}</span>"


def _stream_status_markup(label: str, tone: str = "working") -> str:
    return f"<span class='stream-stage-badge is-{tone}'>{escape(label)}</span>"


def _stream_final_status_markup(message: str, tone: str = "waiting") -> str:
    icon = "🔍" if tone == "waiting" else "✨"
    return (
        f"<div class='stream-final-status is-{tone}'>"
        f"<div class='stream-final-status-icon'>{icon}</div>"
        f"<div class='stream-final-status-text'>{escape(message)}</div>"
        "</div>"
    )


@app.route("/login", methods=["GET", "POST"])
async def login():
    # KLUDGE: Set a static password for now to prevent token abuse
    # from non-GN testers.
    users = {"test@balg-qa.genenetwork.org": os.environ.get("USER_PASS")}
    if quart.request.method == "GET":
        return await render_template("login.html")
    form = await request.form
    email, password = form.get("email"), form.get("password")

    if email in users and users[email] == password:
        login_user(AuthUser(email))
        return redirect(url_for("index"))

    flash("Set the correct user/password")
    return redirect(url_for("login"))


@app.route("/logout")
async def logout():
    logout_user()
    return "Logged out"


@app.errorhandler(Unauthorized)
async def redirect_to_login(*_: Exception) -> ResponseReturnValue:
    return redirect(url_for("login"))


@app.route("/")
@login_required
async def index():
    """Serve the AI search web interface."""
    return await render_template("index.html")


@app.route("/search/stream-shell", methods=["GET"])
@limiter.limit("300 per day")
@login_required
async def search_stream_shell():
    """Shell endpoint that returns the streaming message container."""
    query = request.args.get("q", "").strip()
    if not query:
        return "<div class='error-message'>Missing query parameter 'q'</div>", 400
    if len(query) > 1000:
        return "<div class='error-message'>Query too long</div>", 400
    if "user_id" not in session:
        session["user_id"] = str(uuid.uuid4())
    return await render_template("partials/stream_shell.html", query=query)


@app.route("/search/stream", methods=["GET"])
@limiter.limit("300 per day")
@login_required
async def search_stream():
    """SSE endpoint for live hybrid search updates."""
    query = request.args.get("q", "").strip()
    if not query:
        return Response(
            _format_sse(
                "final_html",
                "<div class='error-message'>Missing query parameter 'q'</div>",
            ),
            mimetype="text/event-stream",
        )
    if len(query) > 1000:
        return Response(
            _format_sse(
                "final_html", "<div class='error-message'>Query too long</div>"
            ),
            mimetype="text/event-stream",
        )

    user_id = session.get("user_id")
    if not user_id:
        session["user_id"] = str(uuid.uuid4())
        user_id = session["user_id"]

    async def event_stream():
        completed = set()
        final_sent = False

        yield _format_sse("search_state", _stream_status_markup("Streaming", "working"))
        yield _format_sse(
            "final_html",
            _stream_final_status_markup("Waiting for searches to complete…", "waiting"),
        )
        try:
            async for event in hybrid_search(query, user_id=user_id, memory=_MEMORY):
                if final_sent:
                    break

                source = event["source"]
                kind = event["kind"]
                content = event["content"]

                if source in {"rag", "grag", "agent"} and kind == "status":
                    yield _format_sse(
                        f"{source}_chunk",
                        f"<div class='stream-status-msg'>{escape(content)}</div>",
                    )
                    continue

                if source in {"rag", "grag", "agent"} and kind == "chunk":
                    yield _format_sse(
                        f"{source}_chunk",
                        _stream_chunk_markup(content),
                    )
                    continue

                if source in {"rag", "grag", "agent"} and kind == "final":
                    yield _format_sse(
                        f"{source}_final_html",
                        str(content),
                    )
                    continue

                if source in {"rag", "grag", "agent"} and kind == "timing":
                    yield _format_sse(
                        f"{source}_timing",
                        escape(content),
                    )
                    continue

                if source in {"rag", "grag", "agent"} and kind == "error":
                    yield _format_sse(
                        f"{source}_chunk",
                        _stream_chunk_markup(content, error=True),
                    )
                    yield _format_sse(
                        f"{source}_done",
                        _stream_status_markup("Error", "error"),
                    )
                    completed.add(source)
                    if len(completed) == 3 and not final_sent:
                        yield _format_sse(
                            "search_state",
                            _stream_status_markup("Synthesizing", "working"),
                        )
                        yield _format_sse(
                            "final_html",
                            "<div class='synthesis-stream' sse-swap='synthesis_chunk' hx-swap='beforeend'></div>",
                        )
                    continue

                if source in {"rag", "grag", "agent"} and kind == "done":
                    yield _format_sse(
                        f"{source}_done",
                        _stream_status_markup("Complete", "complete"),
                    )
                    completed.add(source)
                    if len(completed) == 3 and not final_sent:
                        yield _format_sse(
                            "search_state",
                            _stream_status_markup("Synthesizing", "working"),
                        )
                        yield _format_sse(
                            "final_html",
                            "<div class='synthesis-stream' sse-swap='synthesis_chunk' hx-swap='beforeend'></div>",
                        )
                    continue

                if source == "synthesis" and kind == "chunk":
                    yield _format_sse("synthesis_chunk", content)
                    continue

                if source == "hybrid" and kind == "timing":
                    yield _format_sse(
                        "total_timing",
                        f"Total: {escape(content)}",
                    )
                    continue

                if source == "hybrid" and kind == "final":
                    final_sent = True
                    # await asyncio.to_thread(cache.set, cache_key, content, timeout=3600)
                    yield _format_sse("final_html", content)
                    yield _format_sse(
                        "search_state", _stream_status_markup("Complete", "complete")
                    )
                    yield _format_sse("stream_end", "<div></div>")
                    return
        except Exception as exc:
            error_html = f"<div class='error-display'>{escape(str(exc))}</div>"
            yield _format_sse("final_html", error_html)
            yield _format_sse("stream_end", "<div></div>")

    response = Response(
        event_stream(),
        mimetype="text/event-stream",
        headers={
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "X-Accel-Buffering": "no",
        },
    )
    response.timeout = None
    return response


if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5001, debug=True)