diff --git a/.env.example b/.env.example index 4dcaae1..e2ff555 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ # --- Application --- APP_ENV=development # Set to true for development only. Production MUST use false. -APP_DEBUG=false +APP_DEBUG=true APP_HOST=0.0.0.0 APP_PORT=8000 # SECURITY: Change this to a random secret key in production! @@ -59,31 +59,66 @@ OLLAMA_MODEL=llama3 # --- Embedding --- # Provider: local (HuggingFace) | api (OpenAI) | mock EMBEDDING_PROVIDER=local -EMBEDDING_MODEL=BAAI/bge-m3 +EMBEDDING_MODEL=Qwen/Qwen3-Embedding-8B EMBEDDING_API_KEY= -RERANKER_MODEL=BAAI/bge-reranker-v2-m3 +RERANKER_MODEL=tomaarsen/Qwen3-Reranker-8B-seq-cls # --- OCR --- # PaddleOCR language: ch (Chinese+English) | en (English only) OCR_LANG=ch -# --- PDF Parsing --- +# --- PDF Parsing / MinerU --- # Parser selection: auto (pdfplumber first, fallback to MinerU) | mineru | pdfplumber -PDF_PARSER=auto +PDF_PARSER=mineru # MinerU independent API service URL MINERU_API_URL=http://localhost:8010 # MinerU backend: pipeline | hybrid-auto-engine | vlm-auto-engine MINERU_BACKEND=pipeline # Timeout per PDF in seconds -MINERU_TIMEOUT=300 +MINERU_TIMEOUT=8000 +# Auto start/stop MinerU subprocess (true = Omelette manages MinerU lifecycle) +MINERU_AUTO_MANAGE=true +# Conda environment name for MinerU (used with conda run) +MINERU_CONDA_ENV=mineru +# Stop MinerU after N seconds idle (0 = never auto-stop) +MINERU_TTL_SECONDS=600 +# MinerU startup timeout in seconds +MINERU_STARTUP_TIMEOUT=120 +# GPU IDs for MinerU (empty = inherit CUDA_VISIBLE_DEVICES) +MINERU_GPU_IDS= # --- GPU --- # Comma-separated GPU IDs for OCR/embedding tasks -CUDA_VISIBLE_DEVICES=0,3 +CUDA_VISIBLE_DEVICES= + +# Auto-unload GPU models after N seconds idle (0 = never auto-unload) +MODEL_TTL_SECONDS=300 +# TTL check interval in seconds +MODEL_TTL_CHECK_INTERVAL=30 + +# GPU preset mode: conservative | balanced | aggressive +# conservative: batch=1, parallel=1, safe for small VRAM / debugging +# balanced: batch=8/16, auto parallel, good default +# aggressive: batch=32/50, parallel=GPU*2, max throughput (32G+ VRAM) +GPU_MODE=balanced + +# Per-service overrides (0 = follow GPU_MODE preset) +# EMBED_BATCH_SIZE=0 +# RERANK_BATCH_SIZE=0 + +# Pin models to specific GPU index (-1 = auto-select by free memory) +# EMBED_GPU_ID=-1 +# RERANK_GPU_ID=-1 + +# Comma-separated GPU IDs for OCR workers (empty = use all visible GPUs) +# OCR_GPU_IDS= + +# Max parallel OCR tasks. 0=auto (GPU count, or GPU*2 in aggressive mode) +# OCR_PARALLEL_LIMIT=0 # --- Network Proxy --- -HTTP_PROXY=http://127.0.0.1:20171/ -HTTPS_PROXY=http://127.0.0.1:20171/ +# HTTP_PROXY=http://your-proxy:port +# HTTPS_PROXY=http://your-proxy:port # --- HuggingFace Mirror --- # For users in China, set to https://hf-mirror.com to speed up model downloads diff --git a/README.md b/README.md index fa2a83e..7bcd6a2 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Omelette automates the full research literature pipeline — from keyword manage Multi-channel download via Unpaywall, arXiv, and direct URL fallback strategies. **📝 OCR Processing** - Native text extraction with PaddleOCR GPU fallback for scanned documents. + Native text extraction via MinerU (auto-managed subprocess) or PaddleOCR GPU fallback. **🧠 RAG Knowledge Base** LlamaIndex engine with ChromaDB, GPU-aware embeddings, hybrid retrieval, and cited answers. @@ -69,7 +69,10 @@ Omelette automates the full research literature pipeline — from keyword manage Summarization, citation generation (GB/T 7714, APA, MLA), review outlines, and gap analysis. **🔄 LangGraph Pipeline** - Pipeline orchestration with human-in-the-loop interrupt and resume. + Pipeline orchestration with HITL interrupt/resume and persistent checkpointing. + + **⚡ GPU Resource Management** + TTL-based auto-unload for GPU models, MinerU subprocess auto-management, monitoring API, and exit cleanup watchdog. **🔗 MCP Integration** Model Context Protocol server for AI IDE clients (Cursor, Claude Code, etc.). @@ -103,7 +106,7 @@ Keywords ─→ Search ─→ Dedup ─→ Crawler ─→ OCR ─→ RAG ─→ | **RAG** | LlamaIndex with GPU-aware embeddings | | **LLM** | LangChain (OpenAI, Anthropic, Aliyun, Volcengine, Ollama) | | **Orchestration** | LangGraph with HITL interrupt/resume | -| **OCR** | pdfplumber (native) + PaddleOCR (scanned, optional) | +| **OCR** | MinerU (auto-managed) + pdfplumber (native) + PaddleOCR (scanned) | | **MCP** | Model Context Protocol server | | **Docs** | VitePress (bilingual EN/ZH) | @@ -147,6 +150,10 @@ cp .env.example .env | `ALIYUN_API_KEY` | Aliyun Bailian API key | | `VOLCENGINE_API_KEY` | Volcengine Doubao API key | | `SEMANTIC_SCHOLAR_API_KEY` | Optional; increases Semantic Scholar rate limit | +| `GPU_MODE` | GPU preset: `conservative`, `balanced` (default), `aggressive` | +| `MODEL_TTL_SECONDS` | Auto-unload GPU models after N seconds idle (default: 300) | +| `MINERU_AUTO_MANAGE` | Auto start/stop MinerU subprocess (default: true) | +| `PDF_PARSER` | `auto`, `mineru`, or `pdfplumber` | See [`.env.example`](.env.example) for the full list. @@ -156,10 +163,31 @@ See [`.env.example`](.env.example) for the full list. ```bash cd backend + +# Run database migrations +alembic upgrade head + +# Start server uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 ``` -### 4. Start frontend +On startup, the backend automatically: +- Writes a PID file to `DATA_DIR/omelette.pid` +- Starts a GPU model TTL monitor (auto-unloads idle models) +- If `MINERU_AUTO_MANAGE=true`, manages MinerU subprocess lifecycle +- Registers cleanup handlers (`atexit` + `SIGHUP`) so GPU resources are released even if the process exits unexpectedly + +### 4. (Optional) GPU watchdog + +For extra safety against `kill -9` or crashes, run the external watchdog: + +```bash +python backend/scripts/gpu_watchdog.py --daemon +``` + +The watchdog monitors the Omelette process and cleans up GPU resources if it terminates abnormally. + +### 5. Start frontend ```bash cd frontend @@ -169,13 +197,19 @@ npm run dev Open [http://localhost:3000](http://localhost:3000) in your browser. -### 5. (Optional) OCR & Embeddings +### 6. (Optional) MinerU setup + +If using MinerU for PDF parsing (`PDF_PARSER=mineru`): ```bash -cd backend -pip install -e ".[ocr,ml]" +# Create a separate conda env for MinerU +conda create -n mineru python=3.10 +conda activate mineru +pip install magic-pdf[full] ``` +Set `MINERU_CONDA_ENV=mineru` in `.env`. Omelette will auto-start MinerU when needed. + > **Troubleshooting:** If you get `ModuleNotFoundError: No module named 'fastapi'`, ensure the conda environment is activated: `conda activate omelette`. ## 📂 Project Layout @@ -194,7 +228,8 @@ omelette/ │ │ └── main.py # App entry, lifespan, CORS │ ├── mcp_server.py # MCP (Model Context Protocol) server │ ├── alembic/ # Database migrations -│ ├── tests/ # pytest-asyncio tests (178 tests) +│ ├── scripts/ # Utilities (gpu_watchdog.py) +│ ├── tests/ # pytest-asyncio tests (526 tests) │ └── pyproject.toml # Python dependencies ├── frontend/ # React SPA │ └── src/ @@ -230,7 +265,7 @@ make dev # Start both backend and frontend ### Running Tests ```bash -# Backend (178 tests) +# Backend (526 tests) cd backend && pytest tests/ -v # Frontend unit tests (28 tests — Vitest + Testing Library + MSW) @@ -269,6 +304,8 @@ REST APIs under `/api/v1/`: | `GET/POST /subscriptions` | Subscription management | | `GET/POST /settings` | Settings and health | | `GET /settings/health` | Health check | +| `GET /gpu/status` | GPU model and memory status | +| `POST /gpu/unload` | Manually unload GPU models | MCP server: `/mcp` (WebSocket/SSE for AI IDE clients) diff --git a/README_zh.md b/README_zh.md index e9b9b11..2cfacd9 100644 --- a/README_zh.md +++ b/README_zh.md @@ -60,7 +60,7 @@ Omelette 覆盖科研文献全流程自动化 — 从关键词管理、多源检 Unpaywall、arXiv、直链多通道下载，智能回退策略。 **📝 OCR 解析** - pdfplumber 原生文本提取，PaddleOCR GPU 加速处理扫描件。 + MinerU（自动管理子进程）或 pdfplumber 原生提取，PaddleOCR GPU 加速处理扫描件。 **🧠 RAG 知识库** LlamaIndex 引擎，ChromaDB 向量存储，GPU 感知嵌入，混合检索，带引用回答。 @@ -69,7 +69,10 @@ Omelette 覆盖科研文献全流程自动化 — 从关键词管理、多源检 论文摘要、引用生成（GB/T 7714、APA、MLA）、综述提纲、缺口分析。 **🔄 LangGraph 流水线** - 流水线编排，支持人机协同中断与恢复。 + 流水线编排，支持人机协同中断/恢复与持久化检查点。 + + **⚡ GPU 资源管理** + TTL 自动卸载 GPU 模型、MinerU 子进程自动管理、监控 API、退出清理看门狗。 **🔗 MCP 集成** Model Context Protocol 服务端，面向 AI IDE 客户端（Cursor、Claude Code 等）。 @@ -103,7 +106,7 @@ Keywords ─→ Search ─→ Dedup ─→ Crawler ─→ OCR ─→ RAG ─→ | **RAG** | LlamaIndex，GPU 感知嵌入 | | **LLM** | LangChain（OpenAI、Anthropic、阿里云、火山引擎、Ollama） | | **编排** | LangGraph，支持人机协同中断与恢复 | -| **OCR** | pdfplumber（原生）+ PaddleOCR（扫描件，可选） | +| **OCR** | MinerU（自动管理）+ pdfplumber（原生）+ PaddleOCR（扫描件） | | **MCP** | Model Context Protocol 服务端 | | **文档** | VitePress（中英双语） | @@ -156,10 +159,31 @@ cp .env.example .env ```bash cd backend + +# 执行数据库迁移 +alembic upgrade head + +# 启动服务 uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 ``` -### 4. 启动前端 +启动时后端自动完成以下操作： +- 写入 PID 文件到 `DATA_DIR/omelette.pid` +- 启动 GPU 模型 TTL 监控（自动卸载空闲模型） +- 若 `MINERU_AUTO_MANAGE=true`，自动管理 MinerU 子进程生命周期 +- 注册清理钩子（`atexit` + `SIGHUP`），即使进程意外退出也会释放 GPU 资源 + +### 4.（可选）GPU 看门狗 + +为防止 `kill -9` 或崩溃导致资源泄漏，可运行外部看门狗： + +```bash +python backend/scripts/gpu_watchdog.py --daemon +``` + +看门狗会监控 Omelette 进程，在其异常终止后自动清理 GPU 资源。 + +### 5. 启动前端 ```bash cd frontend @@ -169,13 +193,19 @@ npm run dev 在浏览器中打开 [http://localhost:3000](http://localhost:3000)。 -### 5.（可选）OCR 与嵌入 +### 6.（可选）MinerU 配置 + +若使用 MinerU 解析 PDF（`PDF_PARSER=mineru`）： ```bash -cd backend -pip install -e ".[ocr,ml]" +# 为 MinerU 创建独立 conda 环境 +conda create -n mineru python=3.10 +conda activate mineru +pip install magic-pdf[full] ``` +在 `.env` 中设置 `MINERU_CONDA_ENV=mineru`，Omelette 将在需要时自动启动 MinerU。 + > **常见问题：** 若出现 `ModuleNotFoundError: No module named 'fastapi'`，请确认已激活 conda 环境：`conda activate omelette`。 ## 📂 项目结构 @@ -194,7 +224,8 @@ omelette/ │ │ └── main.py # App entry, lifespan, CORS │ ├── mcp_server.py # MCP (Model Context Protocol) server │ ├── alembic/ # Database migrations -│ ├── tests/ # pytest-asyncio 测试（178 个） +│ ├── scripts/ # 工具脚本（gpu_watchdog.py） +│ ├── tests/ # pytest-asyncio 测试（526 个） │ └── pyproject.toml # Python dependencies ├── frontend/ # React SPA │ └── src/ @@ -230,7 +261,7 @@ make dev # Start both backend and frontend ### 运行测试 ```bash -# 后端（178 个测试） +# 后端（526 个测试） cd backend && pytest tests/ -v # 前端单元测试（28 个测试 — Vitest + Testing Library + MSW） @@ -266,6 +297,8 @@ REST API 位于 `/api/v1/` 下： | `GET/POST /subscriptions` | 订阅管理 | | `GET/POST /settings` | 设置与健康状态 | | `GET /settings/health` | 健康检查 | +| `GET /gpu/status` | GPU 模型与显存状态 | +| `POST /gpu/unload` | 手动卸载 GPU 模型 | MCP 服务端：`/mcp`（WebSocket/SSE，面向 AI IDE 客户端） diff --git a/backend/alembic/versions/a1b2c3d4e5f6_add_composite_indexes.py b/backend/alembic/versions/a1b2c3d4e5f6_add_composite_indexes.py new file mode 100644 index 0000000..8f68f51 --- /dev/null +++ b/backend/alembic/versions/a1b2c3d4e5f6_add_composite_indexes.py @@ -0,0 +1,26 @@ +"""add composite indexes for paper and task tables + +Revision ID: a1b2c3d4e5f6 +Revises: f2bee250c39f +Create Date: 2026-03-18 10:00:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "a1b2c3d4e5f6" +down_revision: str | None = "f2bee250c39f" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_index("ix_paper_project_status", "papers", ["project_id", "status"]) + op.create_index("ix_task_project_status", "tasks", ["project_id", "status"]) + + +def downgrade() -> None: + op.drop_index("ix_task_project_status", table_name="tasks") + op.drop_index("ix_paper_project_status", table_name="papers") diff --git a/backend/alembic/versions/cb8130e58f92_add_paper_project_doi_unique_constraint.py b/backend/alembic/versions/cb8130e58f92_add_paper_project_doi_unique_constraint.py new file mode 100644 index 0000000..543e824 --- /dev/null +++ b/backend/alembic/versions/cb8130e58f92_add_paper_project_doi_unique_constraint.py @@ -0,0 +1,29 @@ +"""add paper project_doi unique constraint + +Revision ID: cb8130e58f92 +Revises: a1b2c3d4e5f6 +Create Date: 2026-03-18 22:54:13.519198 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "cb8130e58f92" +down_revision: str | Sequence[str] | None = "a1b2c3d4e5f6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + with op.batch_alter_table("papers", schema=None) as batch_op: + batch_op.create_unique_constraint("uq_paper_project_doi", ["project_id", "doi"]) + + +def downgrade() -> None: + """Downgrade schema.""" + with op.batch_alter_table("papers", schema=None) as batch_op: + batch_op.drop_constraint("uq_paper_project_doi", type_="unique") diff --git a/backend/alembic/versions/e7a9b1c3d5f7_add_keyword_parent_id_index.py b/backend/alembic/versions/e7a9b1c3d5f7_add_keyword_parent_id_index.py new file mode 100644 index 0000000..2e5a0fe --- /dev/null +++ b/backend/alembic/versions/e7a9b1c3d5f7_add_keyword_parent_id_index.py @@ -0,0 +1,24 @@ +"""add keyword parent_id index + +Revision ID: e7a9b1c3d5f7 +Revises: cb8130e58f92 +Create Date: 2026-03-18 12:00:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "e7a9b1c3d5f7" +down_revision: str | Sequence[str] | None = "cb8130e58f92" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_index(op.f("ix_keywords_parent_id"), "keywords", ["parent_id"], unique=False) + + +def downgrade() -> None: + op.drop_index(op.f("ix_keywords_parent_id"), table_name="keywords") diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 0672d7a..4abd3c8 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -1,13 +1,15 @@ """Shared FastAPI dependencies for dependency injection.""" +from __future__ import annotations + from collections.abc import AsyncGenerator from fastapi import Depends, HTTPException from sqlalchemy.ext.asyncio import AsyncSession -from app.database import get_session +from app.database import Base, get_session from app.models import Project -from app.services.llm_client import LLMClient, get_llm_client +from app.services.llm.client import LLMClient, get_llm_client async def get_db() -> AsyncGenerator[AsyncSession, None]: @@ -15,12 +17,27 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]: yield session +async def get_or_404[T: Base]( + db: AsyncSession, + model: type[T], + resource_id: int, + *, + project_id: int | None = None, + detail: str = "Resource not found", +) -> T: + """Fetch a model instance by primary key, raising 404 if missing or project mismatch.""" + obj = await db.get(model, resource_id) + if not obj: + raise HTTPException(status_code=404, detail=detail) + obj_project_id = getattr(obj, "project_id", None) + if project_id is not None and obj_project_id is not None and obj_project_id != project_id: + raise HTTPException(status_code=404, detail=detail) + return obj + + async def get_project_or_404(project_id: int, db: AsyncSession) -> Project: - """Fetch project by ID. Raises HTTPException 404 if not found. Use when project_id comes from body/query.""" - project = await db.get(Project, project_id) - if not project: - raise HTTPException(status_code=404, detail="Project not found") - return project + """Fetch project by ID. Raises HTTPException 404 if not found.""" + return await get_or_404(db, Project, project_id, detail="Project not found") async def get_project( diff --git a/backend/app/api/v1/__init__.py b/backend/app/api/v1/__init__.py index a3b3ea4..439e01c 100644 --- a/backend/app/api/v1/__init__.py +++ b/backend/app/api/v1/__init__.py @@ -7,6 +7,7 @@ conversations, crawler, dedup, + gpu, keywords, ocr, papers, @@ -41,3 +42,4 @@ api_router.include_router(chat.router) api_router.include_router(rewrite.router) api_router.include_router(pipelines.router) +api_router.include_router(gpu.router) diff --git a/backend/app/api/v1/chat.py b/backend/app/api/v1/chat.py index 70edf2c..e662035 100644 --- a/backend/app/api/v1/chat.py +++ b/backend/app/api/v1/chat.py @@ -7,21 +7,18 @@ import uuid from collections.abc import Callable -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Request from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db +from app.middleware.rate_limit import limiter from app.pipelines.chat.graph import create_chat_pipeline -from app.pipelines.chat.stream_writer import ( - format_done, - format_error, - format_finish, - format_start, -) +from app.pipelines.chat.stream_writer import format_done, format_finish, format_start from app.schemas.common import ApiResponse from app.schemas.conversation import ChatStreamRequest +from app.utils.sse import format_sse_error logger = logging.getLogger(__name__) @@ -83,6 +80,8 @@ async def _stream_chat( "tool_mode": request.tool_mode, "conversation_id": request.conversation_id, "model": request.model or "", + "rag_top_k": request.rag_top_k, + "use_reranker": request.use_reranker, } async for event in pipeline.astream( @@ -95,19 +94,21 @@ async def _stream_chat( yield format_finish() except Exception as e: logger.exception("Chat stream error") - yield format_error(str(e)) + yield format_sse_error(str(e), code=500) finally: yield format_done() -@router.post("/stream") +@router.post("/stream", summary="Stream chat completion") +@limiter.limit("30/minute") async def chat_stream( - request: ChatStreamRequest, + request: Request, + body: ChatStreamRequest, db: AsyncSession = Depends(get_db), ): """Data Stream Protocol (Vercel AI SDK 5.0) chat endpoint.""" return StreamingResponse( - _stream_chat(request, db), + _stream_chat(body, db), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", @@ -118,7 +119,7 @@ async def chat_stream( ) -@router.post("/complete", response_model=ApiResponse[CompletionResponse]) +@router.post("/complete", response_model=ApiResponse[CompletionResponse], summary="Autocomplete suggestion") async def complete(request: CompletionRequest): """Return a short text completion suggestion for autocomplete.""" from app.services.completion_service import CompletionService diff --git a/backend/app/api/v1/conversations.py b/backend/app/api/v1/conversations.py index ffbb48b..3c0a929 100644 --- a/backend/app/api/v1/conversations.py +++ b/backend/app/api/v1/conversations.py @@ -1,7 +1,7 @@ """Conversation CRUD API endpoints.""" from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import func, select +from sqlalchemy import func, select, text from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload @@ -19,7 +19,7 @@ router = APIRouter(prefix="/conversations", tags=["conversations"]) -@router.get("", response_model=ApiResponse[PaginatedData[ConversationListSchema]]) +@router.get("", response_model=ApiResponse[PaginatedData[ConversationListSchema]], summary="List conversations") async def list_conversations( page: int = 1, page_size: int = 20, @@ -27,15 +27,16 @@ async def list_conversations( db: AsyncSession = Depends(get_db), ): """List conversations, newest first.""" - stmt = select(Conversation).order_by(Conversation.updated_at.desc()) - + kb_filter = None if knowledge_base_id is not None: - stmt = stmt.where( - Conversation.knowledge_base_ids.isnot(None), - ) + kb_filter = text( + "EXISTS (SELECT 1 FROM json_each(conversations.knowledge_base_ids) WHERE value = :kb_id)" + ).bindparams(kb_id=knowledge_base_id) - count_stmt = select(func.count()).select_from(stmt.subquery()) - total = (await db.execute(count_stmt)).scalar_one() + count_base = select(func.count(Conversation.id)) + if kb_filter is not None: + count_base = count_base.where(kb_filter) + total = (await db.execute(count_base)).scalar_one() msg_count_sq = ( select(func.count(Message.id)) @@ -60,31 +61,25 @@ async def list_conversations( .offset((page - 1) * page_size) .limit(page_size) ) - if knowledge_base_id is not None: - detail_stmt = detail_stmt.where(Conversation.knowledge_base_ids.isnot(None)) + if kb_filter is not None: + detail_stmt = detail_stmt.where(kb_filter) detail_result = await db.execute(detail_stmt) - items = [] - for conv, msg_count, last_msg_content in detail_result.all(): - if knowledge_base_id is not None: - kb_ids = conv.knowledge_base_ids or [] - if knowledge_base_id not in kb_ids: - continue - - items.append( - ConversationListSchema( - id=conv.id, - title=conv.title, - knowledge_base_ids=conv.knowledge_base_ids, - model=conv.model, - tool_mode=conv.tool_mode, - created_at=conv.created_at, - updated_at=conv.updated_at, - message_count=msg_count or 0, - last_message_preview=(last_msg_content[:100] if last_msg_content else ""), - ) + items = [ + ConversationListSchema( + id=conv.id, + title=conv.title, + knowledge_base_ids=conv.knowledge_base_ids, + model=conv.model, + tool_mode=conv.tool_mode, + created_at=conv.created_at, + updated_at=conv.updated_at, + message_count=msg_count or 0, + last_message_preview=(last_msg_content[:100] if last_msg_content else ""), ) + for conv, msg_count, last_msg_content in detail_result.all() + ] total_pages = (total + page_size - 1) // page_size if total > 0 else 1 @@ -99,7 +94,7 @@ async def list_conversations( ) -@router.post("", response_model=ApiResponse[ConversationSchema]) +@router.post("", response_model=ApiResponse[ConversationSchema], summary="Create conversation") async def create_conversation( body: ConversationCreateSchema, db: AsyncSession = Depends(get_db), @@ -112,7 +107,7 @@ async def create_conversation( tool_mode=body.tool_mode, ) db.add(conv) - await db.commit() + await db.flush() result = await db.execute( select(Conversation).where(Conversation.id == conv.id).options(selectinload(Conversation.messages)) @@ -121,7 +116,7 @@ async def create_conversation( return ApiResponse(data=ConversationSchema.model_validate(conv)) -@router.get("/{conversation_id}", response_model=ApiResponse[ConversationSchema]) +@router.get("/{conversation_id}", response_model=ApiResponse[ConversationSchema], summary="Get conversation") async def get_conversation( conversation_id: int, db: AsyncSession = Depends(get_db), @@ -136,7 +131,7 @@ async def get_conversation( return ApiResponse(data=ConversationSchema.model_validate(conv)) -@router.put("/{conversation_id}", response_model=ApiResponse[ConversationSchema]) +@router.put("/{conversation_id}", response_model=ApiResponse[ConversationSchema], summary="Update conversation") async def update_conversation( conversation_id: int, body: ConversationUpdateSchema, @@ -151,8 +146,6 @@ async def update_conversation( for field, value in body.model_dump(exclude_none=True).items(): setattr(conv, field, value) - await db.commit() - result2 = await db.execute( select(Conversation).where(Conversation.id == conversation_id).options(selectinload(Conversation.messages)) ) @@ -160,7 +153,7 @@ async def update_conversation( return ApiResponse(data=ConversationSchema.model_validate(conv)) -@router.delete("/{conversation_id}", response_model=ApiResponse[dict]) +@router.delete("/{conversation_id}", response_model=ApiResponse[dict], summary="Delete conversation") async def delete_conversation( conversation_id: int, db: AsyncSession = Depends(get_db), @@ -172,5 +165,4 @@ async def delete_conversation( raise HTTPException(status_code=404, detail="Conversation not found") await db.delete(conv) - await db.commit() return ApiResponse(data={"deleted": True, "id": conversation_id}) diff --git a/backend/app/api/v1/crawler.py b/backend/app/api/v1/crawler.py index e251ad8..1667c3c 100644 --- a/backend/app/api/v1/crawler.py +++ b/backend/app/api/v1/crawler.py @@ -1,5 +1,7 @@ """PDF crawler API endpoints.""" +from typing import Literal + from fastapi import APIRouter, Depends from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession @@ -12,10 +14,10 @@ router = APIRouter(prefix="/projects/{project_id}/crawl", tags=["crawler"]) -@router.post("/start", response_model=ApiResponse[dict]) +@router.post("/start", response_model=ApiResponse[dict], summary="Start PDF download crawl") async def start_crawl( project_id: int, - priority: str = "high", + priority: Literal["high", "low"] = "low", max_papers: int = 50, db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), @@ -52,7 +54,7 @@ async def start_crawl( return ApiResponse(data=download_results) -@router.get("/stats", response_model=ApiResponse[dict]) +@router.get("/stats", response_model=ApiResponse[dict], summary="Get crawl statistics") async def crawl_stats( project_id: int, db: AsyncSession = Depends(get_db), diff --git a/backend/app/api/v1/dedup.py b/backend/app/api/v1/dedup.py index c32aa24..90dd204 100644 --- a/backend/app/api/v1/dedup.py +++ b/backend/app/api/v1/dedup.py @@ -2,17 +2,19 @@ import logging from pathlib import Path +from typing import Literal -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query, Request from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db, get_llm, get_project from app.config import settings +from app.middleware.rate_limit import limiter from app.models import Paper, PaperStatus, Project -from app.schemas.common import ApiResponse +from app.schemas.common import ApiResponse, PaginatedData, PaginationParams from app.schemas.knowledge_base import AutoResolveRequest, ResolveConflictRequest from app.services.dedup_service import DedupService -from app.services.llm_client import LLMClient +from app.services.llm.client import LLMClient from app.services.pdf_metadata import extract_metadata logger = logging.getLogger(__name__) @@ -20,10 +22,12 @@ router = APIRouter(prefix="/projects/{project_id}/dedup", tags=["dedup"]) -@router.post("/run", response_model=ApiResponse[dict]) +@router.post("/run", response_model=ApiResponse[dict], summary="Run deduplication pipeline") +@limiter.limit("5/minute") async def run_dedup( + request: Request, project_id: int, - strategy: str = "full", + strategy: Literal["full", "doi_only", "title_only"] = "full", db: AsyncSession = Depends(get_db), llm: LLMClient = Depends(get_llm), project: Project = Depends(get_project), @@ -41,19 +45,33 @@ async def run_dedup( return ApiResponse(data=result) -@router.get("/candidates", response_model=ApiResponse[list[dict]]) +@router.get("/candidates", response_model=ApiResponse[PaginatedData[dict]], summary="List dedup candidates") async def list_dedup_candidates( project_id: int, + pagination: PaginationParams = Depends(), db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): - """List potential duplicate pairs for manual review.""" + """List potential duplicate pairs for manual review with pagination.""" + page, page_size = pagination.page, pagination.page_size service = DedupService(db) candidates = await service.find_llm_dedup_candidates(project_id) - return ApiResponse(data=candidates) + total = len(candidates) + start = (page - 1) * page_size + end = start + page_size + items = candidates[start:end] + return ApiResponse( + data=PaginatedData( + items=items, + total=total, + page=page, + page_size=page_size, + total_pages=(total + page_size - 1) // page_size if total else 1, + ) + ) -@router.post("/verify", response_model=ApiResponse[dict]) +@router.post("/verify", response_model=ApiResponse[dict], summary="Verify duplicate with LLM") async def verify_duplicate( project_id: int, paper_a_id: int = Query(..., description="First paper ID"), @@ -68,7 +86,7 @@ async def verify_duplicate( return ApiResponse(data=result) -@router.post("/resolve", response_model=ApiResponse[dict]) +@router.post("/resolve", response_model=ApiResponse[dict], summary="Resolve upload conflict") async def resolve_conflict( project_id: int, body: ResolveConflictRequest, @@ -141,7 +159,7 @@ async def resolve_conflict( raise HTTPException(status_code=400, detail=f"Invalid action: {body.action}") -@router.post("/auto-resolve", response_model=ApiResponse[list[dict]]) +@router.post("/auto-resolve", response_model=ApiResponse[list[dict]], summary="Auto-resolve conflicts with LLM") async def auto_resolve_conflicts( project_id: int, body: AutoResolveRequest, @@ -185,56 +203,14 @@ async def auto_resolve_conflicts( new_metadata = await extract_metadata(pdf_path, fallback_title="Untitled") - if not llm: - resolutions.append( - { - "conflict_id": conflict_id, - "action": "keep_new", - "reason": "LLM not available, defaulting to keep_new", - } - ) - continue - - prompt = f"""Two papers may be duplicates. Decide the best resolution: - -Existing paper (in DB): -- ID: {old_paper.id} -- Title: {old_paper.title} -- DOI: {old_paper.doi or "N/A"} -- Year: {old_paper.year} -- Journal: {old_paper.journal} - -New upload: -- Title: {new_metadata.title} -- DOI: {new_metadata.doi or "N/A"} -- Year: {new_metadata.year} -- Journal: {new_metadata.journal} - -Return JSON: {{"action": "keep_old"|"keep_new"|"merge", "reason": "..."}} -- keep_old: existing is better, discard new -- keep_new: new is better or different work, add new -- merge: combine metadata, add as new paper""" - - try: - result = await llm.chat_json( - messages=[ - {"role": "system", "content": "You are a deduplication expert. Return valid JSON only."}, - {"role": "user", "content": prompt}, - ], - task_type="dedup_resolve", - ) - action = result.get("action", "keep_new") - if action not in ("keep_old", "keep_new", "merge"): - action = "keep_new" - resolutions.append( - { - "conflict_id": conflict_id, - "action": action, - "reason": result.get("reason", ""), - } - ) - except Exception as e: - logger.warning("LLM auto-resolve failed for %s: %s", conflict_id, e) - resolutions.append({"conflict_id": conflict_id, "action": "keep_new", "reason": f"Error: {e}"}) + dedup_svc = DedupService(db, llm) + resolution = await dedup_svc.resolve_conflict( + old_paper=old_paper, + new_title=new_metadata.title, + new_doi=new_metadata.doi, + new_year=new_metadata.year, + new_journal=new_metadata.journal, + ) + resolutions.append({"conflict_id": conflict_id, **resolution}) return ApiResponse(data=resolutions) diff --git a/backend/app/api/v1/gpu.py b/backend/app/api/v1/gpu.py new file mode 100644 index 0000000..d2fff21 --- /dev/null +++ b/backend/app/api/v1/gpu.py @@ -0,0 +1,70 @@ +"""GPU resource monitoring and management API.""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter + +from app.schemas.common import ApiResponse + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/gpu", tags=["gpu"]) + + +def _get_gpu_memory() -> list[dict]: + """Query GPU memory info via torch.cuda (returns empty list if unavailable).""" + try: + import torch + + if not torch.cuda.is_available(): + return [] + + import os + + cuda_ids = os.environ.get("CUDA_VISIBLE_DEVICES", "") + physical_ids = [int(x.strip()) for x in cuda_ids.split(",") if x.strip()] if cuda_ids else [] + + result = [] + for idx in range(torch.cuda.device_count()): + free, total = torch.cuda.mem_get_info(idx) + used = total - free + gpu_id = physical_ids[idx] if idx < len(physical_ids) else idx + result.append( + { + "gpu_id": gpu_id, + "total_mb": round(total / (1024 * 1024)), + "used_mb": round(used / (1024 * 1024)), + "free_mb": round(free / (1024 * 1024)), + } + ) + return result + except (ImportError, RuntimeError): + return [] + + +@router.get("/status", summary="Get GPU status") +async def gpu_status(): + """Return loaded GPU models, MinerU status, and GPU memory usage.""" + from app.services.gpu_model_manager import gpu_model_manager + from app.services.mineru_process_manager import mineru_process_manager + + return ApiResponse( + data={ + "models": gpu_model_manager.get_status(), + "mineru": mineru_process_manager.get_status(), + "gpu_memory": _get_gpu_memory(), + } + ) + + +@router.post("/unload", summary="Unload GPU models") +async def gpu_unload(): + """Immediately unload all GPU models and release VRAM.""" + from app.services.gpu_model_manager import gpu_model_manager + + names = list(gpu_model_manager.loaded_model_names) + gpu_model_manager.unload_all() + logger.info("Manual unload: released models %s", names) + return ApiResponse(data={"unloaded": names}) diff --git a/backend/app/api/v1/keywords.py b/backend/app/api/v1/keywords.py index 45710e9..ae7ffa3 100644 --- a/backend/app/api/v1/keywords.py +++ b/backend/app/api/v1/keywords.py @@ -1,36 +1,51 @@ """Keyword management API endpoints.""" -from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import select +from fastapi import APIRouter, Depends +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.api.deps import get_db, get_llm, get_project +from app.api.deps import get_db, get_llm, get_or_404, get_project from app.models import Keyword, Project -from app.schemas.common import ApiResponse +from app.schemas.common import ApiResponse, KeywordPaginationParams, PaginatedData from app.schemas.keyword import KeywordCreate, KeywordExpandRequest, KeywordExpandResponse, KeywordRead, KeywordUpdate from app.services.keyword_service import KeywordService -from app.services.llm_client import LLMClient +from app.services.llm.client import LLMClient router = APIRouter(prefix="/projects/{project_id}/keywords", tags=["keywords"]) -@router.get("", response_model=ApiResponse[list[KeywordRead]]) +@router.get("", response_model=ApiResponse[PaginatedData[KeywordRead]], summary="List keywords") async def list_keywords( project_id: int, + pagination: KeywordPaginationParams = Depends(), level: int | None = None, db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): - stmt = select(Keyword).where(Keyword.project_id == project_id) + page, page_size = pagination.page, pagination.page_size + base = select(Keyword).where(Keyword.project_id == project_id) if level is not None: - stmt = stmt.where(Keyword.level == level) - stmt = stmt.order_by(Keyword.level, Keyword.id) - result = await db.execute(stmt) - keywords = result.scalars().all() - return ApiResponse(data=[KeywordRead.model_validate(k) for k in keywords]) + base = base.where(Keyword.level == level) + count = (await db.execute(select(func.count()).select_from(base.subquery()))).scalar_one() + items = ( + (await db.execute(base.order_by(Keyword.level, Keyword.id).offset((page - 1) * page_size).limit(page_size))) + .scalars() + .all() + ) + + return ApiResponse( + data=PaginatedData( + items=[KeywordRead.model_validate(k) for k in items], + total=count, + page=page, + page_size=page_size, + total_pages=(count + page_size - 1) // page_size or 1, + ) + ) -@router.post("", response_model=ApiResponse[KeywordRead], status_code=201) + +@router.post("", response_model=ApiResponse[KeywordRead], status_code=201, summary="Create keyword") async def create_keyword( project_id: int, body: KeywordCreate, @@ -44,7 +59,7 @@ async def create_keyword( return ApiResponse(code=201, message="Keyword created", data=KeywordRead.model_validate(keyword)) -@router.post("/bulk", response_model=ApiResponse[dict]) +@router.post("/bulk", response_model=ApiResponse[dict], summary="Bulk create keywords") async def bulk_create_keywords( project_id: int, keywords: list[KeywordCreate], @@ -60,7 +75,7 @@ async def bulk_create_keywords( return ApiResponse(data={"created": created}) -@router.get("/search-formula", response_model=ApiResponse[dict]) +@router.get("/search-formula", response_model=ApiResponse[dict], summary="Generate boolean search formula") async def generate_search_formula( project_id: int, database: str = "wos", @@ -74,7 +89,7 @@ async def generate_search_formula( return ApiResponse(data=result) -@router.put("/{keyword_id}", response_model=ApiResponse[KeywordRead]) +@router.put("/{keyword_id}", response_model=ApiResponse[KeywordRead], summary="Update keyword") async def update_keyword( project_id: int, keyword_id: int, @@ -82,9 +97,7 @@ async def update_keyword( db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): - keyword = await db.get(Keyword, keyword_id) - if not keyword or keyword.project_id != project_id: - raise HTTPException(status_code=404, detail="Keyword not found") + keyword = await get_or_404(db, Keyword, keyword_id, project_id=project_id, detail="Keyword not found") for key, value in body.model_dump(exclude_unset=True).items(): setattr(keyword, key, value) await db.flush() @@ -92,21 +105,19 @@ async def update_keyword( return ApiResponse(data=KeywordRead.model_validate(keyword)) -@router.delete("/{keyword_id}", response_model=ApiResponse) +@router.delete("/{keyword_id}", response_model=ApiResponse, summary="Delete keyword") async def delete_keyword( project_id: int, keyword_id: int, db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): - keyword = await db.get(Keyword, keyword_id) - if not keyword or keyword.project_id != project_id: - raise HTTPException(status_code=404, detail="Keyword not found") + keyword = await get_or_404(db, Keyword, keyword_id, project_id=project_id, detail="Keyword not found") await db.delete(keyword) return ApiResponse(message="Keyword deleted") -@router.post("/expand", response_model=ApiResponse[KeywordExpandResponse]) +@router.post("/expand", response_model=ApiResponse[KeywordExpandResponse], summary="Expand keywords with LLM") async def expand_keywords( project_id: int, body: KeywordExpandRequest, @@ -115,26 +126,17 @@ async def expand_keywords( project: Project = Depends(get_project), ): """Use LLM to expand seed keywords with synonyms and related terms.""" - - prompt = ( - f"Given these seed keywords in the field of scientific research: {body.seed_terms}\n" - f"Language: {body.language}\n" - f"Generate up to {body.max_results} related terms including synonyms, abbreviations, " - "alternate names, and cross-disciplinary terms.\n" - 'Return JSON: {"expanded_terms": [{"term": "...", "term_zh": "...", "relation": "synonym|abbreviation|related"}]}' - ) - - result = await llm.chat_json( - messages=[ - {"role": "system", "content": "You are a scientific terminology expert. Return valid JSON only."}, - {"role": "user", "content": prompt}, - ], - task_type="keyword_expand", + svc = KeywordService(db, llm) + expanded = await svc.expand_keywords_with_llm( + project_id=project_id, + seed_terms=body.seed_terms, + language=body.language, + max_results=body.max_results, ) return ApiResponse( data=KeywordExpandResponse( - expanded_terms=result.get("expanded_terms", []), - source=f"llm:{llm.provider}", + expanded_terms=expanded, + source=f"llm:{llm.provider}" if llm else "none", ) ) diff --git a/backend/app/api/v1/ocr.py b/backend/app/api/v1/ocr.py index b1722b2..209a6a3 100644 --- a/backend/app/api/v1/ocr.py +++ b/backend/app/api/v1/ocr.py @@ -1,12 +1,14 @@ """OCR processing API endpoints.""" +import asyncio import logging -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Request from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db, get_project +from app.middleware.rate_limit import limiter from app.models import Paper, PaperChunk, PaperStatus, Project from app.schemas.common import ApiResponse from app.services.ocr_service import OCRService @@ -16,8 +18,10 @@ router = APIRouter(prefix="/projects/{project_id}/ocr", tags=["ocr"]) -@router.post("/process", response_model=ApiResponse[dict]) +@router.post("/process", response_model=ApiResponse[dict], summary="Run OCR on PDFs") +@limiter.limit("5/minute") async def process_ocr( + request: Request, project_id: int, paper_ids: list[int] | None = None, force_ocr: bool = False, @@ -41,49 +45,47 @@ async def process_ocr( if not papers: return ApiResponse(data={"processed": 0, "failed": 0, "total": 0, "message": "No papers to process"}) - service = OCRService(use_gpu=use_gpu) processed = 0 failed = 0 - for paper in papers: - if not paper.pdf_path: - failed += 1 - continue - - try: - ocr_result = service.process_pdf(paper.pdf_path, force_ocr=force_ocr) - - if ocr_result.get("error"): + with OCRService(use_gpu=use_gpu) as service: + for paper in papers: + if not paper.pdf_path: failed += 1 continue - # Save OCR result - service.save_result(paper.id, ocr_result) - - # Create chunks and store in DB - chunks = service.chunk_text(ocr_result["pages"]) - for chunk_data in chunks: - chunk = PaperChunk( - paper_id=paper.id, - chunk_type=chunk_data["chunk_type"], - content=chunk_data["content"], - page_number=chunk_data.get("page_number"), - chunk_index=chunk_data["chunk_index"], - token_count=chunk_data.get("token_count", 0), - ) - db.add(chunk) - - paper.status = PaperStatus.OCR_COMPLETE - processed += 1 - except Exception as e: - logger.error("OCR failed for paper %s: %s", paper.id, e) - failed += 1 + try: + ocr_result = await asyncio.to_thread(service.process_pdf, paper.pdf_path, force_ocr=force_ocr) + + if ocr_result.get("error"): + failed += 1 + continue + + service.save_result(paper.id, ocr_result) + + chunks = service.chunk_text(ocr_result["pages"]) + for chunk_data in chunks: + chunk = PaperChunk( + paper_id=paper.id, + chunk_type=chunk_data["chunk_type"], + content=chunk_data["content"], + page_number=chunk_data.get("page_number"), + chunk_index=chunk_data["chunk_index"], + token_count=chunk_data.get("token_count", 0), + ) + db.add(chunk) + + paper.status = PaperStatus.OCR_COMPLETE + processed += 1 + except Exception as e: + logger.error("OCR failed for paper %s: %s", paper.id, e) + failed += 1 await db.flush() return ApiResponse(data={"processed": processed, "failed": failed, "total": len(papers)}) -@router.get("/stats", response_model=ApiResponse[dict]) +@router.get("/stats", response_model=ApiResponse[dict], summary="Get OCR statistics") async def ocr_stats( project_id: int, db: AsyncSession = Depends(get_db), diff --git a/backend/app/api/v1/papers.py b/backend/app/api/v1/papers.py index 777fbcb..a632b9d 100644 --- a/backend/app/api/v1/papers.py +++ b/backend/app/api/v1/papers.py @@ -7,20 +7,21 @@ from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.api.deps import get_db, get_project +from app.api.deps import get_db, get_or_404, get_project from app.config import settings from app.models import Paper, Project -from app.schemas.common import ApiResponse, PaginatedData -from app.schemas.paper import PaperBulkImport, PaperCreate, PaperRead, PaperUpdate +from app.models.chunk import PaperChunk +from app.schemas.chunk import ChunkRead +from app.schemas.common import ApiResponse, PaginatedData, PaginationParams +from app.schemas.paper import PaperBatchDeleteRequest, PaperBulkImport, PaperCreate, PaperRead, PaperUpdate router = APIRouter(tags=["papers"]) -@router.get("", response_model=ApiResponse[PaginatedData[PaperRead]]) +@router.get("", response_model=ApiResponse[PaginatedData[PaperRead]], summary="List papers with filters") async def list_papers( project_id: int, - page: int = 1, - page_size: int = 20, + pagination: PaginationParams = Depends(), status: str | None = None, year: int | None = None, q: str | None = Query(default=None, description="Search title/abstract"), @@ -29,6 +30,7 @@ async def list_papers( db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): + page, page_size = pagination.page, pagination.page_size base = select(Paper).where(Paper.project_id == project_id) count_base = select(func.count(Paper.id)).where(Paper.project_id == project_id) @@ -64,7 +66,7 @@ async def list_papers( ) -@router.post("", response_model=ApiResponse[PaperRead], status_code=201) +@router.post("", response_model=ApiResponse[PaperRead], status_code=201, summary="Create paper") async def create_paper( project_id: int, body: PaperCreate, @@ -78,7 +80,7 @@ async def create_paper( return ApiResponse(code=201, message="Paper created", data=PaperRead.model_validate(paper)) -@router.post("/bulk", response_model=ApiResponse[dict]) +@router.post("/bulk", response_model=ApiResponse[dict], summary="Bulk import papers") async def bulk_import_papers( project_id: int, body: PaperBulkImport, @@ -102,20 +104,38 @@ async def bulk_import_papers( return ApiResponse(data={"created": created, "skipped": skipped, "total": len(body.papers)}) -@router.get("/{paper_id}", response_model=ApiResponse[PaperRead]) +@router.post("/batch-delete", response_model=ApiResponse[dict], summary="Batch delete papers") +async def batch_delete_papers( + project_id: int, + body: PaperBatchDeleteRequest, + db: AsyncSession = Depends(get_db), + project: Project = Depends(get_project), +): + """Delete multiple papers at once.""" + stmt = select(Paper).where( + Paper.project_id == project_id, + Paper.id.in_(body.paper_ids), + ) + result = await db.execute(stmt) + papers = list(result.scalars().all()) + for paper in papers: + await db.delete(paper) + await db.flush() + return ApiResponse(data={"deleted": len(papers), "requested": len(body.paper_ids)}) + + +@router.get("/{paper_id}", response_model=ApiResponse[PaperRead], summary="Get paper by ID") async def get_paper( project_id: int, paper_id: int, db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): - paper = await db.get(Paper, paper_id) - if not paper or paper.project_id != project_id: - raise HTTPException(status_code=404, detail="Paper not found") + paper = await get_or_404(db, Paper, paper_id, project_id=project_id, detail="Paper not found") return ApiResponse(data=PaperRead.model_validate(paper)) -@router.put("/{paper_id}", response_model=ApiResponse[PaperRead]) +@router.put("/{paper_id}", response_model=ApiResponse[PaperRead], summary="Update paper") async def update_paper( project_id: int, paper_id: int, @@ -123,9 +143,7 @@ async def update_paper( db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): - paper = await db.get(Paper, paper_id) - if not paper or paper.project_id != project_id: - raise HTTPException(status_code=404, detail="Paper not found") + paper = await get_or_404(db, Paper, paper_id, project_id=project_id, detail="Paper not found") for key, value in body.model_dump(exclude_unset=True).items(): setattr(paper, key, value) await db.flush() @@ -133,21 +151,19 @@ async def update_paper( return ApiResponse(data=PaperRead.model_validate(paper)) -@router.delete("/{paper_id}", response_model=ApiResponse) +@router.delete("/{paper_id}", response_model=ApiResponse, summary="Delete paper") async def delete_paper( project_id: int, paper_id: int, db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): - paper = await db.get(Paper, paper_id) - if not paper or paper.project_id != project_id: - raise HTTPException(status_code=404, detail="Paper not found") + paper = await get_or_404(db, Paper, paper_id, project_id=project_id, detail="Paper not found") await db.delete(paper) return ApiResponse(message="Paper deleted") -@router.get("/{paper_id}/pdf") +@router.get("/{paper_id}/pdf", summary="Serve PDF file") async def serve_pdf( project_id: int, paper_id: int, @@ -155,9 +171,7 @@ async def serve_pdf( project: Project = Depends(get_project), ): """Serve the PDF file for a paper.""" - paper = await db.get(Paper, paper_id) - if not paper or paper.project_id != project_id: - raise HTTPException(status_code=404, detail="Paper not found") + paper = await get_or_404(db, Paper, paper_id, project_id=project_id, detail="Paper not found") if not paper.pdf_path or not Path(paper.pdf_path).exists(): raise HTTPException(status_code=404, detail="PDF file not available") @@ -174,7 +188,42 @@ async def serve_pdf( return FileResponse(str(pdf_path), media_type="application/pdf", filename=f"{paper.title[:80]}.pdf") -@router.get("/{paper_id}/citation-graph", response_model=ApiResponse) +@router.get("/{paper_id}/chunks", response_model=ApiResponse[PaginatedData[ChunkRead]], summary="List paper chunks") +async def list_paper_chunks( + project_id: int, + paper_id: int, + page: int = 1, + page_size: int = Query(default=50, ge=1, le=200), + chunk_type: str | None = Query(default=None, description="Filter by chunk type"), + db: AsyncSession = Depends(get_db), + project: Project = Depends(get_project), +): + """List chunks for a specific paper.""" + await get_or_404(db, Paper, paper_id, project_id=project_id, detail="Paper not found") + + base = select(PaperChunk).where(PaperChunk.paper_id == paper_id) + count_base = select(func.count(PaperChunk.id)).where(PaperChunk.paper_id == paper_id) + + if chunk_type: + base = base.where(PaperChunk.chunk_type == chunk_type) + count_base = count_base.where(PaperChunk.chunk_type == chunk_type) + + total = (await db.execute(count_base)).scalar() or 0 + base = base.order_by(PaperChunk.chunk_index).offset((page - 1) * page_size).limit(page_size) + chunks = (await db.execute(base)).scalars().all() + + return ApiResponse( + data=PaginatedData( + items=[ChunkRead.model_validate(c) for c in chunks], + total=total, + page=page, + page_size=page_size, + total_pages=(total + page_size - 1) // page_size if total else 1, + ) + ) + + +@router.get("/{paper_id}/citation-graph", response_model=ApiResponse, summary="Get citation graph") async def get_citation_graph( project_id: int, paper_id: int, diff --git a/backend/app/api/v1/pipelines.py b/backend/app/api/v1/pipelines.py index 4a2301f..ad628bc 100644 --- a/backend/app/api/v1/pipelines.py +++ b/backend/app/api/v1/pipelines.py @@ -3,13 +3,19 @@ import asyncio import logging import uuid +from typing import Literal -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket, WebSocketDisconnect from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db, get_project_or_404 +from app.config import settings +from app.middleware.rate_limit import limiter +from app.models.task import Task, TaskStatus, TaskType +from app.pipelines.cancellation import clear_cancelled, mark_cancelled from app.schemas.common import ApiResponse +from app.websocket.manager import pipeline_manager logger = logging.getLogger(__name__) @@ -30,12 +36,43 @@ class UploadPipelineRequest(BaseModel): pdf_paths: list[str] +class ResolvedConflict(BaseModel): + conflict_id: str + action: Literal["keep_old", "keep_new", "merge", "skip"] + merged_paper: dict | None = None + new_paper: dict | None = None + + class ResumeRequest(BaseModel): - resolved_conflicts: list[dict] = [] + resolved_conflicts: list[ResolvedConflict] = [] + + +@router.get("", response_model=ApiResponse[list[dict]], summary="List pipelines") +async def list_pipelines( + status: str | None = None, +): + """List all pipelines (running, interrupted, completed, failed, cancelled).""" + data = [] + for thread_id, task in _running_tasks.items(): + if status and task["status"] != status: + continue + data.append( + { + "thread_id": thread_id, + "status": task["status"], + "task_id": task.get("task_id"), + } + ) + return ApiResponse(data=data) -@router.post("/search", response_model=ApiResponse[dict]) -async def start_search_pipeline(body: SearchPipelineRequest, db: AsyncSession = Depends(get_db)): +@router.post("/search", response_model=ApiResponse[dict], summary="Start search pipeline") +@limiter.limit("10/minute") +async def start_search_pipeline( + request: Request, + body: SearchPipelineRequest, + db: AsyncSession = Depends(get_db), +): """Start a keyword-search pipeline: search → dedup → crawl → OCR → index.""" await get_project_or_404(body.project_id, db) @@ -66,7 +103,24 @@ async def start_search_pipeline(body: SearchPipelineRequest, db: AsyncSession = config = {"configurable": {"thread_id": thread_id}} - _running_tasks[thread_id] = {"status": "running", "pipeline": pipeline, "config": config} + task_record = Task( + project_id=body.project_id, + task_type=TaskType.SEARCH, + status=TaskStatus.RUNNING, + progress=0, + total=100, + result={"thread_id": thread_id, "pipeline_type": "search"}, + ) + db.add(task_record) + await db.flush() + + _running_tasks[thread_id] = { + "status": "running", + "pipeline": pipeline, + "config": config, + "task_id": task_record.id, + "project_id": body.project_id, + } async def _run(): try: @@ -78,24 +132,49 @@ async def _run(): else: _running_tasks[thread_id]["status"] = "completed" _running_tasks[thread_id]["result"] = result + await pipeline_manager.broadcast_to_room( + thread_id, + { + "type": "status", + "status": _running_tasks[thread_id]["status"], + "stage": result.get("stage", ""), + "progress": result.get("progress", 0), + }, + ) + except asyncio.CancelledError: + _running_tasks[thread_id]["status"] = "cancelled" + await pipeline_manager.broadcast_to_room(thread_id, {"type": "status", "status": "cancelled"}) except Exception as e: logger.error("Pipeline %s failed: %s", thread_id, e) _running_tasks[thread_id]["status"] = "failed" _running_tasks[thread_id]["error"] = str(e) + await pipeline_manager.broadcast_to_room(thread_id, {"type": "error", "message": str(e)}) + finally: + s = _running_tasks.get(thread_id, {}).get("status") + if s in ("completed", "failed", "cancelled"): + clear_cancelled(thread_id) + _running_tasks.pop(thread_id, None) - asyncio.create_task(_run()) + task_ref = asyncio.create_task(_run()) + _running_tasks[thread_id]["asyncio_task"] = task_ref return ApiResponse( data={ "thread_id": thread_id, "status": "running", "project_id": body.project_id, + "task_id": task_record.id, } ) -@router.post("/upload", response_model=ApiResponse[dict]) -async def start_upload_pipeline(body: UploadPipelineRequest, db: AsyncSession = Depends(get_db)): +@router.post("/upload", response_model=ApiResponse[dict], summary="Start upload pipeline") +@limiter.limit("10/minute") +async def start_upload_pipeline( + request: Request, + body: UploadPipelineRequest, + db: AsyncSession = Depends(get_db), +): """Start a PDF-upload pipeline: extract → dedup → OCR → index.""" from pathlib import Path as _Path @@ -107,7 +186,7 @@ async def start_upload_pipeline(body: UploadPipelineRequest, db: AsyncSession = safe_paths: list[str] = [] for p in body.pdf_paths: resolved = _Path(p).resolve() - if not str(resolved).startswith(str(allowed_root)): + if not resolved.is_relative_to(allowed_root): raise HTTPException(status_code=400, detail=f"Path not within allowed directory: {p}") safe_paths.append(str(resolved)) @@ -133,7 +212,25 @@ async def start_upload_pipeline(body: UploadPipelineRequest, db: AsyncSession = } config = {"configurable": {"thread_id": thread_id}} - _running_tasks[thread_id] = {"status": "running", "pipeline": pipeline, "config": config} + + task_record = Task( + project_id=body.project_id, + task_type=TaskType.OCR, + status=TaskStatus.RUNNING, + progress=0, + total=100, + result={"thread_id": thread_id, "pipeline_type": "upload"}, + ) + db.add(task_record) + await db.flush() + + _running_tasks[thread_id] = { + "status": "running", + "pipeline": pipeline, + "config": config, + "task_id": task_record.id, + "project_id": body.project_id, + } async def _run(): try: @@ -145,23 +242,38 @@ async def _run(): else: _running_tasks[thread_id]["status"] = "completed" _running_tasks[thread_id]["result"] = result + await pipeline_manager.broadcast_to_room( + thread_id, + { + "type": "status", + "status": _running_tasks[thread_id]["status"], + "stage": result.get("stage", ""), + "progress": result.get("progress", 0), + }, + ) + except asyncio.CancelledError: + _running_tasks[thread_id]["status"] = "cancelled" + await pipeline_manager.broadcast_to_room(thread_id, {"type": "status", "status": "cancelled"}) except Exception as e: logger.error("Pipeline %s failed: %s", thread_id, e) _running_tasks[thread_id]["status"] = "failed" _running_tasks[thread_id]["error"] = str(e) + await pipeline_manager.broadcast_to_room(thread_id, {"type": "error", "message": str(e)}) - asyncio.create_task(_run()) + task_ref = asyncio.create_task(_run()) + _running_tasks[thread_id]["asyncio_task"] = task_ref return ApiResponse( data={ "thread_id": thread_id, "status": "running", "project_id": body.project_id, + "task_id": task_record.id, } ) -@router.get("/{thread_id}/status", response_model=ApiResponse[dict]) +@router.get("/{thread_id}/status", response_model=ApiResponse[dict], summary="Get pipeline status") async def get_pipeline_status(thread_id: str): """Get pipeline execution status.""" task = _running_tasks.get(thread_id) @@ -200,7 +312,7 @@ async def get_pipeline_status(thread_id: str): return ApiResponse(data=data) -@router.post("/{thread_id}/resume", response_model=ApiResponse[dict]) +@router.post("/{thread_id}/resume", response_model=ApiResponse[dict], summary="Resume pipeline") async def resume_pipeline(thread_id: str, body: ResumeRequest): """Resume an interrupted pipeline with resolved conflicts.""" from langgraph.types import Command @@ -208,6 +320,8 @@ async def resume_pipeline(thread_id: str, body: ResumeRequest): task = _running_tasks.get(thread_id) if not task: raise HTTPException(status_code=404, detail="Pipeline not found") + if task["status"] == "cancelled": + raise HTTPException(status_code=400, detail="Pipeline was cancelled, cannot resume") if task["status"] != "interrupted": raise HTTPException(status_code=400, detail=f"Pipeline is {task['status']}, not interrupted") @@ -215,10 +329,12 @@ async def resume_pipeline(thread_id: str, body: ResumeRequest): config = task["config"] task["status"] = "running" + raw_conflicts = [rc.model_dump() for rc in body.resolved_conflicts] + async def _resume(): try: result = await pipeline.ainvoke( - Command(resume=body.resolved_conflicts), + Command(resume=raw_conflicts), config=config, ) snapshot = pipeline.get_state(config) @@ -227,22 +343,71 @@ async def _resume(): else: task["status"] = "completed" task["result"] = result + except asyncio.CancelledError: + task["status"] = "cancelled" except Exception as e: logger.error("Pipeline resume %s failed: %s", thread_id, e) task["status"] = "failed" task["error"] = str(e) + finally: + s = task.get("status") + if s in ("completed", "failed", "cancelled"): + clear_cancelled(thread_id) + _running_tasks.pop(thread_id, None) - asyncio.create_task(_resume()) + task_ref = asyncio.create_task(_resume()) + task["asyncio_task"] = task_ref return ApiResponse(data={"thread_id": thread_id, "status": "running"}) -@router.post("/{thread_id}/cancel", response_model=ApiResponse[dict]) +@router.post("/{thread_id}/cancel", response_model=ApiResponse[dict], summary="Cancel pipeline") async def cancel_pipeline(thread_id: str): - """Cancel a running pipeline.""" + """Cancel a running or interrupted pipeline.""" task = _running_tasks.get(thread_id) if not task: raise HTTPException(status_code=404, detail="Pipeline not found") + if task["status"] in ("completed", "failed"): + raise HTTPException(status_code=400, detail=f"Pipeline already {task['status']}") + mark_cancelled(thread_id) task["status"] = "cancelled" + + asyncio_task = task.get("asyncio_task") + if asyncio_task and not asyncio_task.done(): + asyncio_task.cancel() + else: + # Interrupted pipeline: no running task, cleanup here to avoid leak + clear_cancelled(thread_id) + _running_tasks.pop(thread_id, None) + + await pipeline_manager.broadcast_to_room(thread_id, {"type": "status", "status": "cancelled"}) return ApiResponse(data={"thread_id": thread_id, "status": "cancelled"}) + + +@router.websocket("/{thread_id}/ws") +async def pipeline_status_websocket( + websocket: WebSocket, + thread_id: str, + api_key: str | None = Query(default=None), +): + """WebSocket endpoint for real-time pipeline status updates.""" + if settings.api_secret_key and api_key != settings.api_secret_key: + await websocket.close(code=4008) + return + + await pipeline_manager.connect(websocket, thread_id) + try: + task = _running_tasks.get(thread_id) + if task: + await websocket.send_json( + { + "type": "status", + "status": task["status"], + "thread_id": thread_id, + } + ) + while True: + await websocket.receive_text() + except WebSocketDisconnect: + pipeline_manager.disconnect(websocket, thread_id) diff --git a/backend/app/api/v1/projects.py b/backend/app/api/v1/projects.py index 4fa0446..262f48b 100644 --- a/backend/app/api/v1/projects.py +++ b/backend/app/api/v1/projects.py @@ -1,24 +1,43 @@ """Project CRUD API endpoints.""" -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends +from pydantic import BaseModel from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.api.deps import get_db -from app.models import Keyword, Paper, Project -from app.schemas.common import ApiResponse, PaginatedData -from app.schemas.project import ProjectCreate, ProjectRead, ProjectUpdate +from app.api.deps import get_db, get_or_404 +from app.models import Keyword, Paper, Project, Subscription +from app.schemas.common import ApiResponse, PaginatedData, PaginationParams +from app.schemas.project import ( + KeywordImportItem, + PaperImportItem, + ProjectCreate, + ProjectRead, + ProjectUpdate, + SubscriptionImportItem, +) from app.services.pipeline_service import PipelineService router = APIRouter(tags=["projects"]) -@router.get("", response_model=ApiResponse[PaginatedData[ProjectRead]]) +class ProjectImportRequest(BaseModel): + """Request body for project import.""" + + name: str + description: str = "" + domain: str = "" + papers: list[PaperImportItem] = [] + keywords: list[KeywordImportItem] = [] + subscriptions: list[SubscriptionImportItem] = [] + + +@router.get("", response_model=ApiResponse[PaginatedData[ProjectRead]], summary="List all projects") async def list_projects( - page: int = 1, - page_size: int = 20, + pagination: PaginationParams = Depends(), db: AsyncSession = Depends(get_db), ): + page, page_size = pagination.page, pagination.page_size total_stmt = select(func.count(Project.id)) total = (await db.execute(total_stmt)).scalar() or 0 @@ -71,7 +90,7 @@ async def list_projects( ) -@router.post("", response_model=ApiResponse[ProjectRead], status_code=201) +@router.post("", response_model=ApiResponse[ProjectRead], status_code=201, summary="Create a project") async def create_project(body: ProjectCreate, db: AsyncSession = Depends(get_db)): project = Project(**body.model_dump()) db.add(project) @@ -92,11 +111,9 @@ async def create_project(body: ProjectCreate, db: AsyncSession = Depends(get_db) ) -@router.get("/{project_id}", response_model=ApiResponse[ProjectRead]) +@router.get("/{project_id}", response_model=ApiResponse[ProjectRead], summary="Get project by ID") async def get_project(project_id: int, db: AsyncSession = Depends(get_db)): - project = await db.get(Project, project_id) - if not project: - raise HTTPException(status_code=404, detail="Project not found") + project = await get_or_404(db, Project, project_id, detail="Project not found") paper_count = (await db.execute(select(func.count(Paper.id)).where(Paper.project_id == project_id))).scalar() or 0 kw_count = (await db.execute(select(func.count(Keyword.id)).where(Keyword.project_id == project_id))).scalar() or 0 return ApiResponse( @@ -114,11 +131,9 @@ async def get_project(project_id: int, db: AsyncSession = Depends(get_db)): ) -@router.put("/{project_id}", response_model=ApiResponse[ProjectRead]) +@router.put("/{project_id}", response_model=ApiResponse[ProjectRead], summary="Update project") async def update_project(project_id: int, body: ProjectUpdate, db: AsyncSession = Depends(get_db)): - project = await db.get(Project, project_id) - if not project: - raise HTTPException(status_code=404, detail="Project not found") + project = await get_or_404(db, Project, project_id, detail="Project not found") for key, value in body.model_dump(exclude_unset=True).items(): setattr(project, key, value) await db.flush() @@ -140,35 +155,119 @@ async def update_project(project_id: int, body: ProjectUpdate, db: AsyncSession ) -@router.delete("/{project_id}", response_model=ApiResponse) +@router.delete("/{project_id}", response_model=ApiResponse, summary="Delete project") async def delete_project(project_id: int, db: AsyncSession = Depends(get_db)): - project = await db.get(Project, project_id) - if not project: - raise HTTPException(status_code=404, detail="Project not found") + project = await get_or_404(db, Project, project_id, detail="Project not found") await db.delete(project) return ApiResponse(message="Project deleted") -@router.post("/{project_id}/pipeline/run", response_model=ApiResponse[dict]) +@router.get("/{project_id}/export", response_model=ApiResponse[dict], summary="Export project as JSON") +async def export_project(project_id: int, db: AsyncSession = Depends(get_db)): + """Export project data as JSON (papers, keywords, subscriptions).""" + project = await get_or_404(db, Project, project_id, detail="Project not found") + + papers = (await db.execute(select(Paper).where(Paper.project_id == project_id))).scalars().all() + keywords = (await db.execute(select(Keyword).where(Keyword.project_id == project_id))).scalars().all() + subs = (await db.execute(select(Subscription).where(Subscription.project_id == project_id))).scalars().all() + + return ApiResponse( + data={ + "name": project.name, + "description": project.description, + "domain": project.domain, + "papers": [ + { + "title": p.title, + "abstract": p.abstract, + "doi": p.doi, + "authors": p.authors, + "year": p.year, + "journal": p.journal, + "source": p.source, + "pdf_url": p.pdf_url, + "status": p.status, + "citation_count": p.citation_count, + } + for p in papers + ], + "keywords": [ + {"term": k.term, "term_en": k.term_en, "level": k.level, "category": k.category, "synonyms": k.synonyms} + for k in keywords + ], + "subscriptions": [ + { + "name": s.name, + "query": s.query, + "sources": s.sources, + "frequency": s.frequency, + "max_results": s.max_results, + } + for s in subs + ], + } + ) + + +@router.post("/import", response_model=ApiResponse[ProjectRead], status_code=201, summary="Import project from JSON") +async def import_project(body: ProjectImportRequest, db: AsyncSession = Depends(get_db)): + """Import a previously exported project.""" + project = Project(name=body.name, description=body.description, domain=body.domain) + db.add(project) + await db.flush() + + paper_cols = {c.name for c in Paper.__table__.columns} - {"id", "project_id", "created_at", "updated_at"} + kw_cols = {c.name for c in Keyword.__table__.columns} - {"id", "project_id", "created_at"} + sub_cols = {c.name for c in Subscription.__table__.columns} - {"id", "project_id", "created_at", "updated_at"} + + for pd in body.papers: + db.add(Paper(project_id=project.id, **{k: v for k, v in pd.model_dump().items() if k in paper_cols})) + + for kd in body.keywords: + db.add(Keyword(project_id=project.id, **{k: v for k, v in kd.model_dump().items() if k in kw_cols})) + + for sd in body.subscriptions: + db.add(Subscription(project_id=project.id, **{k: v for k, v in sd.model_dump().items() if k in sub_cols})) + + await db.flush() + await db.refresh(project) + + paper_count = (await db.execute(select(func.count(Paper.id)).where(Paper.project_id == project.id))).scalar() or 0 + kw_count = (await db.execute(select(func.count(Keyword.id)).where(Keyword.project_id == project.id))).scalar() or 0 + + return ApiResponse( + code=201, + message="Project imported", + data=ProjectRead( + id=project.id, + name=project.name, + description=project.description, + domain=project.domain, + settings=project.settings, + created_at=project.created_at, + updated_at=project.updated_at, + paper_count=paper_count, + keyword_count=kw_count, + ), + ) + + +@router.post("/{project_id}/pipeline/run", response_model=ApiResponse[dict], summary="Run crawl-OCR-index pipeline") async def run_pipeline(project_id: int, db: AsyncSession = Depends(get_db)): """Trigger the crawl → OCR → index pipeline for all pending papers.""" - project = await db.get(Project, project_id) - if not project: - raise HTTPException(status_code=404, detail="Project not found") + await get_or_404(db, Project, project_id, detail="Project not found") svc = PipelineService(db) result = await svc.process_project_pending(project_id) return ApiResponse(data=result) -@router.post("/{project_id}/pipeline/paper/{paper_id}", response_model=ApiResponse[dict]) +@router.post( + "/{project_id}/pipeline/paper/{paper_id}", response_model=ApiResponse[dict], summary="Run pipeline for single paper" +) async def run_paper_pipeline(project_id: int, paper_id: int, db: AsyncSession = Depends(get_db)): """Trigger the pipeline for a single paper.""" - project = await db.get(Project, project_id) - if not project: - raise HTTPException(status_code=404, detail="Project not found") - paper = await db.get(Paper, paper_id) - if not paper or paper.project_id != project_id: - raise HTTPException(status_code=404, detail="Paper not found in this project") + await get_or_404(db, Project, project_id, detail="Project not found") + await get_or_404(db, Paper, paper_id, project_id=project_id, detail="Paper not found in this project") svc = PipelineService(db) result = await svc.process_paper(paper_id) return ApiResponse(data=result) diff --git a/backend/app/api/v1/rag.py b/backend/app/api/v1/rag.py index 607d914..d3fe099 100644 --- a/backend/app/api/v1/rag.py +++ b/backend/app/api/v1/rag.py @@ -4,27 +4,30 @@ import json import logging -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Request from fastapi.responses import StreamingResponse -from pydantic import BaseModel +from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload -from app.api.deps import get_db, get_llm -from app.models import Paper, PaperStatus +from app.api.deps import get_db, get_llm, get_project +from app.middleware.rate_limit import limiter +from app.models import Paper, PaperStatus, Project from app.schemas.common import ApiResponse -from app.services.llm_client import LLMClient +from app.services.llm.client import LLMClient from app.services.rag_service import RAGService +from app.utils.sse import format_sse_error logger = logging.getLogger(__name__) + router = APIRouter(prefix="/projects/{project_id}/rag", tags=["rag"]) class RAGQueryRequest(BaseModel): question: str - top_k: int = 10 + top_k: int = Field(default=10, ge=1, le=50) use_reranker: bool = True include_sources: bool = True @@ -39,11 +42,12 @@ def get_rag_service(llm: LLMClient = Depends(get_llm)) -> RAGService: return RAGService(llm=llm) -@router.post("/query", response_model=ApiResponse[RAGQueryResponse]) +@router.post("/query", response_model=ApiResponse[RAGQueryResponse], summary="RAG query over literature") async def rag_query( project_id: int, body: RAGQueryRequest, rag: RAGService = Depends(get_rag_service), + project: Project = Depends(get_project), ): """Answer a question using RAG over the project's indexed literature.""" result = await rag.query( @@ -56,8 +60,10 @@ async def rag_query( return ApiResponse(data=RAGQueryResponse(**result)) -@router.post("/index", response_model=ApiResponse[dict]) +@router.post("/index", response_model=ApiResponse[dict], summary="Build vector index") +@limiter.limit("5/minute") async def build_index( + request: Request, project_id: int, db: AsyncSession = Depends(get_db), rag: RAGService = Depends(get_rag_service), @@ -95,7 +101,14 @@ async def build_index( } ) - index_result = await rag.index_chunks(project_id=project_id, chunks=chunks_to_index) + try: + index_result = await rag.index_chunks(project_id=project_id, chunks=chunks_to_index) + except RuntimeError as exc: + if "CUDA out of memory" not in str(exc): + raise + logger.warning("CUDA OOM during indexing, reloading model on best GPU and retrying") + rag._reload_embed_model() + index_result = await rag.index_chunks(project_id=project_id, chunks=chunks_to_index) # Update paper status to INDEXED for paper in papers: @@ -110,11 +123,12 @@ async def build_index( ) -@router.post("/index/stream") +@router.post("/index/stream", summary="Build index with SSE progress") async def build_index_stream( project_id: int, db: AsyncSession = Depends(get_db), rag: RAGService = Depends(get_rag_service), + project: Project = Depends(get_project), ): """SSE streaming rebuild — sends progress events so the UI stays responsive.""" @@ -191,7 +205,7 @@ def on_progress(stage: str, percent: int) -> None: ) except Exception as exc: logger.exception("SSE index build failed") - yield _sse("error", {"message": str(exc)}) + yield format_sse_error(str(exc), code=500) return StreamingResponse( _generate(), @@ -204,15 +218,23 @@ def on_progress(stage: str, percent: int) -> None: ) -@router.get("/stats", response_model=ApiResponse[dict]) -async def index_stats(project_id: int, rag: RAGService = Depends(get_rag_service)): +@router.get("/stats", response_model=ApiResponse[dict], summary="Get index statistics") +async def index_stats( + project_id: int, + rag: RAGService = Depends(get_rag_service), + project: Project = Depends(get_project), +): """Return indexing statistics.""" stats = await rag.get_stats(project_id=project_id) return ApiResponse(data=stats) -@router.delete("/index", response_model=ApiResponse[dict]) -async def delete_index(project_id: int, rag: RAGService = Depends(get_rag_service)): +@router.delete("/index", response_model=ApiResponse[dict], summary="Delete vector index") +async def delete_index( + project_id: int, + rag: RAGService = Depends(get_rag_service), + project: Project = Depends(get_project), +): """Delete the vector index for the project.""" result = await rag.delete_index(project_id=project_id) return ApiResponse(data=result) diff --git a/backend/app/api/v1/rewrite.py b/backend/app/api/v1/rewrite.py index 5b40fd2..a22bfe0 100644 --- a/backend/app/api/v1/rewrite.py +++ b/backend/app/api/v1/rewrite.py @@ -7,41 +7,24 @@ import logging from typing import Literal +import httpx from fastapi import APIRouter, Depends from fastapi.responses import StreamingResponse from pydantic import BaseModel, field_validator, model_validator from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db +from app.config import settings +from app.prompts.rewrite import REWRITE_PROMPTS from app.services.llm.client import get_llm_client from app.services.user_settings_service import UserSettingsService +from app.utils.sse import format_sse_error logger = logging.getLogger(__name__) -router = APIRouter(prefix="/chat", tags=["rewrite"]) +router = APIRouter(prefix="/chat", tags=["chat"]) -_rewrite_semaphore = asyncio.Semaphore(3) - -REWRITE_PROMPTS: dict[str, str] = { - "simplify": ( - "Rewrite the following academic text in plain, accessible language. " - "Keep the core meaning and key concepts intact, but make it understandable " - "to a general audience. Output only the rewritten text, no explanations." - ), - "academic": ( - "Rewrite the following text in formal academic style. " - "Use precise terminology, passive voice where appropriate, and proper " - "academic conventions. Maintain the original meaning. Output only the rewritten text." - ), - "translate_en": ( - "Translate the following text into English. " - "Preserve academic terminology and the original meaning. " - "Output only the translation, no explanations." - ), - "translate_zh": ("将以下文本翻译为中文。保留学术术语和原意。仅输出翻译结果，不要添加解释。"), -} - -REWRITE_TIMEOUT = 30.0 +_rewrite_semaphore = asyncio.Semaphore(settings.rewrite_semaphore_limit) class RewriteRequest(BaseModel): @@ -85,12 +68,15 @@ async def _stream_rewrite(request: RewriteRequest, db: AsyncSession): full_text = "" try: - async with asyncio.timeout(REWRITE_TIMEOUT): + async with asyncio.timeout(settings.rewrite_timeout): async for token in llm.chat_stream(messages, temperature=0.3, task_type="rewrite"): full_text += token yield _sse("rewrite_delta", {"delta": token}) except TimeoutError: - yield _sse("error", {"code": "timeout", "message": "Rewrite timed out after 30s"}) + yield format_sse_error( + f"Rewrite timed out after {settings.rewrite_timeout}s", + code=408, + ) return yield _sse("rewrite_end", {"full_text": full_text}) @@ -98,12 +84,12 @@ async def _stream_rewrite(request: RewriteRequest, db: AsyncSession): except asyncio.CancelledError: logger.info("Rewrite stream cancelled by client") return - except Exception as e: + except (httpx.HTTPError, ValueError, RuntimeError) as e: logger.exception("Rewrite stream error") - yield _sse("error", {"code": "rewrite_error", "message": str(e)}) + yield format_sse_error(str(e), code=500) -@router.post("/rewrite") +@router.post("/rewrite", summary="Stream excerpt rewrite") async def rewrite_stream( request: RewriteRequest, db: AsyncSession = Depends(get_db), diff --git a/backend/app/api/v1/search.py b/backend/app/api/v1/search.py index e2c8f66..569a36d 100644 --- a/backend/app/api/v1/search.py +++ b/backend/app/api/v1/search.py @@ -1,10 +1,12 @@ """Literature search API endpoints — multi-source federated search.""" -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request +from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db, get_project +from app.middleware.rate_limit import limiter from app.models import Keyword, Paper, Project from app.schemas.common import ApiResponse from app.services.search_service import SearchService @@ -12,17 +14,29 @@ router = APIRouter(prefix="/projects/{project_id}/search", tags=["search"]) -@router.post("/execute", response_model=ApiResponse[dict]) +class SearchExecuteRequest(BaseModel): + """Request body for federated search execution.""" + + query: str = Field(default="", description="Search query; if empty, built from project keywords") + sources: list[str] | None = Field(default=None, description="Search sources to use") + max_results: int = Field(default=100, ge=1, le=500, description="Maximum results per source") + auto_import: bool = Field(default=False, description="Import results into project") + + +@router.post("/execute", response_model=ApiResponse[dict], summary="Execute federated search") +@limiter.limit("10/minute") async def execute_search( + request: Request, project_id: int, - query: str = "", - sources: list[str] | None = None, - max_results: int = 100, - auto_import: bool = False, + body: SearchExecuteRequest, db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): """Execute federated search. If auto_import=True, import results to project.""" + query = body.query + sources = body.sources + max_results = body.max_results + auto_import = body.auto_import # If no query, build from project keywords if not query: @@ -38,7 +52,7 @@ async def execute_search( ) service = SearchService() - results = await service.search(query, sources, max_results) + results = await service.search(query, sources=sources, max_results=max_results) # Optionally auto-import results if auto_import and results["papers"]: @@ -79,8 +93,8 @@ async def execute_search( return ApiResponse(data=results) -@router.get("/sources", response_model=ApiResponse[list[dict]]) -async def list_search_sources(): +@router.get("/sources", response_model=ApiResponse[list[dict]], summary="List search sources") +async def list_search_sources(project: Project = Depends(get_project)): """Return available search sources and their status.""" return ApiResponse( data=[ diff --git a/backend/app/api/v1/settings_api.py b/backend/app/api/v1/settings_api.py index 73cc9c1..8d4abce 100644 --- a/backend/app/api/v1/settings_api.py +++ b/backend/app/api/v1/settings_api.py @@ -1,6 +1,6 @@ """Application settings API — CRUD, model listing, and connection testing.""" -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db @@ -11,7 +11,7 @@ router = APIRouter(prefix="/settings", tags=["settings"]) -@router.get("", response_model=ApiResponse[SettingsSchema]) +@router.get("", response_model=ApiResponse[SettingsSchema], summary="Get settings") async def get_settings(db: AsyncSession = Depends(get_db)): """Return merged settings (DB overrides .env); API keys are masked.""" svc = UserSettingsService(db) @@ -19,7 +19,7 @@ async def get_settings(db: AsyncSession = Depends(get_db)): return ApiResponse(data=merged) -@router.put("", response_model=ApiResponse[SettingsSchema]) +@router.put("", response_model=ApiResponse[SettingsSchema], summary="Update settings") async def put_settings( payload: SettingsUpdateSchema, db: AsyncSession = Depends(get_db), @@ -31,13 +31,13 @@ async def put_settings( return ApiResponse(data=merged) -@router.get("/models", response_model=ApiResponse[list[ProviderModelInfo]]) +@router.get("/models", response_model=ApiResponse[list[ProviderModelInfo]], summary="List available models") async def list_models(): """Return available LLM providers and their model lists.""" return ApiResponse(data=get_available_models()) -@router.post("/test-connection", response_model=ApiResponse[dict]) +@router.post("/test-connection", response_model=ApiResponse[dict], summary="Test LLM connection") async def test_connection(db: AsyncSession = Depends(get_db)): """Test the current LLM configuration by sending a simple prompt.""" svc = UserSettingsService(db) @@ -53,14 +53,10 @@ async def test_connection(db: AsyncSession = Depends(get_db)): ) return ApiResponse(data={"success": True, "response": response[:200]}) except Exception as e: - return ApiResponse( - code=500, - message="Connection test failed", - data={"success": False, "error": str(e)}, - ) + raise HTTPException(status_code=502, detail=f"Connection test failed: {e}") from e -@router.get("/health", response_model=ApiResponse[dict]) +@router.get("/health", response_model=ApiResponse[dict], summary="Health check") async def health_check(): """Simple health check endpoint.""" return ApiResponse(data={"status": "healthy", "version": "0.1.0"}) diff --git a/backend/app/api/v1/subscription.py b/backend/app/api/v1/subscription.py index 3c8ee73..f161c13 100644 --- a/backend/app/api/v1/subscription.py +++ b/backend/app/api/v1/subscription.py @@ -3,12 +3,12 @@ from datetime import datetime, timedelta from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db, get_project from app.models import Project, Subscription -from app.schemas.common import ApiResponse +from app.schemas.common import ApiResponse, PaginatedData, PaginationParams from app.schemas.subscription import ( SubscriptionCreate, SubscriptionRead, @@ -20,26 +20,30 @@ router = APIRouter(prefix="/projects/{project_id}/subscriptions", tags=["subscriptions"]) -@router.get("/feeds", response_model=ApiResponse[list[dict]]) +@router.get("/feeds", response_model=ApiResponse[list[dict]], summary="List common RSS feeds") async def list_common_feeds(): """Return common academic RSS feed templates.""" return ApiResponse(data=SubscriptionService.get_common_feeds()) -@router.post("/check-rss", response_model=ApiResponse[dict]) +@router.post("/check-rss", response_model=ApiResponse[dict], summary="Check RSS feed for entries") async def check_rss( project_id: int, feed_url: str = Query(..., description="RSS/Atom feed URL"), since_days: int = Query(7, ge=1, le=365), + project: Project = Depends(get_project), ): """Check an RSS feed for new entries since the given number of days.""" service = SubscriptionService() since = datetime.now() - timedelta(days=since_days) - entries = await service.check_rss_feed(feed_url, since) + try: + entries = await service.check_rss_feed(feed_url, since) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e return ApiResponse(data={"entries": entries, "count": len(entries)}) -@router.post("/check-updates", response_model=ApiResponse[dict]) +@router.post("/check-updates", response_model=ApiResponse[dict], summary="Check API for new papers") async def check_updates( project_id: int, query: str = Query(""), @@ -47,6 +51,7 @@ async def check_updates( since_days: int = Query(7, ge=1, le=365), max_results: int = Query(50, ge=1, le=200), db: AsyncSession = Depends(get_db), + project: Project = Depends(get_project), ): """Check for new papers via API search (incremental update).""" service = SubscriptionService() @@ -54,19 +59,38 @@ async def check_updates( return ApiResponse(data=result) -@router.get("", response_model=ApiResponse[list[SubscriptionRead]]) +@router.get("", response_model=ApiResponse[PaginatedData[SubscriptionRead]], summary="List subscriptions") async def list_subscriptions( project_id: int, + pagination: PaginationParams = Depends(), db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): - """List all subscriptions for a project.""" - result = await db.execute(select(Subscription).where(Subscription.project_id == project_id)) + """List subscriptions for a project with pagination.""" + page, page_size = pagination.page, pagination.page_size + count_stmt = select(func.count(Subscription.id)).where(Subscription.project_id == project_id) + total = (await db.execute(count_stmt)).scalar() or 0 + stmt = ( + select(Subscription) + .where(Subscription.project_id == project_id) + .order_by(Subscription.created_at.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + ) + result = await db.execute(stmt) subs = result.scalars().all() - return ApiResponse(data=[SubscriptionRead.model_validate(s) for s in subs]) + return ApiResponse( + data=PaginatedData( + items=[SubscriptionRead.model_validate(s) for s in subs], + total=total, + page=page, + page_size=page_size, + total_pages=(total + page_size - 1) // page_size if total else 1, + ) + ) -@router.post("", response_model=ApiResponse[SubscriptionRead], status_code=201) +@router.post("", response_model=ApiResponse[SubscriptionRead], status_code=201, summary="Create subscription") async def create_subscription( project_id: int, body: SubscriptionCreate, @@ -81,7 +105,7 @@ async def create_subscription( return ApiResponse(code=201, message="Subscription created", data=SubscriptionRead.model_validate(sub)) -@router.get("/{sub_id}", response_model=ApiResponse[SubscriptionRead]) +@router.get("/{sub_id}", response_model=ApiResponse[SubscriptionRead], summary="Get subscription by ID") async def get_subscription( project_id: int, sub_id: int, @@ -97,7 +121,7 @@ async def get_subscription( return ApiResponse(data=SubscriptionRead.model_validate(sub)) -@router.put("/{sub_id}", response_model=ApiResponse[SubscriptionRead]) +@router.put("/{sub_id}", response_model=ApiResponse[SubscriptionRead], summary="Update subscription") async def update_subscription( project_id: int, sub_id: int, @@ -119,7 +143,7 @@ async def update_subscription( return ApiResponse(data=SubscriptionRead.model_validate(sub)) -@router.delete("/{sub_id}", response_model=ApiResponse[None]) +@router.delete("/{sub_id}", response_model=ApiResponse[None], summary="Delete subscription") async def delete_subscription( project_id: int, sub_id: int, @@ -136,15 +160,20 @@ async def delete_subscription( return ApiResponse(message="Subscription deleted", data=None) -@router.post("/{sub_id}/trigger", response_model=ApiResponse[SubscriptionRunResult]) +@router.post( + "/{sub_id}/trigger", response_model=ApiResponse[SubscriptionRunResult], summary="Trigger subscription update" +) async def trigger_subscription( project_id: int, sub_id: int, since_days: int = Query(7, ge=1, le=365), + auto_import: bool = Query(False, description="Auto-import new papers into project"), db: AsyncSession = Depends(get_db), project: Project = Depends(get_project), ): """Manually trigger a subscription update (check API for new papers).""" + from app.models import Paper + sub = ( await db.execute(select(Subscription).where(Subscription.id == sub_id, Subscription.project_id == project_id)) ).scalar_one_or_none() @@ -160,6 +189,23 @@ async def trigger_subscription( new_papers = result.get("new_papers", []) total_found = result.get("total_found", 0) sources_checked = result.get("sources_checked", {}) + + imported_count = 0 + if auto_import and new_papers: + for paper_data in new_papers: + paper = Paper( + project_id=project_id, + title=paper_data.get("title", "Untitled"), + abstract=paper_data.get("abstract", ""), + doi=paper_data.get("doi"), + authors=paper_data.get("authors"), + year=paper_data.get("year"), + source=paper_data.get("source", "subscription"), + pdf_url=paper_data.get("pdf_url", ""), + ) + db.add(paper) + imported_count += 1 + sub.last_run_at = datetime.now() sub.total_found = total_found await db.flush() @@ -169,5 +215,6 @@ async def trigger_subscription( new_papers=len(new_papers), total_checked=total_found, sources_searched=list(sources_checked.keys()) if sources_checked else [], + imported=imported_count, ) ) diff --git a/backend/app/api/v1/tasks.py b/backend/app/api/v1/tasks.py index 5e76fa9..9bded5a 100644 --- a/backend/app/api/v1/tasks.py +++ b/backend/app/api/v1/tasks.py @@ -1,21 +1,19 @@ """Task status and management API endpoints.""" from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.api.deps import get_db +from app.api.deps import get_db, get_or_404 from app.models import Task -from app.schemas.common import ApiResponse +from app.schemas.common import ApiResponse, PaginatedData router = APIRouter(prefix="/tasks", tags=["tasks"]) -@router.get("/{task_id}", response_model=ApiResponse[dict]) +@router.get("/{task_id}", response_model=ApiResponse[dict], summary="Get task by ID") async def get_task(task_id: int, db: AsyncSession = Depends(get_db)): - task = await db.get(Task, task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") + task = await get_or_404(db, Task, task_id, detail="Task not found") return ApiResponse( data={ "id": task.id, @@ -34,41 +32,49 @@ async def get_task(task_id: int, db: AsyncSession = Depends(get_db)): ) -@router.get("", response_model=ApiResponse[list[dict]]) +@router.get("", response_model=ApiResponse[PaginatedData[dict]], summary="List tasks") async def list_tasks( project_id: int | None = None, status: str | None = None, - limit: int = 50, + page: int = 1, + page_size: int = 50, db: AsyncSession = Depends(get_db), ): - stmt = select(Task).order_by(Task.created_at.desc()).limit(limit) + base = select(Task) if project_id: - stmt = stmt.where(Task.project_id == project_id) + base = base.where(Task.project_id == project_id) if status: - stmt = stmt.where(Task.status == status) - result = await db.execute(stmt) + base = base.where(Task.status == status) + + total = (await db.execute(select(func.count()).select_from(base.subquery()))).scalar_one() + result = await db.execute(base.order_by(Task.created_at.desc()).offset((page - 1) * page_size).limit(page_size)) tasks = result.scalars().all() + return ApiResponse( - data=[ - { - "id": t.id, - "project_id": t.project_id, - "task_type": t.task_type, - "status": t.status, - "progress": t.progress, - "total": t.total, - "created_at": t.created_at.isoformat() if t.created_at else None, - } - for t in tasks - ] + data=PaginatedData( + items=[ + { + "id": t.id, + "project_id": t.project_id, + "task_type": t.task_type, + "status": t.status, + "progress": t.progress, + "total": t.total, + "created_at": t.created_at.isoformat() if t.created_at else None, + } + for t in tasks + ], + total=total, + page=page, + page_size=page_size, + total_pages=(total + page_size - 1) // page_size or 1, + ) ) -@router.post("/{task_id}/cancel", response_model=ApiResponse) +@router.post("/{task_id}/cancel", response_model=ApiResponse, summary="Cancel task") async def cancel_task(task_id: int, db: AsyncSession = Depends(get_db)): - task = await db.get(Task, task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") + task = await get_or_404(db, Task, task_id, detail="Task not found") if task.status in ("completed", "failed", "cancelled"): raise HTTPException(status_code=400, detail=f"Cannot cancel task in {task.status} state") task.status = "cancelled" diff --git a/backend/app/api/v1/upload.py b/backend/app/api/v1/upload.py index 2ca0487..bdd902e 100644 --- a/backend/app/api/v1/upload.py +++ b/backend/app/api/v1/upload.py @@ -6,12 +6,13 @@ from difflib import SequenceMatcher from pathlib import Path -from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile +from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db, get_project from app.config import settings +from app.middleware.rate_limit import limiter from app.models import Paper, PaperStatus, Project from app.schemas.common import ApiResponse from app.schemas.knowledge_base import DedupConflictPair, NewPaperData, UploadResult @@ -24,12 +25,11 @@ router = APIRouter(tags=["papers"]) -MAX_FILE_SIZE_MB = 50 -TITLE_SIMILARITY_THRESHOLD = 0.85 - -@router.post("/upload", response_model=ApiResponse[UploadResult]) +@router.post("/upload", response_model=ApiResponse[UploadResult], summary="Upload PDF files") +@limiter.limit("5/minute") async def upload_pdfs( + request: Request, project_id: int, files: list[UploadFile] = File(...), db: AsyncSession = Depends(get_db), @@ -41,7 +41,7 @@ async def upload_pdfs( project_pdf_dir = pdf_dir / str(project_id) project_pdf_dir.mkdir(parents=True, exist_ok=True) - max_bytes = MAX_FILE_SIZE_MB * 1024 * 1024 + max_bytes = settings.max_upload_size_mb * 1024 * 1024 papers: list[NewPaperData] = [] conflicts: list[DedupConflictPair] = [] new_paper_objects: list[Paper] = [] @@ -66,7 +66,7 @@ async def upload_pdfs( if len(content) > max_bytes: raise HTTPException( status_code=413, - detail=f"File {upload_file.filename} exceeds {MAX_FILE_SIZE_MB}MB limit", + detail=f"File {upload_file.filename} exceeds {settings.max_upload_size_mb}MB limit", ) safe_filename = Path(upload_file.filename or "upload.pdf").name.replace("..", "") @@ -98,7 +98,7 @@ async def upload_pdfs( norm_new = DedupService.normalize_title(metadata.title) if norm_existing and norm_new: sim = SequenceMatcher(None, norm_existing, norm_new).ratio() - if sim >= TITLE_SIMILARITY_THRESHOLD: + if sim >= settings.title_similarity_threshold: conflict_id = f"{existing.id}:{saved_name}" conflicts.append( DedupConflictPair( @@ -156,7 +156,7 @@ async def upload_pdfs( ) -@router.post("/process", response_model=ApiResponse[dict]) +@router.post("/process", response_model=ApiResponse[dict], summary="Trigger OCR and RAG indexing") async def process_papers( project_id: int, paper_ids: list[int] | None = Query(default=None), diff --git a/backend/app/api/v1/writing.py b/backend/app/api/v1/writing.py index 92488c5..8097379 100644 --- a/backend/app/api/v1/writing.py +++ b/backend/app/api/v1/writing.py @@ -1,14 +1,15 @@ """Writing assistance API endpoints.""" -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_db, get_llm, get_project +from app.middleware.rate_limit import limiter from app.models import Project from app.schemas.common import ApiResponse -from app.services.llm_client import LLMClient +from app.services.llm.client import LLMClient from app.services.rag_service import RAGService from app.services.writing_service import WritingService @@ -57,8 +58,10 @@ def get_writing_service( return WritingService(db=db, llm=llm, rag=rag) -@router.post("/assist", response_model=ApiResponse[WritingAssistResponse]) +@router.post("/assist", response_model=ApiResponse[WritingAssistResponse], summary="AI writing assistance") +@limiter.limit("10/minute") async def writing_assist( + request: Request, project_id: int, body: WritingAssistRequest, db: AsyncSession = Depends(get_db), @@ -86,10 +89,9 @@ async def writing_assist( result = await svc.analyze_gaps(project_id=project_id, research_topic=topic) content = result["analysis"] else: - return ApiResponse( - code=400, - message=f"Unknown task: {body.task}. Use summarize, cite, review_outline, or gap_analysis.", - data=WritingAssistResponse(content="", citations=[], suggestions=[]), + raise HTTPException( + status_code=400, + detail=f"Unknown task: {body.task}. Use summarize, cite, review_outline, or gap_analysis.", ) return ApiResponse( @@ -101,7 +103,7 @@ async def writing_assist( ) -@router.post("/summarize", response_model=ApiResponse[dict]) +@router.post("/summarize", response_model=ApiResponse[dict], summary="Summarize papers") async def summarize_papers( project_id: int, body: SummarizeRequest, @@ -117,7 +119,7 @@ async def summarize_papers( return ApiResponse(data={"summaries": summaries}) -@router.post("/citations", response_model=ApiResponse[dict]) +@router.post("/citations", response_model=ApiResponse[dict], summary="Generate citations") async def generate_citations( project_id: int, body: CitationsRequest, @@ -133,7 +135,7 @@ async def generate_citations( return ApiResponse(data={"citations": citations, "style": body.style}) -@router.post("/review-outline", response_model=ApiResponse[dict]) +@router.post("/review-outline", response_model=ApiResponse[dict], summary="Generate review outline") async def generate_review_outline( project_id: int, body: ReviewOutlineRequest, @@ -150,7 +152,7 @@ async def generate_review_outline( return ApiResponse(data=result) -@router.post("/gap-analysis", response_model=ApiResponse[dict]) +@router.post("/gap-analysis", response_model=ApiResponse[dict], summary="Analyze research gaps") async def analyze_gaps( project_id: int, body: GapAnalysisRequest, @@ -173,8 +175,10 @@ class ReviewDraftRequest(BaseModel): language: str = Field(default="zh", pattern=r"^(zh|en)$") -@router.post("/review-draft/stream") +@router.post("/review-draft/stream", summary="Stream literature review draft") +@limiter.limit("10/minute") async def stream_review_draft( + request: Request, project_id: int, body: ReviewDraftRequest, svc: WritingService = Depends(get_writing_service), diff --git a/backend/app/config.py b/backend/app/config.py index a44b69c..aa8ca3d 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,15 +1,44 @@ """Application configuration using Pydantic Settings.""" import os +from enum import StrEnum from pathlib import Path from typing import Literal -from pydantic import Field +from pydantic import Field, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +class GpuMode(StrEnum): + CONSERVATIVE = "conservative" + BALANCED = "balanced" + AGGRESSIVE = "aggressive" + + +GPU_MODE_PRESETS: dict[GpuMode, dict[str, int]] = { + GpuMode.CONSERVATIVE: { + "ocr_parallel_limit": 1, + "embed_batch_size": 1, + "rerank_batch_size": 1, + "reranker_concurrency_limit": 1, + }, + GpuMode.BALANCED: { + "ocr_parallel_limit": 0, + "embed_batch_size": 8, + "rerank_batch_size": 16, + "reranker_concurrency_limit": 1, + }, + GpuMode.AGGRESSIVE: { + "ocr_parallel_limit": 0, + "embed_batch_size": 32, + "rerank_batch_size": 50, + "reranker_concurrency_limit": 2, + }, +} + + class Settings(BaseSettings): model_config = SettingsConfigDict( env_file=str(PROJECT_ROOT / ".env"), @@ -65,28 +94,80 @@ class Settings(BaseSettings): # Embedding embedding_provider: str = "local" # local | api | mock - embedding_model: str = "BAAI/bge-m3" + embedding_model: str = "Qwen/Qwen3-Embedding-0.6B" embedding_api_key: str = "" - reranker_model: str = "BAAI/bge-reranker-v2-m3" + reranker_model: str = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls" # OCR ocr_lang: str = "ch" # PaddleOCR language: ch (Chinese+English) | en (English only) # PDF Parsing / MinerU - pdf_parser: str = "auto" # auto | mineru | pdfplumber + pdf_parser: str = "mineru" # auto | mineru | pdfplumber mineru_api_url: str = "http://localhost:8010" mineru_backend: str = "pipeline" # pipeline | hybrid-auto-engine | vlm-auto-engine - mineru_timeout: int = 300 + mineru_timeout: int = 8000 + mineru_auto_manage: bool = Field(default=True, description="Auto start/stop MinerU subprocess") + mineru_conda_env: str = Field(default="mineru", description="Conda env name for MinerU") + mineru_ttl_seconds: int = Field(default=600, ge=0, description="Stop MinerU after N seconds idle. 0=disable") + mineru_startup_timeout: int = Field(default=120, ge=10, le=600, description="MinerU startup timeout") + mineru_gpu_ids: str = Field(default="", description="GPU IDs for MinerU. Empty=inherit cuda_visible_devices") + + # Semantic Scholar API + s2_api_base: str = "https://api.semanticscholar.org/graph/v1" + s2_timeout: int = Field(default=15, ge=1, le=60) + s2_max_per_request: int = Field(default=50, ge=1, le=100) + + # Upload + title_similarity_threshold: float = Field(default=0.85, ge=0.0, le=1.0) + + # Rewrite + rewrite_timeout: float = Field(default=30.0, ge=5.0, le=120.0) # Dedup thresholds dedup_title_hard_threshold: float = 0.90 dedup_title_llm_threshold: float = 0.80 + # App + app_version: str = "0.1.0" + + # Concurrency limits + max_upload_size_mb: int = Field(default=50, ge=1, le=500) + rate_limit: str = Field(default="120/minute", description="API rate limit") + clean_semaphore_limit: int = Field(default=3, ge=1) + rewrite_semaphore_limit: int = Field(default=3, ge=1) + llm_parallel_limit: int = Field(default=5, ge=1, description="Max parallel LLM calls for batch operations") + ocr_parallel_limit: int = Field( + default=0, + ge=0, + le=16, + description="Max parallel OCR tasks. 0=auto (GPU count or 1 for CPU)", + ) + + # RAG retrieval + rag_default_top_k: int = Field(default=10, ge=1, le=100, description="Default retrieval top-k") + rag_oversample_factor: int = Field(default=3, ge=1, le=10, description="Multiplier for oversampling before rerank") + rag_mmr_threshold: float = Field( + default=0.5, ge=0.0, le=1.0, description="MMR diversity threshold (0=max diversity, 1=max relevance)" + ) + reranker_concurrency_limit: int = Field(default=1, ge=1, le=4, description="Max concurrent reranker calls") + # LangGraph langgraph_checkpoint_dir: str = "" + pipeline_checkpoint_db: str = "" # SQLite checkpoint DB path; defaults to {data_dir}/pipeline_checkpoints.db + pid_file: str = "" # PID file path; defaults to {data_dir}/omelette.pid # GPU - cuda_visible_devices: str = "0,3" + cuda_visible_devices: str = "" # Empty = use all available GPUs + model_ttl_seconds: int = Field( + default=300, ge=0, description="Auto-unload GPU models after N seconds idle. 0=disable" + ) + model_ttl_check_interval: int = Field(default=30, ge=5, le=300, description="TTL check interval in seconds") + gpu_mode: GpuMode = Field(default=GpuMode.BALANCED, description="GPU preset: conservative/balanced/aggressive") + embed_batch_size: int = Field(default=0, ge=0, le=128, description="Embedding batch size. 0=follow GPU_MODE") + rerank_batch_size: int = Field(default=0, ge=0, le=128, description="Reranker internal top_n. 0=follow GPU_MODE") + embed_gpu_id: int = Field(default=-1, ge=-1, le=15, description="Pin embedding to GPU N. -1=auto select") + rerank_gpu_id: int = Field(default=-1, ge=-1, le=15, description="Pin reranker to GPU N. -1=auto select") + ocr_gpu_ids: str = Field(default="", description="Comma-separated GPU IDs for OCR. Empty=all") # Network Proxy http_proxy: str = "" @@ -103,6 +184,20 @@ class Settings(BaseSettings): frontend_url: str = "http://localhost:3000" cors_origins: str = "http://localhost:3000,http://0.0.0.0:3000" + @model_validator(mode="after") + def _apply_gpu_mode_defaults(self) -> "Settings": + """Fill zero-valued GPU params from the active GPU_MODE preset.""" + preset = GPU_MODE_PRESETS.get(self.gpu_mode, GPU_MODE_PRESETS[GpuMode.BALANCED]) + if self.embed_batch_size == 0: + self.embed_batch_size = preset["embed_batch_size"] + if self.rerank_batch_size == 0: + self.rerank_batch_size = preset["rerank_batch_size"] + if self.ocr_parallel_limit == 0: + self.ocr_parallel_limit = preset["ocr_parallel_limit"] + if self.reranker_concurrency_limit == 1 and preset["reranker_concurrency_limit"] != 1: + self.reranker_concurrency_limit = preset["reranker_concurrency_limit"] + return self + def __init__(self, **kwargs): super().__init__(**kwargs) if not self.pdf_dir: @@ -113,6 +208,10 @@ def __init__(self, **kwargs): self.chroma_db_dir = f"{self.data_dir}/chroma_db" if not self.langgraph_checkpoint_dir: self.langgraph_checkpoint_dir = f"{self.data_dir}/langgraph_checkpoints" + if not self.pipeline_checkpoint_db: + self.pipeline_checkpoint_db = f"{self.data_dir}/pipeline_checkpoints.db" + if not self.pid_file: + self.pid_file = f"{self.data_dir}/omelette.pid" @property def cors_origin_list(self) -> list[str]: diff --git a/backend/app/main.py b/backend/app/main.py index 682deea..56011ab 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,15 +1,22 @@ """Omelette — Scientific Literature Lifecycle Management System.""" +import atexit +import contextlib import logging +import os +import signal +import sys from contextlib import asynccontextmanager +from pathlib import Path -from fastapi import FastAPI, Request +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from app.api.v1 import api_router from app.config import settings -from app.database import init_db +from app.database import engine, init_db from app.middleware.auth import ApiKeyMiddleware from app.middleware.rate_limit import setup_rate_limiting from app.schemas.common import ApiResponse @@ -20,22 +27,109 @@ ) logger = logging.getLogger("omelette") +_cleanup_done = False + + +def _sync_cleanup() -> None: + """Synchronous cleanup: release GPU models and kill MinerU processes. + + Safe to call multiple times (idempotent via _cleanup_done flag). + Registered with atexit and called from signal handlers. + """ + global _cleanup_done # noqa: PLW0603 + if _cleanup_done: + return + _cleanup_done = True + + logger.info("Running sync cleanup (atexit / signal)") + try: + from app.services.gpu_model_manager import gpu_model_manager + + gpu_model_manager.unload_all() + except Exception: + logger.warning("GPU model cleanup failed", exc_info=True) + + try: + from app.services.mineru_process_manager import mineru_process_manager + + mineru_process_manager.stop_sync() + except Exception: + logger.warning("MinerU cleanup failed", exc_info=True) + + with contextlib.suppress(OSError): + Path(settings.pid_file).unlink(missing_ok=True) + + +def _handle_sighup(signum: int, frame: object) -> None: + """Handle terminal close (SIGHUP): cleanup and exit.""" + logger.info("Received SIGHUP — cleaning up and exiting") + _sync_cleanup() + sys.exit(0) + @asynccontextmanager async def lifespan(app: FastAPI): - logger.info("Starting Omelette v0.1.0 ...") + from app.pipelines.graphs import set_checkpointer + from app.services.gpu_model_manager import gpu_model_manager + from app.services.mineru_process_manager import mineru_process_manager + + logger.info("Starting Omelette v%s ...", settings.app_version) if settings.app_env == "production" and settings.app_secret_key == "change-me-to-a-random-secret-key": logger.warning("SECURITY: Using default secret key in production! Set APP_SECRET_KEY in .env") await init_db() logger.info("Database initialized") + + # Write PID file + pid_path = Path(settings.pid_file) + pid_path.parent.mkdir(parents=True, exist_ok=True) + pid_path.write_text(str(os.getpid())) + logger.info("PID file: %s (pid=%d)", pid_path, os.getpid()) + + # Register safety nets + atexit.register(_sync_cleanup) + with contextlib.suppress(OSError, ValueError): + signal.signal(signal.SIGHUP, _handle_sighup) + + # Pipeline checkpoint persistence (AsyncSqliteSaver) + checkpoint_cm = None + try: + from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver + + db_path = settings.pipeline_checkpoint_db + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + cm = AsyncSqliteSaver.from_conn_string(db_path) + saver = await cm.__aenter__() + checkpoint_cm = cm + set_checkpointer(saver) + logger.info("Pipeline checkpoint DB: %s", db_path) + except Exception as e: + logger.warning("AsyncSqliteSaver unavailable, using MemorySaver: %s", e) + set_checkpointer(None) + + await gpu_model_manager.start() + await mineru_process_manager.start() yield logger.info("Shutting down Omelette") + await mineru_process_manager.stop() + await gpu_model_manager.stop() + if checkpoint_cm is not None: + try: + await checkpoint_cm.__aexit__(None, None, None) + except Exception as e: + logger.warning("Checkpoint saver teardown: %s", e) + set_checkpointer(None) + await engine.dispose() + + with contextlib.suppress(OSError): + Path(settings.pid_file).unlink(missing_ok=True) + global _cleanup_done # noqa: PLW0603 + _cleanup_done = True app = FastAPI( title="Omelette API", description="Scientific Literature Lifecycle Management System / 科研文献全生命周期管理系统", - version="0.1.0", + version=settings.app_version, lifespan=lifespan, docs_url="/docs", redoc_url="/redoc", @@ -48,12 +142,38 @@ async def lifespan(app: FastAPI): allow_credentials=True, allow_methods=["*"], allow_headers=["*"], + expose_headers=["X-Request-ID", "X-Process-Time"], + max_age=600, ) setup_rate_limiting(app) app.include_router(api_router) +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + """Wrap HTTPException in ApiResponse format for consistent frontend handling.""" + return JSONResponse( + status_code=exc.status_code, + content={"code": exc.status_code, "message": exc.detail, "data": None}, + ) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + """Wrap Pydantic validation errors in ApiResponse format.""" + errors = [] + for err in exc.errors(): + clean = {k: v for k, v in err.items() if k != "ctx"} + if "ctx" in err: + clean["ctx"] = {k: str(v) for k, v in err["ctx"].items()} + errors.append(clean) + return JSONResponse( + status_code=422, + content={"code": 422, "message": "Validation error", "data": errors}, + ) + + @app.exception_handler(Exception) async def global_exception_handler(request: Request, exc: Exception): """Return sanitised error in production, full detail in debug mode.""" @@ -72,8 +192,14 @@ async def global_exception_handler(request: Request, exc: Exception): mcp_app = mcp_server.streamable_http_app() app.mount("/mcp", mcp_app) logger.info("MCP server mounted at /mcp") -except Exception as e: - logger.warning("MCP server mount failed: %s", e) +except Exception: + logger.error("MCP server mount failed", exc_info=True) + + +@app.get("/health") +async def health(): + """Health check endpoint — exempt from API key authentication.""" + return ApiResponse(data={"status": "ok"}) @app.get("/") @@ -81,7 +207,7 @@ async def root(): return ApiResponse( data={ "name": "Omelette", - "version": "0.1.0", + "version": settings.app_version, "description": "Scientific Literature Lifecycle Management System", "docs": "/docs", } diff --git a/backend/app/mcp_server.py b/backend/app/mcp_server.py index 792b1ec..8e233b7 100644 --- a/backend/app/mcp_server.py +++ b/backend/app/mcp_server.py @@ -72,8 +72,11 @@ async def search_knowledge_base(query: str, kb_id: int, top_k: int = 5) -> str: Args: query: The search question or keywords kb_id: Knowledge base ID (use list_knowledge_bases to find IDs) - top_k: Number of result chunks to return (default 5) + top_k: Number of result chunks to return (default 5, max 50) """ + if top_k < 1 or top_k > 50: + return "Error: top_k must be between 1 and 50." + from app.services.rag_service import RAGService rag = RAGService() @@ -195,6 +198,13 @@ async def add_paper_by_doi(doi: str, kb_id: int) -> str: doi: The paper's DOI kb_id: Target knowledge base ID """ + from app.services.url_validator import validate_doi + + try: + validate_doi(doi) + except ValueError as e: + return f"Error: {e}" + from sqlalchemy import select async with get_session() as db: @@ -289,6 +299,9 @@ async def get_paper_summary(paper_id: int, summary_type: str = "abstract") -> st if not paper: return f"Error: Paper {paper_id} not found." + if summary_type not in ("abstract", "llm"): + return f"Error: Unknown summary type '{summary_type}'. Use 'abstract' or 'llm'." + if summary_type == "abstract": return f"""## Paper Summary @@ -315,8 +328,11 @@ async def search_papers_by_keyword(query: str, sources: str = "", max_results: i Args: query: Search keywords sources: Comma-separated data sources (semantic_scholar,openalex,arxiv,crossref). Empty = all. - max_results: Maximum number of results (default 20) + max_results: Maximum number of results (default 20, max 100) """ + if max_results < 1 or max_results > 100: + return "Error: max_results must be between 1 and 100." + from app.services.search_service import SearchService source_list = [s.strip() for s in sources.split(",") if s.strip()] if sources else None @@ -350,6 +366,118 @@ async def search_papers_by_keyword(query: str, sources: str = "", max_results: i return "\n".join(lines) +@mcp.tool() +async def summarize_papers(kb_id: int, paper_ids: list[int] | None = None, language: str = "en") -> str: + """Summarize papers in a knowledge base. + + Args: + kb_id: Knowledge base ID + paper_ids: Optional list of specific paper IDs to summarize. If empty, summarizes all. + language: Output language (en/zh) + """ + from app.services.writing_service import WritingService + + svc = WritingService() + result = await svc.summarize(project_id=kb_id, paper_ids=paper_ids, language=language) + return f"## Summary\n\n{result.get('content', 'No summary generated.')}" + + +@mcp.tool() +async def generate_review_outline(kb_id: int, topic: str, language: str = "en") -> str: + """Generate a literature review outline based on papers in a knowledge base. + + Args: + kb_id: Knowledge base ID + topic: Research topic for the review + language: Output language (en/zh) + """ + from app.services.writing_service import WritingService + + svc = WritingService() + result = await svc.generate_review_outline(project_id=kb_id, topic=topic, language=language) + return f"## Review Outline\n\n{result.get('outline', 'No outline generated.')}" + + +@mcp.tool() +async def analyze_gaps(kb_id: int, research_topic: str) -> str: + """Analyze research gaps in the literature of a knowledge base. + + Args: + kb_id: Knowledge base ID + research_topic: The research topic to analyze gaps for + """ + from app.services.writing_service import WritingService + + svc = WritingService() + result = await svc.analyze_gaps(project_id=kb_id, research_topic=research_topic) + return f"## Gap Analysis\n\n{result.get('analysis', 'No gap analysis generated.')}" + + +@mcp.tool() +async def manage_keywords(kb_id: int, action: str = "list", term: str = "", language: str = "en") -> str: + """Manage keywords for a knowledge base — list, add, expand, or delete. + + Args: + kb_id: Knowledge base ID + action: One of: list, add, expand, delete + term: Keyword term (required for add/expand/delete) + language: Language for keyword expansion (en/zh) + """ + if action not in ("list", "add", "expand", "delete"): + return "Error: action must be one of: list, add, expand, delete." + + from sqlalchemy import select + + from app.models.keyword import Keyword + + if action == "list": + async with get_session() as db: + stmt = select(Keyword).where(Keyword.project_id == kb_id).order_by(Keyword.level, Keyword.term) + result = await db.execute(stmt) + keywords = result.scalars().all() + if not keywords: + return "No keywords found in this knowledge base." + lines = ["## Keywords\n", "| Term | EN | Level | Category |", "|---|---|---|---|"] + for kw in keywords: + lines.append(f"| {kw.term} | {kw.term_en} | {kw.level} | {kw.category} |") + return "\n".join(lines) + + if not term: + return f"Error: 'term' is required for action '{action}'." + + if action == "add": + async with get_session() as db: + kw = Keyword(project_id=kb_id, term=term, level=1) + db.add(kw) + await db.flush() + return f"Added keyword: {term}" + + if action == "expand": + from app.services.keyword_service import KeywordService + + svc = KeywordService() + result = await svc.expand_keywords([term], language=language) + expanded = result.get("expanded_terms", []) + if not expanded: + return "No expanded terms found." + lines = [f"## Expanded from: {term}\n"] + for et in expanded: + lines.append(f"- {et.get('term', '')} ({et.get('relation', '')})") + return "\n".join(lines) + + if action == "delete": + async with get_session() as db: + stmt = select(Keyword).where(Keyword.project_id == kb_id, Keyword.term == term) + result = await db.execute(stmt) + kw = result.scalar_one_or_none() + if not kw: + return f"Keyword '{term}' not found." + await db.delete(kw) + return f"Deleted keyword: {term}" + + return "Unknown action." + + # ==================== RESOURCES ==================== diff --git a/backend/app/middleware/auth.py b/backend/app/middleware/auth.py index f98c9c7..9cf1a2a 100644 --- a/backend/app/middleware/auth.py +++ b/backend/app/middleware/auth.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -EXEMPT_PATHS = frozenset({"/", "/health", "/docs", "/openapi.json", "/redoc"}) +EXEMPT_PATHS = frozenset({"/", "/health", "/api/v1/settings/health", "/docs", "/openapi.json", "/redoc"}) EXEMPT_PREFIXES = ("/mcp",) @@ -26,7 +26,7 @@ async def dispatch(self, request: Request, call_next) -> Response: if path in EXEMPT_PATHS or any(path.startswith(p) for p in EXEMPT_PREFIXES): return await call_next(request) - api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key") + api_key = request.headers.get("X-API-Key") if api_key != settings.api_secret_key: return JSONResponse( status_code=401, diff --git a/backend/app/middleware/rate_limit.py b/backend/app/middleware/rate_limit.py index 220a42a..c5086e4 100644 --- a/backend/app/middleware/rate_limit.py +++ b/backend/app/middleware/rate_limit.py @@ -8,12 +8,15 @@ from slowapi.middleware import SlowAPIMiddleware from slowapi.util import get_remote_address +from app.config import settings + logger = logging.getLogger(__name__) limiter = Limiter( key_func=get_remote_address, - default_limits=["120/minute"], + default_limits=[settings.rate_limit], storage_uri="memory://", + enabled=settings.app_env != "testing", ) @@ -22,4 +25,4 @@ def setup_rate_limiting(app: FastAPI) -> None: app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) app.add_middleware(SlowAPIMiddleware) - logger.info("Rate limiting enabled (default: 120/min)") + logger.info("Rate limiting enabled (default: %s)", settings.rate_limit) diff --git a/backend/app/models/keyword.py b/backend/app/models/keyword.py index c92a437..d21b3fc 100644 --- a/backend/app/models/keyword.py +++ b/backend/app/models/keyword.py @@ -17,7 +17,7 @@ class Keyword(Base): term_en: Mapped[str] = mapped_column(String(500), default="") level: Mapped[int] = mapped_column(Integer, default=1) # 1=core, 2=sub-domain, 3=expanded category: Mapped[str] = mapped_column(String(100), default="") - parent_id: Mapped[int | None] = mapped_column(Integer, ForeignKey("keywords.id"), default=None) + parent_id: Mapped[int | None] = mapped_column(Integer, ForeignKey("keywords.id"), default=None, index=True) synonyms: Mapped[str] = mapped_column(Text, default="") created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now()) diff --git a/backend/app/models/paper.py b/backend/app/models/paper.py index 23e5e5d..55aa28b 100644 --- a/backend/app/models/paper.py +++ b/backend/app/models/paper.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import StrEnum -from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy import JSON, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint, func from sqlalchemy.orm import Mapped, mapped_column, relationship from app.database import Base @@ -20,6 +20,10 @@ class PaperStatus(StrEnum): class Paper(Base): __tablename__ = "papers" + __table_args__ = ( + Index("ix_paper_project_status", "project_id", "status"), + UniqueConstraint("project_id", "doi", name="uq_paper_project_doi"), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) project_id: Mapped[int] = mapped_column(Integer, ForeignKey("projects.id"), nullable=False, index=True) diff --git a/backend/app/models/task.py b/backend/app/models/task.py index 0a42012..dbe1ce5 100644 --- a/backend/app/models/task.py +++ b/backend/app/models/task.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import StrEnum -from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy import JSON, DateTime, ForeignKey, Index, Integer, String, Text, func from sqlalchemy.orm import Mapped, mapped_column, relationship from app.database import Base @@ -28,6 +28,7 @@ class TaskType(StrEnum): class Task(Base): __tablename__ = "tasks" + __table_args__ = (Index("ix_task_project_status", "project_id", "status"),) id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) project_id: Mapped[int] = mapped_column(Integer, ForeignKey("projects.id"), nullable=False, index=True) diff --git a/backend/app/pipelines/cancellation.py b/backend/app/pipelines/cancellation.py new file mode 100644 index 0000000..2b362c4 --- /dev/null +++ b/backend/app/pipelines/cancellation.py @@ -0,0 +1,18 @@ +"""Shared cancellation state for pipeline execution.""" + +_cancelled: dict[str, bool] = {} + + +def is_cancelled(thread_id: str) -> bool: + """Check if pipeline has been cancelled via the API.""" + return _cancelled.get(thread_id, False) + + +def mark_cancelled(thread_id: str) -> None: + """Mark a pipeline as cancelled.""" + _cancelled[thread_id] = True + + +def clear_cancelled(thread_id: str) -> None: + """Clear cancellation flag for a pipeline (call when pipeline ends).""" + _cancelled.pop(thread_id, None) diff --git a/backend/app/pipelines/chat/nodes.py b/backend/app/pipelines/chat/nodes.py index d4ad3e7..f53f6fe 100644 --- a/backend/app/pipelines/chat/nodes.py +++ b/backend/app/pipelines/chat/nodes.py @@ -15,45 +15,23 @@ from langchain_core.runnables import RunnableConfig from langgraph.config import get_stream_writer +from app.config import settings from app.pipelines.chat.config_helpers import ( get_chat_db, get_chat_llm, get_chat_rag, ) from app.pipelines.chat.state import ChatMessageDict, ChatState, CitationDict +from app.prompts.chat import ( + CHAT_FALLBACK_SYSTEM, + CHAT_QA_SYSTEM, + CHAT_TOOL_MODE_PROMPTS, + EXCERPT_CLEAN_SYSTEM, +) logger = logging.getLogger(__name__) -TOOL_MODE_PROMPTS = { - "qa": ( - "You are a scientific research assistant. Answer the question based on the provided context. " - "Use inline citations like [1], [2] to reference source papers. " - "If the context doesn't contain enough information, say so honestly." - ), - "citation_lookup": ( - "You are a citation finder. Given the user's text, identify and list the most relevant " - "references from the provided context. Format as a numbered list with paper titles, authors, " - "and brief explanations of relevance. Keep your own commentary minimal." - ), - "review_outline": ( - "You are a literature review expert. Based on the provided context, generate a structured " - "review outline with sections, subsections, and key points. Use citations like [1], [2] " - "to reference sources. Suggest a logical flow and highlight key themes." - ), - "gap_analysis": ( - "You are a research gap analyst. Based on the provided literature context, identify " - "research gaps, unexplored areas, and potential future directions. Cite existing work " - "using [1], [2] format. Be specific about what has been studied and what remains open." - ), -} - -EXCERPT_CLEAN_PROMPT = ( - "Clean up the following text extracted from an academic PDF. " - "Fix OCR errors, add missing spaces between words, restore formatting. " - "Keep the original meaning intact. Output only the cleaned text, nothing else." -) - -_clean_semaphore = asyncio.Semaphore(3) +_clean_semaphore = asyncio.Semaphore(settings.clean_semaphore_limit) def _emit_thinking( @@ -114,14 +92,7 @@ async def understand_node(state: ChatState, config: RunnableConfig) -> dict[str, # Build system prompt kb_ids = state.get("knowledge_base_ids", []) tool_mode = state.get("tool_mode", "qa") - if kb_ids: - system_prompt = TOOL_MODE_PROMPTS.get(tool_mode, TOOL_MODE_PROMPTS["qa"]) - else: - system_prompt = ( - "You are a helpful scientific research assistant. " - "Answer questions clearly and accurately. " - "If you don't know the answer, say so honestly." - ) + system_prompt = CHAT_TOOL_MODE_PROMPTS.get(tool_mode, CHAT_QA_SYSTEM) if kb_ids else CHAT_FALLBACK_SYSTEM _emit_thinking( writer, @@ -158,7 +129,11 @@ async def retrieve_node(state: ChatState, config: RunnableConfig) -> dict[str, A ) top_k = state.get("rag_top_k") or 10 - tasks = [rag.retrieve_only(project_id=kb_id, question=state["message"], top_k=top_k) for kb_id in kb_ids] + use_reranker = state.get("use_reranker", False) + tasks = [ + rag.retrieve_only(project_id=kb_id, question=state["message"], top_k=top_k, use_reranker=use_reranker) + for kb_id in kb_ids + ] results = await asyncio.gather(*tasks, return_exceptions=True) all_sources: list[dict[str, Any]] = [] @@ -258,7 +233,7 @@ async def _clean_single_excerpt(llm, excerpt: str) -> str: return excerpt async with _clean_semaphore: messages = [ - {"role": "system", "content": EXCERPT_CLEAN_PROMPT}, + {"role": "system", "content": EXCERPT_CLEAN_SYSTEM}, {"role": "user", "content": excerpt}, ] result = "" @@ -427,7 +402,7 @@ async def persist_node(state: ChatState, config: RunnableConfig) -> dict[str, An ) db.add(user_msg) db.add(assistant_msg) - await db.commit() + await db.flush() citation_count = len(state.get("citations") or []) _emit_thinking( diff --git a/backend/app/pipelines/graphs.py b/backend/app/pipelines/graphs.py index 2ef4b11..f3e4497 100644 --- a/backend/app/pipelines/graphs.py +++ b/backend/app/pipelines/graphs.py @@ -4,6 +4,7 @@ import logging +from langgraph.checkpoint.base import BaseCheckpointSaver from langgraph.checkpoint.memory import MemorySaver from langgraph.graph import END, StateGraph @@ -23,16 +24,22 @@ logger = logging.getLogger(__name__) _memory_saver = MemorySaver() +_checkpoint_saver: BaseCheckpointSaver | None = None + + +def set_checkpointer(saver: BaseCheckpointSaver | None) -> None: + """Set the persistent checkpointer (called from lifespan). None restores MemorySaver fallback.""" + global _checkpoint_saver + _checkpoint_saver = saver def _get_checkpointer(): """Return a checkpointer for pipeline state persistence. - AsyncSqliteSaver.from_conn_string returns an async context manager, - not a direct BaseCheckpointSaver instance. Use MemorySaver which is - sufficient for single-process deployments with in-memory task tracking. + Uses AsyncSqliteSaver when available (set via set_checkpointer in lifespan), + otherwise falls back to MemorySaver for single-process in-memory persistence. """ - return _memory_saver + return _checkpoint_saver if _checkpoint_saver is not None else _memory_saver def _route_after_dedup(state: PipelineState) -> str: diff --git a/backend/app/pipelines/nodes.py b/backend/app/pipelines/nodes.py index dea9545..6cffde4 100644 --- a/backend/app/pipelines/nodes.py +++ b/backend/app/pipelines/nodes.py @@ -12,8 +12,19 @@ logger = logging.getLogger(__name__) +def _is_cancelled(state: PipelineState) -> bool: + """Check if pipeline has been cancelled via the API.""" + from app.pipelines.cancellation import is_cancelled + + thread_id = state.get("thread_id", "") + return is_cancelled(thread_id) or state.get("cancelled", False) + + async def search_node(state: PipelineState) -> dict[str, Any]: """Run multi-source federated search.""" + if _is_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + from app.services.search_service import SearchService params = state.get("params", {}) @@ -34,6 +45,9 @@ async def search_node(state: PipelineState) -> dict[str, Any]: async def extract_metadata_node(state: PipelineState) -> dict[str, Any]: """Extract metadata from uploaded PDF files.""" + if _is_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + from app.services.pdf_metadata import extract_metadata params = state.get("params", {}) @@ -57,6 +71,9 @@ async def extract_metadata_node(state: PipelineState) -> dict[str, Any]: async def dedup_node(state: PipelineState) -> dict[str, Any]: """Check for duplicates against existing papers in the knowledge base.""" + if _is_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + from sqlalchemy import select from app.config import settings @@ -149,20 +166,29 @@ async def hitl_dedup_node(state: PipelineState) -> dict[str, Any]: async def apply_resolution_node(state: PipelineState) -> dict[str, Any]: """Apply conflict resolutions and merge clean papers for import.""" + if _is_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + resolved = state.get("resolved_conflicts", []) clean_papers = list(state.get("papers", [])) for res in resolved: action = res.get("action", "skip") - new_paper = res.get("new_paper", {}) + new_paper = res.get("new_paper") or {} + merged_paper = res.get("merged_paper") or {} if action == "keep_new" and new_paper: clean_papers.append(new_paper) + elif action == "merge" and merged_paper: + clean_papers.append(merged_paper) return {"papers": clean_papers, "stage": "resolved"} async def import_node(state: PipelineState) -> dict[str, Any]: """Import clean papers into the database.""" + if _is_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + from app.database import async_session_factory from app.models import Paper @@ -193,6 +219,9 @@ async def import_node(state: PipelineState) -> dict[str, Any]: async def crawl_node(state: PipelineState) -> dict[str, Any]: """Download PDFs for papers that have pdf_url but no pdf_path.""" + if _is_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + from sqlalchemy import select from app.database import async_session_factory @@ -241,6 +270,9 @@ async def ocr_node(state: PipelineState) -> dict[str, Any]: Uses MinerU (if available) for deep parsing with formula/table/figure recognition, falling back to pdfplumber + PaddleOCR. """ + if _is_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + from sqlalchemy import select from app.database import async_session_factory @@ -258,43 +290,43 @@ async def ocr_node(state: PipelineState) -> dict[str, Any]: Paper.pdf_path != "", ) papers = (await db.execute(stmt)).scalars().all() - ocr = OCRService(use_gpu=True) - for paper in papers: - if state.get("cancelled"): - break - try: - result = await ocr.process_pdf_async(paper.pdf_path) - if result.get("error"): - paper.status = PaperStatus.ERROR - continue + with OCRService(use_gpu=True) as ocr: + for paper in papers: + if state.get("cancelled"): + break + try: + result = await ocr.process_pdf_async(paper.pdf_path) + if result.get("error"): + paper.status = PaperStatus.ERROR + continue - if result.get("method") == "mineru": - chunks = ocr.chunk_mineru_markdown(result["md_content"], chunk_size=1024, overlap=100) - else: - pages = result.get("pages", []) - chunks = ocr.chunk_text(pages, chunk_size=1024, overlap=100) - - for chunk_data in chunks: - db.add( - PaperChunk( - paper_id=paper.id, - content=chunk_data["content"], - page_number=chunk_data.get("page_number", 0), - chunk_index=chunk_data["chunk_index"], - chunk_type=chunk_data.get("chunk_type", "text"), - section=chunk_data.get("section", ""), - token_count=chunk_data.get("token_count", 0), - has_formula=chunk_data.get("has_formula", False), - figure_path=chunk_data.get("figure_path", ""), + if result.get("method") == "mineru": + chunks = ocr.chunk_mineru_markdown(result["md_content"], chunk_size=1024, overlap=100) + else: + pages = result.get("pages", []) + chunks = ocr.chunk_text(pages, chunk_size=1024, overlap=100) + + for chunk_data in chunks: + db.add( + PaperChunk( + paper_id=paper.id, + content=chunk_data["content"], + page_number=chunk_data.get("page_number", 0), + chunk_index=chunk_data["chunk_index"], + chunk_type=chunk_data.get("chunk_type", "text"), + section=chunk_data.get("section", ""), + token_count=chunk_data.get("token_count", 0), + has_formula=chunk_data.get("has_formula", False), + figure_path=chunk_data.get("figure_path", ""), + ) ) - ) - paper.status = PaperStatus.OCR_COMPLETE - processed += 1 - except Exception as e: - logger.warning("OCR failed for paper %d: %s", paper.id, e) - paper.status = PaperStatus.ERROR + paper.status = PaperStatus.OCR_COMPLETE + processed += 1 + except Exception as e: + logger.warning("OCR failed for paper %d: %s", paper.id, e) + paper.status = PaperStatus.ERROR await db.commit() return { @@ -306,6 +338,9 @@ async def ocr_node(state: PipelineState) -> dict[str, Any]: async def index_node(state: PipelineState) -> dict[str, Any]: """Index OCR-processed papers into the RAG vector store.""" + if _is_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + from sqlalchemy import select from app.database import async_session_factory diff --git a/backend/app/prompts/__init__.py b/backend/app/prompts/__init__.py new file mode 100644 index 0000000..741b798 --- /dev/null +++ b/backend/app/prompts/__init__.py @@ -0,0 +1,52 @@ +"""Centralized LLM prompt management for all Omelette backend services.""" + +from app.prompts.chat import ( + CHAT_CITATION_SYSTEM, + CHAT_FALLBACK_SYSTEM, + CHAT_GAP_SYSTEM, + CHAT_OUTLINE_SYSTEM, + CHAT_QA_SYSTEM, + CHAT_TOOL_MODE_PROMPTS, + EXCERPT_CLEAN_SYSTEM, +) +from app.prompts.completion import COMPLETION_SYSTEM +from app.prompts.dedup import DEDUP_RESOLVE_SYSTEM, DEDUP_VERIFY_SYSTEM +from app.prompts.keyword import KEYWORD_EXPAND_SYSTEM +from app.prompts.rag import RAG_ANSWER_SYSTEM +from app.prompts.rewrite import ( + REWRITE_ACADEMIC, + REWRITE_PROMPTS, + REWRITE_SIMPLIFY, + REWRITE_TRANSLATE_EN, + REWRITE_TRANSLATE_ZH, +) +from app.prompts.writing import ( + WRITING_GAP_SYSTEM, + WRITING_OUTLINE_SYSTEM, + WRITING_SECTION_SYSTEM, + WRITING_SUMMARIZE_SYSTEM, +) + +__all__ = [ + "CHAT_CITATION_SYSTEM", + "CHAT_FALLBACK_SYSTEM", + "CHAT_GAP_SYSTEM", + "CHAT_OUTLINE_SYSTEM", + "CHAT_QA_SYSTEM", + "CHAT_TOOL_MODE_PROMPTS", + "COMPLETION_SYSTEM", + "DEDUP_RESOLVE_SYSTEM", + "DEDUP_VERIFY_SYSTEM", + "EXCERPT_CLEAN_SYSTEM", + "KEYWORD_EXPAND_SYSTEM", + "RAG_ANSWER_SYSTEM", + "REWRITE_ACADEMIC", + "REWRITE_PROMPTS", + "REWRITE_SIMPLIFY", + "REWRITE_TRANSLATE_EN", + "REWRITE_TRANSLATE_ZH", + "WRITING_GAP_SYSTEM", + "WRITING_OUTLINE_SYSTEM", + "WRITING_SECTION_SYSTEM", + "WRITING_SUMMARIZE_SYSTEM", +] diff --git a/backend/app/prompts/chat.py b/backend/app/prompts/chat.py new file mode 100644 index 0000000..ee31681 --- /dev/null +++ b/backend/app/prompts/chat.py @@ -0,0 +1,49 @@ +"""Chat pipeline system prompts.""" + +CHAT_QA_SYSTEM = ( + "You are a scientific research assistant. Answer the question based on the provided context. " + "Use inline citations like [1], [2] to reference source papers. " + "If the context doesn't contain enough information, say so honestly. " + "Structure your answer with clear paragraphs. " + "Respond in the same language as the user's question." +) + +CHAT_CITATION_SYSTEM = ( + "You are a citation finder. Given the user's text, identify and list the most relevant " + "references from the provided context. Format as a numbered list with paper titles, authors, " + "and brief explanations of relevance. Include DOI when available. " + "Keep your own commentary minimal." +) + +CHAT_OUTLINE_SYSTEM = ( + "You are a literature review expert. Based on the provided context, generate a structured " + "review outline with sections, subsections, and key points. Use markdown headers for sections. " + "Use citations like [1], [2] to reference sources. Suggest a logical flow and highlight key themes." +) + +CHAT_GAP_SYSTEM = ( + "You are a research gap analyst. Based on the provided literature context, identify " + "research gaps, unexplored areas, and potential future directions. Cite existing work " + "using [1], [2] format. Organize by theme, not by individual papers. " + "Be specific about what has been studied and what remains open." +) + +CHAT_FALLBACK_SYSTEM = ( + "You are a scientific research assistant specializing in academic literature analysis. " + "Answer questions clearly and accurately based on your knowledge. " + "When the user's question is outside your expertise or you are uncertain, say so honestly. " + "Respond in the same language as the user's question." +) + +EXCERPT_CLEAN_SYSTEM = ( + "Clean up the following text extracted from an academic PDF. " + "Fix OCR errors, add missing spaces between words, restore formatting. " + "Keep the original meaning intact. Output only the cleaned text, nothing else." +) + +CHAT_TOOL_MODE_PROMPTS: dict[str, str] = { + "qa": CHAT_QA_SYSTEM, + "citation_lookup": CHAT_CITATION_SYSTEM, + "review_outline": CHAT_OUTLINE_SYSTEM, + "gap_analysis": CHAT_GAP_SYSTEM, +} diff --git a/backend/app/prompts/completion.py b/backend/app/prompts/completion.py new file mode 100644 index 0000000..bef1a5e --- /dev/null +++ b/backend/app/prompts/completion.py @@ -0,0 +1,8 @@ +"""Writing completion system prompts.""" + +COMPLETION_SYSTEM = ( + "You are a scientific writing assistant. Predict and complete the user's text. " + "Return only the completion (do not repeat the user's input), max 50 characters. " + "If you cannot reasonably predict, return an empty string. " + "Return plain text only — no quotes, explanations, or formatting." +) diff --git a/backend/app/prompts/dedup.py b/backend/app/prompts/dedup.py new file mode 100644 index 0000000..e23ee3c --- /dev/null +++ b/backend/app/prompts/dedup.py @@ -0,0 +1,13 @@ +"""Deduplication system prompts.""" + +DEDUP_VERIFY_SYSTEM = ( + "You are a scientific literature deduplication expert. " + "Compare papers carefully based on title, authors, DOI, and journal. " + "Return valid JSON only." +) + +DEDUP_RESOLVE_SYSTEM = ( + "You are a scientific literature deduplication expert. " + "Determine the best resolution for duplicate candidates. " + "Return valid JSON only." +) diff --git a/backend/app/prompts/keyword.py b/backend/app/prompts/keyword.py new file mode 100644 index 0000000..ff84af1 --- /dev/null +++ b/backend/app/prompts/keyword.py @@ -0,0 +1,8 @@ +"""Keyword expansion system prompts.""" + +KEYWORD_EXPAND_SYSTEM = ( + "You are a scientific terminology expert. " + "Generate related terms including synonyms, abbreviations, technical variants, " + "and cross-disciplinary application terms. " + "Return valid JSON only." +) diff --git a/backend/app/prompts/rag.py b/backend/app/prompts/rag.py new file mode 100644 index 0000000..006ff07 --- /dev/null +++ b/backend/app/prompts/rag.py @@ -0,0 +1,8 @@ +"""RAG knowledge base system prompts.""" + +RAG_ANSWER_SYSTEM = ( + "You are a scientific research assistant. " + "Answer questions based strictly on the provided context. " + "Cite sources accurately using the format provided. " + "Respond in the same language as the user's question." +) diff --git a/backend/app/prompts/rewrite.py b/backend/app/prompts/rewrite.py new file mode 100644 index 0000000..42b13a1 --- /dev/null +++ b/backend/app/prompts/rewrite.py @@ -0,0 +1,32 @@ +"""Text rewrite and translation system prompts.""" + +REWRITE_SIMPLIFY = ( + "Rewrite the following academic text in plain, accessible language. " + "Keep the core meaning and key concepts intact, but make it understandable " + "to a general audience. Output only the rewritten text, no explanations." +) + +REWRITE_ACADEMIC = ( + "Rewrite the following text in formal academic style. " + "Use precise terminology, passive voice where appropriate, and proper " + "academic conventions. Maintain the original meaning. Output only the rewritten text." +) + +REWRITE_TRANSLATE_EN = ( + "Translate the following text into English. " + "Preserve academic terminology and the original meaning. " + "Output only the translation, no explanations." +) + +REWRITE_TRANSLATE_ZH = ( + "Translate the following text into Chinese. " + "Preserve academic terminology and the original meaning. " + "Output only the translation, no explanations." +) + +REWRITE_PROMPTS: dict[str, str] = { + "simplify": REWRITE_SIMPLIFY, + "academic": REWRITE_ACADEMIC, + "translate_en": REWRITE_TRANSLATE_EN, + "translate_zh": REWRITE_TRANSLATE_ZH, +} diff --git a/backend/app/prompts/writing.py b/backend/app/prompts/writing.py new file mode 100644 index 0000000..e8babee --- /dev/null +++ b/backend/app/prompts/writing.py @@ -0,0 +1,26 @@ +"""Writing assistant system prompts.""" + +WRITING_SECTION_SYSTEM = ( + "You are an academic review writing expert. Write a review paragraph for the given section. " + "Requirements: " + "1. Use academic language with clear logic. " + "2. Use [1][2] format for citations at appropriate positions. " + "3. Every citation must correspond to a provided reference — do not fabricate. " + "4. Paragraph length: 200-400 words." +) + +WRITING_SUMMARIZE_SYSTEM = ( + "You are a scientific paper analyst. Provide structured, accurate summaries. " + "Focus on empirical findings and methodology. " + "Do not hallucinate information not present in the provided metadata." +) + +WRITING_OUTLINE_SYSTEM = ( + "You are a scientific writing expert. Generate well-structured review outlines " + "organized by research themes with clear section hierarchy." +) + +WRITING_GAP_SYSTEM = ( + "You are a research gap analyst. Identify unexplored areas and innovation opportunities " + "based on the provided literature." +) diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 5dc46e7..dd54771 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -1,6 +1,7 @@ """Pydantic schemas for API request/response validation.""" -from app.schemas.common import ApiResponse, PaginatedData, PaginationParams, TaskResponse +from app.schemas.common import ApiResponse, KeywordPaginationParams, PaginatedData, PaginationParams, TaskResponse +from app.schemas.conversation import ChatStreamRequest, ConversationCreateSchema, ConversationUpdateSchema from app.schemas.keyword import ( KeywordCreate, KeywordExpandRequest, @@ -8,11 +9,14 @@ KeywordRead, KeywordUpdate, ) -from app.schemas.paper import PaperBulkImport, PaperCreate, PaperRead, PaperUpdate +from app.schemas.llm import LLMConfig, ProviderModelInfo, SettingsSchema, SettingsUpdateSchema +from app.schemas.paper import PaperBatchDeleteRequest, PaperBulkImport, PaperCreate, PaperRead, PaperUpdate from app.schemas.project import ProjectCreate, ProjectRead, ProjectUpdate +from app.schemas.subscription import SubscriptionCreate, SubscriptionRead, SubscriptionUpdate __all__ = [ "ApiResponse", + "KeywordPaginationParams", "PaginatedData", "PaginationParams", "TaskResponse", @@ -23,9 +27,20 @@ "PaperRead", "PaperUpdate", "PaperBulkImport", + "PaperBatchDeleteRequest", "KeywordCreate", "KeywordRead", "KeywordUpdate", "KeywordExpandRequest", "KeywordExpandResponse", + "ConversationCreateSchema", + "ConversationUpdateSchema", + "ChatStreamRequest", + "SubscriptionCreate", + "SubscriptionRead", + "SubscriptionUpdate", + "LLMConfig", + "ProviderModelInfo", + "SettingsSchema", + "SettingsUpdateSchema", ] diff --git a/backend/app/schemas/chunk.py b/backend/app/schemas/chunk.py new file mode 100644 index 0000000..a5105ef --- /dev/null +++ b/backend/app/schemas/chunk.py @@ -0,0 +1,21 @@ +"""Pydantic schemas for PaperChunk.""" + +from datetime import datetime + +from pydantic import BaseModel + + +class ChunkRead(BaseModel): + id: int + paper_id: int + chunk_type: str + content: str + section: str + page_number: int | None + chunk_index: int + token_count: int + has_formula: bool + figure_path: str + created_at: datetime + + model_config = {"from_attributes": True} diff --git a/backend/app/schemas/common.py b/backend/app/schemas/common.py index 1fe6986..01c3fa5 100644 --- a/backend/app/schemas/common.py +++ b/backend/app/schemas/common.py @@ -3,11 +3,36 @@ from datetime import UTC, datetime from typing import Generic, TypeVar +from fastapi import Query from pydantic import BaseModel, Field T = TypeVar("T") +class PaginationParams: + """FastAPI dependency for pagination (page, page_size).""" + + def __init__( + self, + page: int = Query(1, ge=1, description="页码"), + page_size: int = Query(20, ge=1, le=100, description="每页数量"), + ): + self.page = page + self.page_size = page_size + + +class KeywordPaginationParams(PaginationParams): + """Pagination for keywords (page_size default 50 for backward compatibility).""" + + def __init__( + self, + page: int = Query(1, ge=1, description="页码"), + page_size: int = Query(50, ge=1, le=100, description="每页数量"), + ): + self.page = page + self.page_size = page_size + + class ApiResponse(BaseModel, Generic[T]): code: int = 200 message: str = "success" @@ -23,11 +48,6 @@ class PaginatedData(BaseModel, Generic[T]): total_pages: int = 1 -class PaginationParams(BaseModel): - page: int = Field(default=1, ge=1) - page_size: int = Field(default=20, ge=1, le=100) - - class TaskResponse(BaseModel): task_id: int status: str diff --git a/backend/app/schemas/conversation.py b/backend/app/schemas/conversation.py index 587a016..2c8e429 100644 --- a/backend/app/schemas/conversation.py +++ b/backend/app/schemas/conversation.py @@ -1,6 +1,7 @@ """Schemas for conversations and messages.""" from datetime import datetime +from typing import Literal from pydantic import BaseModel, Field @@ -44,21 +45,23 @@ class ConversationListSchema(BaseModel): class ConversationCreateSchema(BaseModel): - title: str = "" + title: str = Field(default="", max_length=500) knowledge_base_ids: list[int] | None = None model: str = "" - tool_mode: str = "qa" + tool_mode: Literal["qa", "citation_lookup", "review_outline", "gap_analysis"] = "qa" class ConversationUpdateSchema(BaseModel): - title: str | None = None + title: str | None = Field(default=None, max_length=500) model: str | None = None - tool_mode: str | None = None + tool_mode: Literal["qa", "citation_lookup", "review_outline", "gap_analysis"] | None = None class ChatStreamRequest(BaseModel): conversation_id: int | None = None - knowledge_base_ids: list[int] = Field(default_factory=list) + knowledge_base_ids: list[int] = Field(default_factory=list, max_length=20) model: str | None = None - tool_mode: str = "qa" + tool_mode: Literal["qa", "citation_lookup", "review_outline", "gap_analysis"] = "qa" message: str = Field(min_length=1) + rag_top_k: int = Field(default=10, ge=1, le=50, description="RAG retrieval top-k") + use_reranker: bool = Field(default=False, description="Apply reranker to retrieved nodes") diff --git a/backend/app/schemas/keyword.py b/backend/app/schemas/keyword.py index 577e840..3e49dd8 100644 --- a/backend/app/schemas/keyword.py +++ b/backend/app/schemas/keyword.py @@ -40,9 +40,9 @@ class KeywordRead(BaseModel): class KeywordExpandRequest(BaseModel): - seed_terms: list[str] - language: str = "en" - max_results: int = 20 + seed_terms: list[str] = Field(..., max_length=50) + language: str = Field(default="en", max_length=10) + max_results: int = Field(default=20, ge=1, le=100) class KeywordExpandResponse(BaseModel): diff --git a/backend/app/schemas/knowledge_base.py b/backend/app/schemas/knowledge_base.py index 75ce2c6..f73c716 100644 --- a/backend/app/schemas/knowledge_base.py +++ b/backend/app/schemas/knowledge_base.py @@ -1,13 +1,15 @@ """Pydantic schemas for knowledge base and PDF upload operations.""" -from pydantic import BaseModel +from typing import Literal + +from pydantic import BaseModel, Field from app.schemas.paper import PaperRead class NewPaperData(BaseModel): - title: str - abstract: str = "" + title: str = Field(..., max_length=2000) + abstract: str = Field(default="", max_length=50000) authors: list[dict[str, str]] | None = None doi: str | None = None year: int | None = None @@ -32,7 +34,7 @@ class UploadResult(BaseModel): class ResolveConflictRequest(BaseModel): conflict_id: str - action: str # "keep_old" | "keep_new" | "merge" | "skip" + action: Literal["keep_old", "keep_new", "merge", "skip"] merged_paper: dict | None = None diff --git a/backend/app/schemas/llm.py b/backend/app/schemas/llm.py index 4b21d7b..01dd4d9 100644 --- a/backend/app/schemas/llm.py +++ b/backend/app/schemas/llm.py @@ -66,19 +66,19 @@ class SettingsUpdateSchema(BaseModel): llm_temperature: float | None = Field(default=None, ge=0.0, le=2.0) llm_max_tokens: int | None = Field(default=None, ge=1, le=128000) - openai_api_key: str | None = None - openai_model: str | None = None + openai_api_key: str | None = Field(default=None, max_length=500) + openai_model: str | None = Field(default=None, max_length=200) - anthropic_api_key: str | None = None - anthropic_model: str | None = None + anthropic_api_key: str | None = Field(default=None, max_length=500) + anthropic_model: str | None = Field(default=None, max_length=200) - aliyun_api_key: str | None = None - aliyun_base_url: str | None = None - aliyun_model: str | None = None + aliyun_api_key: str | None = Field(default=None, max_length=500) + aliyun_base_url: str | None = Field(default=None, max_length=500) + aliyun_model: str | None = Field(default=None, max_length=200) - volcengine_api_key: str | None = None - volcengine_base_url: str | None = None - volcengine_model: str | None = None + volcengine_api_key: str | None = Field(default=None, max_length=500) + volcengine_base_url: str | None = Field(default=None, max_length=500) + volcengine_model: str | None = Field(default=None, max_length=200) - ollama_base_url: str | None = None - ollama_model: str | None = None + ollama_base_url: str | None = Field(default=None, max_length=500) + ollama_model: str | None = Field(default=None, max_length=200) diff --git a/backend/app/schemas/paper.py b/backend/app/schemas/paper.py index 1ebe315..97e432a 100644 --- a/backend/app/schemas/paper.py +++ b/backend/app/schemas/paper.py @@ -1,33 +1,34 @@ """Pydantic schemas for Paper operations.""" from datetime import datetime +from typing import Literal from pydantic import BaseModel, Field class PaperCreate(BaseModel): doi: str | None = None - title: str = Field(..., min_length=1) - abstract: str = "" + title: str = Field(..., min_length=1, max_length=2000) + abstract: str = Field(default="", max_length=50000) authors: list[dict[str, str]] | None = None - journal: str = "" - year: int | None = None - citation_count: int = 0 - source: str = "" - source_id: str = "" - pdf_url: str = "" + journal: str = Field(default="", max_length=500) + year: int | None = Field(default=None, ge=1800, le=2100) + citation_count: int = Field(default=0, ge=0) + source: str = Field(default="", max_length=200) + source_id: str = Field(default="", max_length=500) + pdf_url: str = Field(default="", max_length=5000) tags: list[str] | None = None class PaperUpdate(BaseModel): - title: str | None = None - abstract: str | None = None + title: str | None = Field(default=None, max_length=2000) + abstract: str | None = Field(default=None, max_length=50000) authors: list[dict[str, str]] | None = None - journal: str | None = None - year: int | None = None + journal: str | None = Field(default=None, max_length=500) + year: int | None = Field(default=None, ge=1800, le=2100) tags: list[str] | None = None notes: str | None = None - status: str | None = None + status: Literal["pending", "metadata_only", "pdf_downloaded", "ocr_complete", "indexed", "error"] | None = None class PaperRead(BaseModel): @@ -54,4 +55,8 @@ class PaperRead(BaseModel): class PaperBulkImport(BaseModel): - papers: list[PaperCreate] + papers: list[PaperCreate] = Field(..., max_length=500) + + +class PaperBatchDeleteRequest(BaseModel): + paper_ids: list[int] = Field(..., min_length=1, max_length=500) diff --git a/backend/app/schemas/project.py b/backend/app/schemas/project.py index 6eb1930..1b60292 100644 --- a/backend/app/schemas/project.py +++ b/backend/app/schemas/project.py @@ -6,6 +6,41 @@ from pydantic import BaseModel, Field +class PaperImportItem(BaseModel): + """Schema for a single paper in project import.""" + + title: str = "" + abstract: str = "" + doi: str | None = None + authors: list | None = None + year: int | None = None + journal: str = "" + source: str = "" + pdf_url: str = "" + status: str = "" + citation_count: int = 0 + + +class KeywordImportItem(BaseModel): + """Schema for a single keyword in project import.""" + + term: str = Field(..., min_length=1) + term_en: str = "" + level: int = 1 + category: str = "" + synonyms: str = "" + + +class SubscriptionImportItem(BaseModel): + """Schema for a single subscription in project import.""" + + name: str = Field(..., min_length=1) + query: str = "" + sources: list[str] = Field(default_factory=list) + frequency: str = "weekly" + max_results: int = 50 + + class ProjectCreate(BaseModel): name: str = Field(..., min_length=1, max_length=255) description: str = "" diff --git a/backend/app/schemas/subscription.py b/backend/app/schemas/subscription.py index 7982ea2..133ba86 100644 --- a/backend/app/schemas/subscription.py +++ b/backend/app/schemas/subscription.py @@ -1,24 +1,25 @@ """Subscription schemas for request/response.""" from datetime import datetime +from typing import Literal from pydantic import BaseModel, Field class SubscriptionCreate(BaseModel): - name: str = Field(..., min_length=1) - query: str = "" + name: str = Field(..., min_length=1, max_length=500) + query: str = Field(default="", max_length=2000) sources: list[str] = [] - frequency: str = "weekly" + frequency: Literal["daily", "weekly", "monthly"] = "weekly" max_results: int = Field(50, ge=1, le=200) class SubscriptionUpdate(BaseModel): - name: str | None = None - query: str | None = None + name: str | None = Field(default=None, max_length=500) + query: str | None = Field(default=None, max_length=2000) sources: list[str] | None = None - frequency: str | None = None - max_results: int | None = None + frequency: Literal["daily", "weekly", "monthly"] | None = None + max_results: int | None = Field(default=None, ge=1, le=200) is_active: bool | None = None @@ -43,3 +44,4 @@ class SubscriptionRunResult(BaseModel): new_papers: int total_checked: int sources_searched: list[str] + imported: int = 0 diff --git a/backend/app/services/citation_graph_service.py b/backend/app/services/citation_graph_service.py index 8c42167..b5af48f 100644 --- a/backend/app/services/citation_graph_service.py +++ b/backend/app/services/citation_graph_service.py @@ -6,6 +6,7 @@ from typing import Any import httpx +from fastapi import HTTPException from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -14,10 +15,10 @@ logger = logging.getLogger(__name__) -S2_API_BASE = "https://api.semanticscholar.org/graph/v1" S2_FIELDS = "title,year,citationCount,externalIds,authors" -S2_TIMEOUT = 15 -S2_MAX_PER_REQUEST = 50 + +# Error messages (extracted for maintainability) +CITATION_NOT_FOUND = "无法获取引用数据：Semantic Scholar 未收录此论文" class CitationGraphService: @@ -37,16 +38,11 @@ async def get_citation_graph( """Return {nodes, edges, center_id} for a paper's citation network.""" paper = await self._db.get(Paper, paper_id) if not paper or paper.project_id != project_id: - return {"nodes": [], "edges": [], "center_id": None, "error": "Paper not found"} + raise HTTPException(status_code=404, detail="Paper not found") s2_id = await self._resolve_s2_id(paper) if not s2_id: - return { - "nodes": [], - "edges": [], - "center_id": None, - "error": "无法获取引用数据：Semantic Scholar 未收录此论文", - } + raise HTTPException(status_code=502, detail=CITATION_NOT_FOUND) local_source_ids = await self._get_local_source_ids(project_id) @@ -64,7 +60,7 @@ async def get_citation_graph( } nodes[s2_id] = center_node - citations = await self._fetch_s2_list(f"{S2_API_BASE}/paper/{s2_id}/citations", max_nodes // 2) + citations = await self._fetch_s2_list(f"{settings.s2_api_base}/paper/{s2_id}/citations", max_nodes // 2) for item in citations: cited_paper = item.get("citingPaper", {}) cid = cited_paper.get("paperId") @@ -76,7 +72,9 @@ async def get_citation_graph( break if len(nodes) < max_nodes: - references = await self._fetch_s2_list(f"{S2_API_BASE}/paper/{s2_id}/references", max_nodes - len(nodes)) + references = await self._fetch_s2_list( + f"{settings.s2_api_base}/paper/{s2_id}/references", max_nodes - len(nodes) + ) for item in references: ref_paper = item.get("citedPaper", {}) rid = ref_paper.get("paperId") @@ -100,7 +98,7 @@ async def _resolve_s2_id(self, paper: Paper) -> str | None: if paper.doi: try: - data = await self._fetch_s2_json(f"{S2_API_BASE}/paper/DOI:{paper.doi}?fields=paperId") + data = await self._fetch_s2_json(f"{settings.s2_api_base}/paper/DOI:{paper.doi}?fields=paperId") if pid := data.get("paperId"): return pid except Exception: @@ -109,7 +107,7 @@ async def _resolve_s2_id(self, paper: Paper) -> str | None: if paper.title: try: data = await self._fetch_s2_json( - f"{S2_API_BASE}/paper/search", + f"{settings.s2_api_base}/paper/search", params={"query": paper.title[:200], "limit": "1", "fields": "paperId"}, ) papers = data.get("data", []) @@ -147,7 +145,7 @@ def _make_node(self, s2_paper: dict, local_ids: set[str]) -> dict: async def _fetch_s2_list(self, url: str, limit: int) -> list[dict]: """Fetch paginated list from S2 citations/references endpoint.""" - actual_limit = min(limit, S2_MAX_PER_REQUEST) + actual_limit = min(limit, settings.s2_max_per_request) try: data = await self._fetch_s2_json(url, params={"fields": S2_FIELDS, "limit": str(actual_limit)}) return data.get("data", []) @@ -160,7 +158,7 @@ async def _fetch_s2_json(self, url: str, params: dict | None = None) -> dict: if settings.semantic_scholar_api_key: headers["x-api-key"] = settings.semantic_scholar_api_key - async with httpx.AsyncClient(timeout=S2_TIMEOUT) as client: + async with httpx.AsyncClient(timeout=settings.s2_timeout) as client: resp = await client.get(url, headers=headers, params=params) if resp.status_code == 429: logger.warning("S2 API rate limited") diff --git a/backend/app/services/completion_service.py b/backend/app/services/completion_service.py index 864f2bf..085c942 100644 --- a/backend/app/services/completion_service.py +++ b/backend/app/services/completion_service.py @@ -4,17 +4,11 @@ import logging +from app.prompts.completion import COMPLETION_SYSTEM from app.services.llm.client import LLMClient, get_llm_client logger = logging.getLogger(__name__) -COMPLETION_SYSTEM_PROMPT = ( - "你是一个科研写作助手。根据用户已输入的文本，预测并补全后续内容。\n" - "只返回补全的部分（不要重复用户已输入的内容），最多50个字符。\n" - "如果无法合理预测，返回空字符串。\n" - "不要添加任何解释、引号或格式标记，只返回纯文本补全内容。" -) - class CompletionService: """Generates short text completions for chat input autocomplete.""" @@ -38,7 +32,7 @@ async def complete( return {"completion": "", "confidence": 0.0} messages: list[dict[str, str]] = [ - {"role": "system", "content": COMPLETION_SYSTEM_PROMPT}, + {"role": "system", "content": COMPLETION_SYSTEM}, ] if recent_messages: diff --git a/backend/app/services/crawler_service.py b/backend/app/services/crawler_service.py index 59b477e..2567d32 100644 --- a/backend/app/services/crawler_service.py +++ b/backend/app/services/crawler_service.py @@ -1,5 +1,6 @@ """PDF crawler service — download papers via Unpaywall, arXiv, and direct URLs.""" +import asyncio import hashlib import logging from pathlib import Path @@ -80,6 +81,13 @@ def _build_unpaywall_url(self, doi: str) -> str: async def _download_pdf(self, url: str, paper: Paper) -> dict: """Download a PDF from a URL and save to disk.""" + from app.services.url_validator import validate_url_safe + + try: + await asyncio.to_thread(validate_url_safe, url) + except ValueError as e: + return {"success": False, "error": f"URL blocked: {e}"} + proxy = _get_proxy() timeout = httpx.Timeout(60.0, connect=15.0) @@ -93,6 +101,10 @@ async def _download_pdf(self, url: str, paper: Paper) -> dict: pdf_url = best_oa.get("url_for_pdf") or best_oa.get("url") if best_oa else None if not pdf_url: return {"success": False, "error": "No open access PDF found"} + try: + await asyncio.to_thread(validate_url_safe, pdf_url) + except ValueError as e: + return {"success": False, "error": f"Resolved URL blocked: {e}"} url = pdf_url # Download the actual PDF @@ -132,8 +144,6 @@ def _get_file_path(self, paper: Paper) -> Path: async def batch_download(self, papers: list[Paper], max_concurrent: int = 5) -> dict: """Download PDFs for multiple papers with concurrency control.""" - import asyncio - semaphore = asyncio.Semaphore(max_concurrent) results = {"success": 0, "failed": 0, "skipped": 0, "details": []} diff --git a/backend/app/services/dedup_service.py b/backend/app/services/dedup_service.py index 7a25a74..a4694a0 100644 --- a/backend/app/services/dedup_service.py +++ b/backend/app/services/dedup_service.py @@ -9,7 +9,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.models import Paper, PaperStatus -from app.services.llm_client import LLMClient +from app.prompts.dedup import DEDUP_RESOLVE_SYSTEM, DEDUP_VERIFY_SYSTEM +from app.services.llm.client import LLMClient logger = logging.getLogger(__name__) @@ -219,12 +220,57 @@ async def llm_verify_duplicate(self, paper_a_id: int, paper_b_id: int) -> dict: result = await self.llm.chat_json( messages=[ - { - "role": "system", - "content": "You are a scientific literature deduplication expert. Return valid JSON only.", - }, + {"role": "system", "content": DEDUP_VERIFY_SYSTEM}, {"role": "user", "content": prompt}, ], task_type="dedup_check", ) return result + + async def resolve_conflict( + self, + old_paper: Paper, + new_title: str, + new_doi: str | None, + new_year: int | None, + new_journal: str | None, + ) -> dict: + """Use LLM to decide how to resolve a duplicate conflict.""" + if not self.llm: + return {"action": "keep_new", "reason": "LLM not available, defaulting to keep_new"} + + prompt = f"""Two papers may be duplicates. Decide the best resolution: + +Existing paper (in DB): +- ID: {old_paper.id} +- Title: {old_paper.title} +- DOI: {old_paper.doi or "N/A"} +- Year: {old_paper.year} +- Journal: {old_paper.journal} + +New upload: +- Title: {new_title} +- DOI: {new_doi or "N/A"} +- Year: {new_year} +- Journal: {new_journal} + +Return JSON: {{"action": "keep_old"|"keep_new"|"merge", "reason": "..."}} +- keep_old: existing is better, discard new +- keep_new: new is better or different work, add new +- merge: combine metadata, add as new paper""" + + try: + result = await self.llm.chat_json( + messages=[ + {"role": "system", "content": DEDUP_RESOLVE_SYSTEM}, + {"role": "user", "content": prompt}, + ], + task_type="dedup_resolve", + ) + action = result.get("action", "keep_new") + if action not in ("keep_old", "keep_new", "merge"): + action = "keep_new" + return {"action": action, "reason": result.get("reason", "")} + except Exception as e: + logger.warning("LLM auto-resolve failed: %s", e) + return {"action": "keep_new", "reason": f"Error: {e}"} diff --git a/backend/app/services/embedding_service.py b/backend/app/services/embedding_service.py index f146c7c..fcae7cd 100644 --- a/backend/app/services/embedding_service.py +++ b/backend/app/services/embedding_service.py @@ -13,7 +13,6 @@ logger = logging.getLogger(__name__) -_cached_embed_model: BaseEmbedding | None = None _env_injected = False @@ -37,21 +36,34 @@ def _inject_hf_env() -> None: logger.info("Using HuggingFace mirror: %s", settings.hf_endpoint) -def detect_gpu() -> tuple[bool, int, str]: - """Detect GPU availability. Returns (has_gpu, device_count, device_string).""" +def detect_gpu(*, pinned_gpu_id: int = -1) -> tuple[bool, int, str]: + """Detect GPU availability and pick the best device. + + Args: + pinned_gpu_id: If >= 0, skip auto-detection and return ``cuda:N``. + + Returns (has_gpu, device_count, device_string) where device_string is + ``"cuda:N"`` (best/pinned device) or ``"cpu"``. + """ try: import torch if torch.cuda.is_available(): count = torch.cuda.device_count() if count > 0: + if 0 <= pinned_gpu_id < count: + device = f"cuda:{pinned_gpu_id}" + logger.info("GPU pinned: %s (of %d device(s))", device, count) + return True, count, device devices_env = os.environ.get("CUDA_VISIBLE_DEVICES", settings.cuda_visible_devices) + best_device = _pick_best_gpu(count) logger.info( - "GPU detected: %d device(s), CUDA_VISIBLE_DEVICES=%s", + "GPU detected: %d device(s), CUDA_VISIBLE_DEVICES=%s, selected=%s", count, devices_env, + best_device, ) - return True, count, "cuda" + return True, count, best_device logger.info("No CUDA GPU available, using CPU") return False, 0, "cpu" except ImportError: @@ -59,6 +71,49 @@ def detect_gpu() -> tuple[bool, int, str]: return False, 0, "cpu" +def _make_api_loader(model_name: str): + """Return a callable that builds API embedding (avoids lambda for ruff E731).""" + + def _load() -> BaseEmbedding: + return _build_api_embedding(model_name) + + return _load + + +def _make_local_loader(model_name: str): + """Return a callable that builds local embedding (avoids lambda for ruff E731).""" + + def _load() -> BaseEmbedding: + return _build_local_embedding(model_name) + + return _load + + +def _pick_best_gpu(device_count: int) -> str: + """Select the CUDA device with the most free memory.""" + if device_count <= 1: + return "cuda:0" + try: + import torch + + best_idx = 0 + best_free = 0 + for idx in range(device_count): + free, _total = torch.cuda.mem_get_info(idx) + if free > best_free: + best_free = free + best_idx = idx + logger.info( + "GPU selection: device cuda:%d has %.1f GiB free (best of %d)", + best_idx, + best_free / (1024**3), + device_count, + ) + return f"cuda:{best_idx}" + except Exception: + return "cuda:0" + + def get_embedding_model( *, provider: str | None = None, @@ -71,37 +126,55 @@ def get_embedding_model( - "local": HuggingFaceEmbedding with GPU auto-detection - "api": OpenAIEmbedding (works with any OpenAI-compatible endpoint) - "mock": Deterministic mock for tests + + Local models are managed by :class:`GPUModelManager` which provides + TTL-based auto-unloading. """ - global _cached_embed_model - if _cached_embed_model is not None and not force_reload: - return _cached_embed_model + from app.services.gpu_model_manager import gpu_model_manager prov = provider or getattr(settings, "embedding_provider", "local") name = model_name or settings.embedding_model if prov == "mock": - model = _build_mock_embedding() + loader = _build_mock_embedding + device = "cpu" elif prov == "api": - model = _build_api_embedding(name) + loader = _make_api_loader(name) + device = "cpu" else: - model = _build_local_embedding(name) + _, _, device = detect_gpu(pinned_gpu_id=settings.embed_gpu_id) + loader = _make_local_loader(name) + + return gpu_model_manager.acquire( + "embedding", + loader, + model_name=name, + device=device, + force_reload=force_reload, + ) + + +def _cleanup_gpu_memory() -> None: + """Force garbage collection and release cached GPU memory.""" + from app.services.gpu_utils import release_gpu_memory - _cached_embed_model = model - return model + release_gpu_memory(caller="embedding_service") def _build_local_embedding(model_name: str) -> BaseEmbedding: from llama_index.embeddings.huggingface import HuggingFaceEmbedding _inject_hf_env() + _cleanup_gpu_memory() - has_gpu, _count, device = detect_gpu() - logger.info("Loading local embedding model=%s device=%s", model_name, device) + has_gpu, _count, device = detect_gpu(pinned_gpu_id=settings.embed_gpu_id) + batch_size = settings.embed_batch_size + logger.info("Loading local embedding model=%s device=%s batch_size=%d", model_name, device, batch_size) return HuggingFaceEmbedding( model_name=model_name, device=device, - embed_batch_size=32 if has_gpu else 8, + embed_batch_size=batch_size, ) diff --git a/backend/app/services/gpu_model_manager.py b/backend/app/services/gpu_model_manager.py new file mode 100644 index 0000000..416fba2 --- /dev/null +++ b/backend/app/services/gpu_model_manager.py @@ -0,0 +1,177 @@ +"""GPU model lifecycle manager with TTL-based auto-unloading. + +Uses threading locks so that ``acquire`` / ``release`` work from both sync +and async code. Only the background TTL sweep needs the event loop. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +from app.config import settings +from app.services.gpu_utils import release_gpu_memory + +logger = logging.getLogger(__name__) + + +@dataclass +class _ModelEntry: + model: Any + last_used_at: float = field(default_factory=time.monotonic) + model_name: str = "" + device: str = "" + + +class GPUModelManager: + """Manages GPU model lifecycle with TTL-based auto-unloading. + + Models are loaded on-demand via ``acquire()`` and automatically unloaded + after ``model_ttl_seconds`` of inactivity. Set ``model_ttl_seconds=0`` + to disable auto-unloading (models persist for the process lifetime). + """ + + def __init__( + self, + ttl_seconds: int | None = None, + check_interval: int | None = None, + ): + self._ttl = ttl_seconds if ttl_seconds is not None else settings.model_ttl_seconds + self._interval = check_interval if check_interval is not None else settings.model_ttl_check_interval + self._models: dict[str, _ModelEntry] = {} + self._locks: dict[str, threading.Lock] = {} + self._global_lock = threading.Lock() + self._cleanup_task: asyncio.Task[None] | None = None + + # -- lifecycle -------------------------------------------------------- + + async def start(self) -> None: + """Start the background TTL cleanup loop (requires a running event loop).""" + if self._ttl > 0 and self._cleanup_task is None: + self._cleanup_task = asyncio.create_task(self._cleanup_loop()) + logger.info("GPU model manager started (TTL=%ds, interval=%ds)", self._ttl, self._interval) + + async def stop(self) -> None: + """Cancel the cleanup loop and unload all models.""" + if self._cleanup_task is not None: + self._cleanup_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._cleanup_task + self._cleanup_task = None + self.unload_all() + logger.info("GPU model manager stopped") + + # -- model access (sync-safe) ----------------------------------------- + + def _get_lock(self, name: str) -> threading.Lock: + with self._global_lock: + if name not in self._locks: + self._locks[name] = threading.Lock() + return self._locks[name] + + def acquire( + self, + name: str, + loader_fn: Callable[[], Any], + *, + model_name: str = "", + device: str = "", + force_reload: bool = False, + ) -> Any: + """Return a cached model or load it on demand (thread-safe, sync). + + Concurrent callers for the same *name* block on a shared lock so + the loader runs at most once. + """ + lock = self._get_lock(name) + with lock: + entry = self._models.get(name) + + if entry is not None and not force_reload: + entry.last_used_at = time.monotonic() + return entry.model + + if entry is not None: + self._do_unload(name, entry) + + model = loader_fn() + self._models[name] = _ModelEntry( + model=model, + model_name=model_name, + device=device, + ) + logger.info("Loaded GPU model %r (model=%s, device=%s)", name, model_name, device) + return model + + def touch(self, name: str) -> None: + """Update the last-used timestamp for a loaded model.""" + entry = self._models.get(name) + if entry is not None: + entry.last_used_at = time.monotonic() + + def unload(self, name: str) -> None: + """Unload a single model by name.""" + lock = self._get_lock(name) + with lock: + entry = self._models.pop(name, None) + if entry is not None: + self._do_unload(name, entry) + + def unload_all(self) -> None: + """Unload all managed models.""" + names = list(self._models.keys()) + for name in names: + self.unload(name) + + def is_loaded(self, name: str) -> bool: + return name in self._models + + # -- internals -------------------------------------------------------- + + def _do_unload(self, name: str, entry: _ModelEntry) -> None: + logger.info("Unloading GPU model %r", name) + del entry.model + release_gpu_memory(caller=f"gpu_model_manager:{name}") + + async def _cleanup_loop(self) -> None: + """Periodically check for idle models and unload them.""" + while True: + await asyncio.sleep(self._interval) + now = time.monotonic() + expired = [name for name, entry in self._models.items() if (now - entry.last_used_at) > self._ttl] + for name in expired: + logger.info("TTL expired for model %r, unloading", name) + self.unload(name) + + # -- status ----------------------------------------------------------- + + def get_status(self) -> list[dict[str, Any]]: + """Return status information for all managed models.""" + now = time.monotonic() + result = [] + for name, entry in self._models.items(): + idle = now - entry.last_used_at + result.append( + { + "name": name, + "model_name": entry.model_name, + "loaded": True, + "device": entry.device, + "idle_seconds": round(idle, 1), + "ttl_remaining_seconds": max(0, round(self._ttl - idle, 1)) if self._ttl > 0 else None, + } + ) + return result + + @property + def loaded_model_names(self) -> list[str]: + return list(self._models.keys()) + + +gpu_model_manager = GPUModelManager() diff --git a/backend/app/services/gpu_utils.py b/backend/app/services/gpu_utils.py new file mode 100644 index 0000000..3f286c6 --- /dev/null +++ b/backend/app/services/gpu_utils.py @@ -0,0 +1,19 @@ +"""GPU memory utilities — shared cleanup logic for embedding, OCR, and model manager.""" + +import gc +import logging + +logger = logging.getLogger(__name__) + + +def release_gpu_memory(caller: str = "") -> None: + """Force garbage collection and release cached GPU memory.""" + gc.collect() + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + logger.info("%s: released GPU memory", caller or "gpu_utils") + except ImportError: + pass diff --git a/backend/app/services/keyword_service.py b/backend/app/services/keyword_service.py index 20e8089..c60469a 100644 --- a/backend/app/services/keyword_service.py +++ b/backend/app/services/keyword_service.py @@ -6,7 +6,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.models import Keyword -from app.services.llm_client import LLMClient +from app.prompts.keyword import KEYWORD_EXPAND_SYSTEM +from app.services.llm.client import LLMClient logger = logging.getLogger(__name__) @@ -62,7 +63,7 @@ async def expand_keywords_with_llm( try: result = await self.llm.chat_json( messages=[ - {"role": "system", "content": "You are a scientific terminology expert. Return valid JSON only."}, + {"role": "system", "content": KEYWORD_EXPAND_SYSTEM}, {"role": "user", "content": prompt}, ], task_type="keyword_expand", diff --git a/backend/app/services/llm_client.py b/backend/app/services/llm_client.py deleted file mode 100644 index 37c9c9c..0000000 --- a/backend/app/services/llm_client.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Backward-compatibility shim — imports redirect to app.services.llm.""" - -from app.services.llm.client import LLMClient, get_llm_client - -__all__ = ["LLMClient", "get_llm_client"] diff --git a/backend/app/services/mineru_process_manager.py b/backend/app/services/mineru_process_manager.py new file mode 100644 index 0000000..979c429 --- /dev/null +++ b/backend/app/services/mineru_process_manager.py @@ -0,0 +1,331 @@ +"""MinerU subprocess lifecycle manager with TTL-based auto-stop.""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import shutil +import signal +import subprocess +import time +from typing import Any + +import httpx + +from app.config import settings + +logger = logging.getLogger(__name__) + + +class MinerUProcessManager: + """Manages a MinerU FastAPI subprocess, starting it on demand and + stopping it after a configurable idle period. + + When ``mineru_auto_manage`` is ``False`` the manager is a no-op — + callers should use the existing ``MinerUClient`` / health-check flow. + """ + + def __init__(self) -> None: + self._process: subprocess.Popen[bytes] | None = None + self._lock = asyncio.Lock() + self._last_used_at: float = 0.0 + self._cleanup_task: asyncio.Task[None] | None = None + self._is_external: bool = False + + # -- lifecycle -------------------------------------------------------- + + async def start(self) -> None: + """Start the background TTL watcher (does NOT start MinerU yet).""" + if not settings.mineru_auto_manage: + logger.info("MinerU auto-manage disabled") + return + ttl = settings.mineru_ttl_seconds + if ttl > 0 and self._cleanup_task is None: + self._cleanup_task = asyncio.create_task(self._cleanup_loop()) + logger.info("MinerU process manager started (TTL=%ds)", ttl) + + async def stop(self) -> None: + """Cancel the watcher and kill all MinerU processes (owned + external).""" + if self._cleanup_task is not None: + self._cleanup_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._cleanup_task + self._cleanup_task = None + await self._kill_process() + self.kill_external_by_port() + logger.info("MinerU process manager stopped") + + def stop_sync(self) -> None: + """Synchronous cleanup for atexit — kill subprocess and external MinerU.""" + if self._process is not None and self._process.poll() is None: + pid = self._process.pid + try: + self._process.send_signal(signal.SIGTERM) + self._process.wait(timeout=5) + except (subprocess.TimeoutExpired, OSError, ProcessLookupError): + try: + self._process.kill() + self._process.wait(timeout=3) + except (OSError, ProcessLookupError): + pass + logger.info("Sync cleanup: stopped MinerU subprocess pid=%d", pid) + self._process = None + self.kill_external_by_port() + + # -- public API ------------------------------------------------------- + + async def ensure_running(self) -> bool: + """Make sure MinerU is reachable. Returns ``True`` on success. + + 1. If an external process already serves the port → use it. + 2. Otherwise start a subprocess via ``conda run``. + 3. Poll ``/docs`` until healthy or timeout. + """ + if not settings.mineru_auto_manage: + return False + + async with self._lock: + if await self._health_check(): + self._touch() + if self._process is None: + self._is_external = True + return True + + self._is_external = False + if not self._start_subprocess(): + return False + + ok = await self._wait_healthy(settings.mineru_startup_timeout) + if ok: + self._touch() + else: + logger.warning("MinerU failed to become healthy within %ds", settings.mineru_startup_timeout) + await self._kill_process() + return ok + + def touch(self) -> None: + """Update idle timer (call after every MinerU request).""" + self._touch() + + async def shutdown_mineru(self) -> None: + """Immediately stop the managed subprocess.""" + async with self._lock: + await self._kill_process() + + def get_status(self) -> dict[str, Any]: + now = time.monotonic() + if self._process is not None and self._process.poll() is None: + idle = now - self._last_used_at if self._last_used_at else 0 + ttl = settings.mineru_ttl_seconds + return { + "status": "running", + "pid": self._process.pid, + "port": self._port, + "idle_seconds": round(idle, 1), + "ttl_remaining_seconds": max(0, round(ttl - idle, 1)) if ttl > 0 else None, + } + if self._is_external: + return {"status": "external", "pid": None, "port": self._port} + return {"status": "stopped", "pid": None, "port": self._port} + + # -- internals -------------------------------------------------------- + + @property + def _port(self) -> int: + url = settings.mineru_api_url.rstrip("/") + try: + return int(url.rsplit(":", 1)[-1]) + except (ValueError, IndexError): + return 8010 + + @property + def _host(self) -> str: + url = settings.mineru_api_url.rstrip("/") + return url.rsplit(":", 1)[0].split("//")[-1] if "//" in url else "0.0.0.0" + + def _touch(self) -> None: + self._last_used_at = time.monotonic() + + async def _health_check(self) -> bool: + try: + async with httpx.AsyncClient(timeout=5) as client: + resp = await client.get(f"{settings.mineru_api_url.rstrip('/')}/docs") + return resp.status_code == 200 + except Exception: + return False + + def _start_subprocess(self) -> bool: + conda_path = shutil.which("conda") + if not conda_path: + logger.warning("conda not found on PATH, cannot auto-start MinerU") + return False + + gpu_ids = settings.mineru_gpu_ids or settings.cuda_visible_devices + env_name = settings.mineru_conda_env + + cmd = [ + conda_path, + "run", + "-n", + env_name, + "python", + "-m", + "mineru.cli.fast_api", + "--host", + self._host, + "--port", + str(self._port), + ] + + import os + + env = os.environ.copy() + if gpu_ids: + env["CUDA_VISIBLE_DEVICES"] = gpu_ids + + try: + self._process = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + logger.info( + "Started MinerU subprocess pid=%d (env=%s, gpu=%s, port=%d)", + self._process.pid, + env_name, + gpu_ids, + self._port, + ) + return True + except (OSError, FileNotFoundError) as exc: + logger.warning("Failed to start MinerU subprocess: %s", exc) + return False + + async def _wait_healthy(self, timeout: int) -> bool: + deadline = time.monotonic() + timeout + interval = 2.0 + while time.monotonic() < deadline: + if self._process is not None and self._process.poll() is not None: + stderr_data = await asyncio.to_thread(self._process.stderr.read) + stderr = (stderr_data or b"").decode(errors="replace")[:500] + logger.warning("MinerU process exited early (code=%s): %s", self._process.returncode, stderr) + return False + if await self._health_check(): + return True + await asyncio.sleep(interval) + interval = min(interval * 1.5, 10.0) + return False + + async def _kill_process(self) -> None: + if self._process is None: + return + if self._process.poll() is not None: + self._process = None + return + + pid = self._process.pid + logger.info("Stopping MinerU subprocess pid=%d", pid) + try: + self._process.send_signal(signal.SIGTERM) + try: + await asyncio.to_thread(self._process.wait, 10) + except subprocess.TimeoutExpired: + logger.warning("MinerU pid=%d did not exit after SIGTERM, sending SIGKILL", pid) + self._process.kill() + await asyncio.to_thread(self._process.wait, 5) + except (OSError, ProcessLookupError): + pass + finally: + self._process = None + + def kill_external_by_port(self) -> None: + """Find and kill the process listening on the MinerU port (sync).""" + import os + + port = self._port + my_pid = os.getpid() + target_pid = self._find_pid_by_port(port) + if target_pid is None or target_pid == my_pid: + return + if not self._is_mineru_process(target_pid): + logger.info("Port %d held by non-MinerU process (pid=%d), skipping", port, target_pid) + return + try: + os.kill(target_pid, signal.SIGTERM) + logger.info("Sent SIGTERM to external MinerU pid=%d (port=%d)", target_pid, port) + except (OSError, ProcessLookupError) as exc: + logger.warning("Failed to kill external MinerU pid=%d: %s", target_pid, exc) + + @staticmethod + def _find_pid_by_port(port: int) -> int | None: + """Find PID listening on a TCP port using /proc or lsof.""" + import os + + try: + with open("/proc/net/tcp") as f: + hex_port = f":{port:04X}" + for line in f: + parts = line.strip().split() + if len(parts) >= 10 and hex_port in parts[1] and parts[3] == "0A": + inode = parts[9] + for pid_dir in os.listdir("/proc"): + if not pid_dir.isdigit(): + continue + try: + fd_dir = f"/proc/{pid_dir}/fd" + for fd in os.listdir(fd_dir): + link = os.readlink(f"{fd_dir}/{fd}") + if f"socket:[{inode}]" in link: + return int(pid_dir) + except (OSError, PermissionError): + continue + except (OSError, PermissionError): + pass + + import shutil + + lsof_path = shutil.which("lsof") + if lsof_path: + try: + result = subprocess.run( + [lsof_path, "-ti", f":{port}"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0 and result.stdout.strip(): + return int(result.stdout.strip().split("\n")[0]) + except (subprocess.TimeoutExpired, ValueError, OSError): + pass + return None + + @staticmethod + def _is_mineru_process(pid: int) -> bool: + """Check if a PID is a MinerU process by reading its cmdline.""" + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + cmdline = f.read().decode(errors="replace").lower() + return "mineru" in cmdline + except (OSError, PermissionError): + return False + + async def _cleanup_loop(self) -> None: + ttl = settings.mineru_ttl_seconds + interval = max(ttl // 4, 30) + while True: + await asyncio.sleep(interval) + if self._process is None or self._is_external: + continue + if self._process.poll() is not None: + logger.info("MinerU subprocess exited unexpectedly") + self._process = None + continue + idle = time.monotonic() - self._last_used_at + if self._last_used_at > 0 and idle > ttl: + logger.info("MinerU idle for %.0fs (TTL=%ds), stopping", idle, ttl) + await self._kill_process() + + +mineru_process_manager = MinerUProcessManager() diff --git a/backend/app/services/ocr_service.py b/backend/app/services/ocr_service.py index 5f28ec4..eb6aaf1 100644 --- a/backend/app/services/ocr_service.py +++ b/backend/app/services/ocr_service.py @@ -10,11 +10,13 @@ import json import logging import re +import tempfile from pathlib import Path import pdfplumber from app.config import settings +from app.services.gpu_utils import release_gpu_memory logger = logging.getLogger(__name__) @@ -31,6 +33,25 @@ def __init__(self, use_gpu: bool = True, gpu_id: int = 0): self.output_dir = Path(settings.ocr_output_dir) self.output_dir.mkdir(parents=True, exist_ok=True) + def close(self) -> None: + """Release PaddleOCR model and free GPU memory.""" + if self._paddle_ocr is not None: + del self._paddle_ocr + self._paddle_ocr = None + if self._marker_converter is not None: + del self._marker_converter + self._marker_converter = None + release_gpu_memory(caller="OCRService") + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __del__(self): + self.close() + def extract_text_native(self, pdf_path: str) -> list[dict]: """Extract text from native (non-scanned) PDF using pdfplumber.""" pages = [] @@ -145,17 +166,16 @@ def extract_text_ocr(self, pdf_path: str) -> list[dict]: else: import fitz - pdf_doc = fitz.open(pdf_path) result = [] - for page_num in range(len(pdf_doc)): - page = pdf_doc[page_num] - pix = page.get_pixmap(dpi=150) - img_path = f"/tmp/omelette_ocr_page_{page_num}.png" - pix.save(img_path) - page_result = ocr.ocr(img_path, cls=False) - result.append(page_result[0] if page_result else []) - Path(img_path).unlink(missing_ok=True) - pdf_doc.close() + with fitz.open(pdf_path) as pdf_doc: + for page_num in range(len(pdf_doc)): + page = pdf_doc[page_num] + pix = page.get_pixmap(dpi=150) + img_path = str(Path(tempfile.gettempdir()) / f"omelette_ocr_page_{page_num}.png") + pix.save(img_path) + page_result = ocr.ocr(img_path, cls=False) + result.append(page_result[0] if page_result else []) + Path(img_path).unlink(missing_ok=True) for i, page_result in enumerate(result): text_lines = [] @@ -188,15 +208,26 @@ async def _extract_with_mineru(self, pdf_path: str) -> dict | None: return None from app.services.mineru_client import MinerUClient + from app.services.mineru_process_manager import mineru_process_manager + + if settings.mineru_auto_manage: + ok = await mineru_process_manager.ensure_running() + if not ok: + logger.info("MinerU auto-start failed, falling back to pdfplumber") + return None if self._mineru_client is None: self._mineru_client = MinerUClient() - if not await self._mineru_client.health_check(): + if not settings.mineru_auto_manage and not await self._mineru_client.health_check(): logger.info("MinerU service not available, skipping") return None result = await self._mineru_client.parse_pdf(pdf_path) + + if settings.mineru_auto_manage: + mineru_process_manager.touch() + if result.get("error"): logger.warning("MinerU failed for %s: %s", pdf_path, result["error"]) return None diff --git a/backend/app/services/paper_processor.py b/backend/app/services/paper_processor.py index 1f7a4ee..ff7c303 100644 --- a/backend/app/services/paper_processor.py +++ b/backend/app/services/paper_processor.py @@ -2,15 +2,24 @@ Designed to run as a fire-and-forget ``asyncio.create_task`` so the upload API can return immediately while processing continues in the background. + +GPU parallelisation: + - Multiple PDFs are OCR-ed concurrently via ``asyncio.gather``. + - Each worker gets a distinct ``gpu_id`` (round-robin) so that all + visible GPUs are utilised. + - DB writes and RAG indexing remain serial (ChromaDB limitation). """ from __future__ import annotations +import asyncio import logging +import time from sqlalchemy import select from sqlalchemy.orm import selectinload +from app.config import GpuMode, settings from app.database import async_session_factory from app.models import Paper, PaperStatus from app.models.chunk import PaperChunk @@ -20,6 +29,47 @@ logger = logging.getLogger(__name__) +def _detect_gpu_count() -> int: + """Return the number of CUDA devices visible to this process (0 = CPU-only).""" + try: + import torch + + if torch.cuda.is_available(): + return torch.cuda.device_count() + except ImportError: + pass + return 0 + + +def _parse_ocr_gpu_ids(gpu_count: int) -> list[int]: + """Parse OCR_GPU_IDS into a list of valid indices. + + Empty string → all GPUs ``[0 .. gpu_count-1]``. + """ + raw = settings.ocr_gpu_ids.strip() + if not raw or gpu_count == 0: + return list(range(max(gpu_count, 1))) + ids = [] + for tok in raw.split(","): + tok = tok.strip() + if tok.isdigit(): + idx = int(tok) + if idx < gpu_count: + ids.append(idx) + return ids or list(range(gpu_count)) + + +def _resolve_parallel_limit(gpu_count: int) -> int: + """Determine how many OCR tasks may run concurrently.""" + configured = settings.ocr_parallel_limit + if configured > 0: + return configured + base = max(gpu_count, 1) + if settings.gpu_mode == GpuMode.AGGRESSIVE: + return base * 2 + return base + + async def process_papers_background( project_id: int, paper_ids: list[int], @@ -32,7 +82,16 @@ async def process_papers_background( async def _process_papers(project_id: int, paper_ids: list[int]) -> None: - ocr = OCRService(use_gpu=True) + gpu_count = _detect_gpu_count() + parallel_limit = _resolve_parallel_limit(gpu_count) + use_gpu = gpu_count > 0 + + logger.info( + "Paper processing: %d papers, %d GPU(s), parallel_limit=%d", + len(paper_ids), + gpu_count, + parallel_limit, + ) async with async_session_factory() as db: stmt = select(Paper).where( @@ -42,31 +101,62 @@ async def _process_papers(project_id: int, paper_ids: list[int]) -> None: papers = list((await db.execute(stmt)).scalars().all()) ocr_done_ids: list[int] = [] + papers_to_ocr: list[Paper] = [] for paper in papers: if paper.status not in (PaperStatus.PDF_DOWNLOADED, PaperStatus.ERROR): if paper.status in (PaperStatus.OCR_COMPLETE, PaperStatus.INDEXED): ocr_done_ids.append(paper.id) continue - if not paper.pdf_path: paper.status = PaperStatus.ERROR continue - - try: - result = await ocr.process_pdf_async(paper.pdf_path) + papers_to_ocr.append(paper) + + if papers_to_ocr: + semaphore = asyncio.Semaphore(parallel_limit) + ocr_gpus = _parse_ocr_gpu_ids(gpu_count) + + async def _ocr_one(paper: Paper, worker_id: int) -> tuple[Paper, dict | None]: + gpu_id = ocr_gpus[worker_id % len(ocr_gpus)] if use_gpu else 0 + with OCRService(use_gpu=use_gpu, gpu_id=gpu_id) as ocr: + async with semaphore: + try: + t0 = time.monotonic() + result = await ocr.process_pdf_async(paper.pdf_path) + elapsed = time.monotonic() - t0 + logger.info( + "OCR worker %d (gpu=%d) finished paper %d in %.1fs", + worker_id, + gpu_id, + paper.id, + elapsed, + ) + return paper, result + except Exception: + logger.exception("OCR failed for paper %d (worker %d)", paper.id, worker_id) + return paper, None + + tasks = [_ocr_one(paper, i) for i, paper in enumerate(papers_to_ocr)] + results = await asyncio.gather(*tasks) + + for paper, result in results: + if result is None: + paper.status = PaperStatus.ERROR + continue if result.get("error"): paper.status = PaperStatus.ERROR logger.warning("OCR error for paper %d: %s", paper.id, result.get("error")) continue - ocr.save_result(paper.id, result) + OCRService(use_gpu=False).save_result(paper.id, result) if result.get("method") == "mineru": - chunks = ocr.chunk_mineru_markdown(result["md_content"]) + chunks = OCRService(use_gpu=False).chunk_mineru_markdown(result["md_content"]) else: - chunks = ocr.chunk_text(result.get("pages", [])) + chunks = OCRService(use_gpu=False).chunk_text(result.get("pages", [])) + for chunk_data in chunks: db.add( PaperChunk( @@ -84,9 +174,6 @@ async def _process_papers(project_id: int, paper_ids: list[int]) -> None: paper.status = PaperStatus.OCR_COMPLETE ocr_done_ids.append(paper.id) logger.info("OCR complete for paper %d (%s)", paper.id, paper.title[:40]) - except Exception: - paper.status = PaperStatus.ERROR - logger.exception("OCR failed for paper %d", paper.id) await db.flush() diff --git a/backend/app/services/pdf_metadata.py b/backend/app/services/pdf_metadata.py index bbeede0..c6c5675 100644 --- a/backend/app/services/pdf_metadata.py +++ b/backend/app/services/pdf_metadata.py @@ -9,6 +9,7 @@ from __future__ import annotations +import asyncio import logging import re from pathlib import Path @@ -36,7 +37,7 @@ async def extract_metadata( fallback_title: str = "Untitled", ) -> NewPaperData: """Extract metadata from *pdf_path*, optionally enriching via Crossref.""" - local = _extract_local(pdf_path, fallback_title) + local = await asyncio.to_thread(_extract_local, pdf_path, fallback_title) if local.doi: enriched = await _crossref_lookup(local.doi) diff --git a/backend/app/services/pipeline_service.py b/backend/app/services/pipeline_service.py index 4df3cc5..88412f7 100644 --- a/backend/app/services/pipeline_service.py +++ b/backend/app/services/pipeline_service.py @@ -77,25 +77,38 @@ async def _download(self, paper: Paper) -> dict: async def _ocr(self, paper: Paper) -> dict: try: - ocr = OCRService(use_gpu=True) - result = ocr.process_pdf(paper.pdf_path) + with OCRService(use_gpu=True) as ocr: + result = await ocr.process_pdf_async(paper.pdf_path) if result.get("error"): paper.status = PaperStatus.ERROR return {"success": False, "reason": result["error"]} - pages = result.get("pages", []) chunks = [] - for page in pages: - if page.get("text", "").strip(): - chunks.append( - { - "paper_id": paper.id, - "content": page["text"], - "page_number": page.get("page_number", 0), - "chunk_index": len(chunks), - } - ) + if result.get("method") == "mineru": + mineru_chunks = ocr.chunk_mineru_markdown(result["md_content"]) + for i, c in enumerate(mineru_chunks): + if c.get("content", "").strip(): + chunks.append( + { + "paper_id": paper.id, + "content": c["content"], + "page_number": c.get("page_number", 1), + "chunk_index": i, + } + ) + else: + pages = result.get("pages", []) + for page in pages: + if page.get("text", "").strip(): + chunks.append( + { + "paper_id": paper.id, + "content": page["text"], + "page_number": page.get("page_number", 0), + "chunk_index": len(chunks), + } + ) for chunk_data in chunks: chunk = PaperChunk(**chunk_data) diff --git a/backend/app/services/rag_service.py b/backend/app/services/rag_service.py index a1af81b..be6b43d 100644 --- a/backend/app/services/rag_service.py +++ b/backend/app/services/rag_service.py @@ -10,7 +10,9 @@ from __future__ import annotations +import asyncio import logging +import time from collections.abc import Callable from pathlib import Path from typing import TYPE_CHECKING @@ -23,7 +25,8 @@ from llama_index.core.schema import Document, NodeRelationship, RelatedNodeInfo, TextNode from app.config import settings -from app.services.llm_client import LLMClient +from app.prompts.rag import RAG_ANSWER_SYSTEM +from app.services.llm.client import LLMClient if TYPE_CHECKING: from llama_index.core.embeddings import BaseEmbedding @@ -34,6 +37,8 @@ class RAGService: """LlamaIndex-powered RAG service with ChromaDB vector store.""" + _COUNT_CACHE_TTL = 60.0 + def __init__( self, llm: LLMClient | None = None, @@ -44,6 +49,21 @@ def __init__( self.llm = llm self._chroma_client = chroma_client self._embed_model = embed_model + self._count_cache: dict[int, tuple[int, float]] = {} + + async def _get_count(self, project_id: int) -> int: + """Get collection count with caching and async wrapping.""" + now = time.monotonic() + cached = self._count_cache.get(project_id) + if cached and now - cached[1] < self._COUNT_CACHE_TTL: + return cached[0] + collection = self._get_collection(project_id) + count = await asyncio.to_thread(collection.count) + self._count_cache[project_id] = (count, now) + return count + + def _invalidate_count(self, project_id: int) -> None: + self._count_cache.pop(project_id, None) def _get_chroma_client(self) -> chromadb.ClientAPI: if self._chroma_client is None: @@ -58,7 +78,12 @@ def _get_chroma_client(self) -> chromadb.ClientAPI: def _get_collection(self, project_id: int) -> chromadb.Collection: return self._get_chroma_client().get_or_create_collection( name=f"project_{project_id}", - metadata={"hnsw:space": "cosine"}, + metadata={ + "hnsw:space": "cosine", + "hnsw:construction_ef": 200, + "hnsw:search_ef": 100, + "hnsw:M": 32, + }, ) def _ensure_embed_model(self) -> BaseEmbedding: @@ -69,6 +94,15 @@ def _ensure_embed_model(self) -> BaseEmbedding: LlamaSettings.embed_model = self._embed_model return self._embed_model + def _reload_embed_model(self) -> BaseEmbedding: + """Force-reload the embedding model onto the best available GPU.""" + from app.services.embedding_service import _cleanup_gpu_memory, get_embedding_model + + _cleanup_gpu_memory() + self._embed_model = get_embedding_model(force_reload=True) + LlamaSettings.embed_model = self._embed_model + return self._embed_model + def _get_vector_store(self, project_id: int): from llama_index.vector_stores.chroma import ChromaVectorStore @@ -104,6 +138,9 @@ async def index_chunks( if on_progress: on_progress("loading_model", 0) + from app.services.embedding_service import _cleanup_gpu_memory + + _cleanup_gpu_memory() index = self._get_index(project_id) if on_progress: @@ -132,8 +169,6 @@ async def index_chunks( node.relationships[NodeRelationship.SOURCE] = RelatedNodeInfo(node_id=ref_doc_id) nodes.append(node) - import asyncio - total = len(nodes) indexed = 0 for i in range(0, total, batch_size): @@ -144,6 +179,7 @@ async def index_chunks( pct = 10 + int(90 * indexed / total) on_progress("indexing", min(pct, 99)) + self._invalidate_count(project_id) return {"indexed": total, "collection": f"project_{project_id}"} async def index_documents( @@ -161,10 +197,9 @@ async def index_documents( splitter = SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap) nodes = splitter.get_nodes_from_documents(documents) - import asyncio - index = self._get_index(project_id) await asyncio.to_thread(index.insert_nodes, nodes) + self._invalidate_count(project_id) return {"indexed": len(nodes), "collection": f"project_{project_id}"} @staticmethod @@ -184,7 +219,7 @@ def _get_adjacent_chunks( chunk_index: int, window: int = 1, ) -> tuple[str, str]: - """Fetch adjacent chunks for context expansion. + """Fetch adjacent chunks for context expansion (single node). Returns ``(prev_text, next_text)`` so the caller can assemble ``[prev] \\n [main] \\n [next]`` in the correct order. @@ -204,9 +239,60 @@ def _get_adjacent_chunks( docs = result.get("documents") or [] next_text = "\n".join(d for d in docs if d) except Exception: - pass + logger.debug("Adjacent chunk fetch failed for paper %d chunk %d", paper_id, chunk_index, exc_info=True) return prev_text, next_text + async def _get_adjacent_chunks_batch( + self, + collection: chromadb.Collection, + nodes: list, + ) -> list[tuple[str, str]]: + """Batch-fetch adjacent chunks for all nodes in a single ChromaDB call. + + Returns a list of (prev_text, next_text) tuples aligned with *nodes*. + """ + all_ids: set[str] = set() + node_adj: list[tuple[str | None, str | None]] = [] + + for n in nodes: + node = n.node if hasattr(n, "node") else n + meta = node.metadata or {} + pid = meta.get("paper_id") + cidx = meta.get("chunk_index") + if pid is None or cidx is None: + node_adj.append((None, None)) + continue + prev_id = f"paper_{pid}_chunk_{cidx - 1}" + next_id = f"paper_{pid}_chunk_{cidx + 1}" + all_ids.update([prev_id, next_id]) + node_adj.append((prev_id, next_id)) + + id_to_doc: dict[str, str] = {} + if all_ids: + try: + result = await asyncio.to_thread(collection.get, ids=list(all_ids), include=["documents"]) + for doc_id, doc in zip(result.get("ids") or [], result.get("documents") or []): + if doc: + id_to_doc[doc_id] = doc + except Exception: + logger.debug("Batch adjacent chunk fetch failed", exc_info=True) + + return [ + (id_to_doc.get(prev_id, ""), id_to_doc.get(next_id, "")) if prev_id is not None else ("", "") + for prev_id, next_id in node_adj + ] + + def _build_retriever(self, index: VectorStoreIndex, fetch_k: int, count: int): + """Build a retriever, optionally with MMR mode.""" + effective_k = min(fetch_k, count) + if settings.rag_mmr_threshold > 0: + return index.as_retriever( + similarity_top_k=effective_k, + vector_store_query_mode="mmr", + vector_store_kwargs={"mmr_threshold": settings.rag_mmr_threshold}, + ) + return index.as_retriever(similarity_top_k=effective_k) + async def query( self, project_id: int, @@ -216,42 +302,40 @@ async def query( include_sources: bool = True, ) -> dict: """Query the knowledge base and generate an answer with citations.""" - collection = self._get_collection(project_id) - if collection.count() == 0: + count = await self._get_count(project_id) + if count == 0: return { "answer": "No documents have been indexed yet. Please process and index papers first.", "sources": [], "confidence": 0.0, } - import asyncio - + collection = self._get_collection(project_id) index = self._get_index(project_id) - retriever = index.as_retriever(similarity_top_k=min(top_k, collection.count())) + + oversample = settings.rag_oversample_factor if use_reranker else 1 + fetch_k = top_k * oversample + retriever = self._build_retriever(index, fetch_k, count) retrieved_nodes = await asyncio.to_thread(retriever.retrieve, question) if not retrieved_nodes: return {"answer": "No relevant documents found.", "sources": [], "confidence": 0.0} + if use_reranker and retrieved_nodes: + from app.services.reranker_service import rerank_nodes + + retrieved_nodes = await rerank_nodes(retrieved_nodes, question, top_n=top_k) + + adj_results = await self._get_adjacent_chunks_batch(collection, retrieved_nodes) + contexts = [] sources = [] - for node_with_score in retrieved_nodes: + for node_with_score, (prev_text, next_text) in zip(retrieved_nodes, adj_results): node = node_with_score.node meta = node.metadata or {} score = node_with_score.score or 0.0 text = node.get_content() - paper_id = meta.get("paper_id") - chunk_idx = meta.get("chunk_index") - prev_text, next_text = "", "" - if paper_id is not None and chunk_idx is not None: - prev_text, next_text = await asyncio.to_thread( - self._get_adjacent_chunks, - collection, - paper_id, - chunk_idx, - ) - parts = [p for p in [prev_text, text, next_text] if p] full_context = "\n".join(parts) @@ -260,7 +344,7 @@ async def query( ) sources.append( { - "paper_id": paper_id, + "paper_id": meta.get("paper_id"), "paper_title": meta.get("paper_title", ""), "page_number": meta.get("page_number"), "chunk_type": meta.get("chunk_type", "text"), @@ -289,45 +373,45 @@ async def retrieve_only( project_id: int, question: str, top_k: int = 10, + use_reranker: bool = False, ) -> list[dict]: """Retrieve relevant chunks without LLM generation. Designed for the Chat Pipeline where the LLM call happens downstream in the generate node, avoiding a redundant call here. """ - collection = self._get_collection(project_id) - if collection.count() == 0: + count = await self._get_count(project_id) + if count == 0: return [] - import asyncio - + collection = self._get_collection(project_id) index = self._get_index(project_id) - retriever = index.as_retriever(similarity_top_k=min(top_k, collection.count())) + + oversample = settings.rag_oversample_factor if use_reranker else 1 + fetch_k = top_k * oversample + retriever = self._build_retriever(index, fetch_k, count) retrieved_nodes = await asyncio.to_thread(retriever.retrieve, question) + if use_reranker and retrieved_nodes: + from app.services.reranker_service import rerank_nodes + + retrieved_nodes = await rerank_nodes(retrieved_nodes, question, top_n=top_k) + + adj_results = await self._get_adjacent_chunks_batch(collection, retrieved_nodes) + sources: list[dict] = [] - for node_with_score in retrieved_nodes: + for node_with_score, (prev_text, next_text) in zip(retrieved_nodes, adj_results): node = node_with_score.node meta = node.metadata or {} score = node_with_score.score or 0.0 text = node.get_content() - paper_id = meta.get("paper_id") - chunk_idx = meta.get("chunk_index") - prev_text, next_text = "", "" - if paper_id is not None and chunk_idx is not None: - prev_text, next_text = await asyncio.to_thread( - self._get_adjacent_chunks, - collection, - paper_id, - chunk_idx, - ) parts = [p for p in [prev_text, text, next_text] if p] full_context = "\n".join(parts) sources.append( { - "paper_id": paper_id, + "paper_id": meta.get("paper_id"), "paper_title": meta.get("paper_title", ""), "page_number": meta.get("page_number"), "chunk_type": meta.get("chunk_type", "text"), @@ -349,14 +433,7 @@ async def _generate_answer(self, question: str, context: str) -> str: ) return await self.llm.chat( messages=[ - { - "role": "system", - "content": ( - "You are a scientific research assistant. " - "Answer questions based strictly on the provided context. " - "Cite sources accurately." - ), - }, + {"role": "system", "content": RAG_ANSWER_SYSTEM}, {"role": "user", "content": prompt}, ], temperature=0.3, @@ -368,7 +445,8 @@ async def delete_index(self, project_id: int) -> dict: client = self._get_chroma_client() name = f"project_{project_id}" try: - client.delete_collection(name) + await asyncio.to_thread(client.delete_collection, name) + self._invalidate_count(project_id) return {"deleted": True, "collection": name} except ValueError: return {"deleted": False, "message": "Collection not found"} @@ -377,7 +455,8 @@ async def delete_paper(self, project_id: int, paper_id: int) -> dict: """Delete all chunks for a single paper from the index.""" collection = self._get_collection(project_id) try: - collection.delete(where={"paper_id": paper_id}) + await asyncio.to_thread(collection.delete, where={"paper_id": paper_id}) + self._invalidate_count(project_id) return {"deleted": True, "paper_id": paper_id} except Exception as e: logger.warning("Failed to delete paper %d from index: %s", paper_id, e) @@ -386,10 +465,11 @@ async def delete_paper(self, project_id: int, paper_id: int) -> dict: async def get_stats(self, project_id: int) -> dict: """Get index statistics for a project.""" try: - collection = self._get_collection(project_id) + count = await self._get_count(project_id) return { - "total_chunks": collection.count(), + "total_chunks": count, "collection_name": f"project_{project_id}", } except Exception: + logger.warning("Failed to get stats for project %d", project_id, exc_info=True) return {"total_chunks": 0, "collection_name": f"project_{project_id}"} diff --git a/backend/app/services/reranker_service.py b/backend/app/services/reranker_service.py new file mode 100644 index 0000000..47abb60 --- /dev/null +++ b/backend/app/services/reranker_service.py @@ -0,0 +1,86 @@ +"""Reranker model loading, caching, and async-safe inference.""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING + +from app.config import settings + +if TYPE_CHECKING: + from llama_index.core.schema import NodeWithScore + +logger = logging.getLogger(__name__) + +_reranker_semaphore: asyncio.Semaphore | None = None + + +def _get_semaphore() -> asyncio.Semaphore: + global _reranker_semaphore + if _reranker_semaphore is None: + _reranker_semaphore = asyncio.Semaphore(settings.reranker_concurrency_limit) + return _reranker_semaphore + + +def _build_reranker(model_name: str): + """Build a SentenceTransformerRerank instance (heavy, runs on GPU).""" + from llama_index.postprocessor.sbert_rerank import SentenceTransformerRerank + + from app.services.embedding_service import _inject_hf_env, detect_gpu + + _inject_hf_env() + + _has_gpu, _count, device = detect_gpu(pinned_gpu_id=settings.rerank_gpu_id) + batch_size = settings.rerank_batch_size + logger.info("Loading reranker model=%s device=%s top_n=%d", model_name, device, batch_size) + return SentenceTransformerRerank( + model=model_name, + top_n=batch_size, # Oversample before rerank, then return top batch_size; aligns with RAG oversample_factor + device=device, + keep_retrieval_score=True, + ) + + +def get_reranker(*, model_name: str | None = None): + """Return a cached reranker via GPUModelManager (TTL-managed).""" + from app.services.embedding_service import detect_gpu + from app.services.gpu_model_manager import gpu_model_manager + + name = model_name or settings.reranker_model + _, _, device = detect_gpu(pinned_gpu_id=settings.rerank_gpu_id) + return gpu_model_manager.acquire( + "reranker", + lambda: _build_reranker(name), + model_name=name, + device=device, + ) + + +async def rerank_nodes( + nodes: list[NodeWithScore], + query: str, + top_n: int, +) -> list[NodeWithScore]: + """Apply reranker with concurrency control and graceful fallback. + + Uses a semaphore to serialize GPU inference and falls back to + the original node order on any failure. + """ + if not nodes: + return [] + try: + from llama_index.core.schema import QueryBundle + + reranker = get_reranker() + query_bundle = QueryBundle(query_str=query) + async with _get_semaphore(): + reranked = await asyncio.to_thread( + reranker.postprocess_nodes, + nodes, + query_bundle=query_bundle, + ) + return reranked[:top_n] + except (ImportError, OSError, RuntimeError): + logger.warning("Reranking failed, returning original nodes", exc_info=True) + return nodes[:top_n] diff --git a/backend/app/services/subscription_service.py b/backend/app/services/subscription_service.py index 11ce2e5..54eb2da 100644 --- a/backend/app/services/subscription_service.py +++ b/backend/app/services/subscription_service.py @@ -1,5 +1,6 @@ """Incremental subscription service — scheduled literature updates via API and RSS.""" +import asyncio import logging from datetime import datetime @@ -20,12 +21,19 @@ def __init__(self): async def check_rss_feed(self, feed_url: str, since: datetime | None = None) -> list[dict]: """Parse an RSS/Atom feed and return new entries since the given date.""" + from app.services.url_validator import validate_url_safe + + try: + await asyncio.to_thread(validate_url_safe, feed_url) + except ValueError as e: + raise ValueError(f"Feed URL blocked (SSRF): {e}") from e + proxy = settings.http_proxy or None async with httpx.AsyncClient(proxy=proxy, timeout=30.0) as client: resp = await client.get(feed_url) resp.raise_for_status() - feed = feedparser.parse(resp.text) + feed = await asyncio.to_thread(feedparser.parse, resp.text) entries = [] for entry in feed.entries: diff --git a/backend/app/services/url_validator.py b/backend/app/services/url_validator.py new file mode 100644 index 0000000..ed73d44 --- /dev/null +++ b/backend/app/services/url_validator.py @@ -0,0 +1,58 @@ +"""URL and DOI validation utilities for SSRF prevention.""" + +import ipaddress +import re +import socket +from urllib.parse import urlparse + +BLOCKED_HOSTNAMES = frozenset( + { + "metadata.google.internal", + "metadata.amazonaws.com", + } +) + +DOI_PATTERN = re.compile(r"^10\.\d{4,9}/[-._;()/:A-Za-z0-9]+$") + + +def validate_url_safe(url: str) -> str: + """Validate URL is safe for server-side fetch. + + Blocks private IPs, loopback, link-local, reserved, multicast, + and known cloud metadata hostnames. + + Raises ValueError if the URL is unsafe. + """ + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + raise ValueError(f"Unsupported scheme: {parsed.scheme}") + + hostname = parsed.hostname + if not hostname: + raise ValueError("Invalid URL: no hostname") + + if hostname in BLOCKED_HOSTNAMES: + raise ValueError(f"Blocked hostname: {hostname}") + + try: + addrinfos = socket.getaddrinfo(hostname, None) + except socket.gaierror as e: + raise ValueError(f"DNS resolution failed for {hostname}: {e}") from e + + for info in addrinfos: + ip_str = info[4][0] + try: + ip = ipaddress.ip_address(ip_str) + except ValueError: + continue + if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast: + raise ValueError(f"Blocked: {hostname} resolves to private/reserved address {ip_str}") + + return url + + +def validate_doi(doi: str) -> str: + """Validate DOI format. Raises ValueError if invalid.""" + if not DOI_PATTERN.match(doi): + raise ValueError(f"Invalid DOI format: {doi}") + return doi diff --git a/backend/app/services/writing_service.py b/backend/app/services/writing_service.py index 4ec1ef8..962d9fe 100644 --- a/backend/app/services/writing_service.py +++ b/backend/app/services/writing_service.py @@ -1,5 +1,6 @@ """Writing assistance service — summarize, cite, outline, gap analysis, literature review.""" +import asyncio import json import logging import re @@ -8,9 +9,17 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from app.config import settings as app_settings from app.models import Paper -from app.services.llm_client import LLMClient +from app.prompts.writing import ( + WRITING_GAP_SYSTEM, + WRITING_OUTLINE_SYSTEM, + WRITING_SECTION_SYSTEM, + WRITING_SUMMARIZE_SYSTEM, +) +from app.services.llm.client import LLMClient from app.services.rag_service import RAGService +from app.utils.sse import format_sse_error logger = logging.getLogger(__name__) @@ -20,14 +29,6 @@ "thematic": "主题性综述 (thematic review)：按研究主题分组对比，突出异同", } -SECTION_SYSTEM_PROMPT = """\ -你是一位学术综述写作专家。请为以下章节撰写综述段落。 -要求： -1. 使用学术语言，逻辑清晰 -2. 在适当位置使用 [1][2] 格式引用 -3. 每个引用必须对应提供的文献，不得捏造 -4. 段落长度 200-400 字""" - class WritingService: def __init__(self, db: AsyncSession, llm: LLMClient, rag: RAGService | None = None): @@ -35,18 +36,15 @@ def __init__(self, db: AsyncSession, llm: LLMClient, rag: RAGService | None = No self.llm = llm self.rag = rag + _summarize_semaphore = asyncio.Semaphore(app_settings.llm_parallel_limit) + async def summarize_papers(self, paper_ids: list[int], language: str = "en") -> list[dict]: - """Generate summaries for selected papers.""" + """Generate summaries for selected papers (parallelized with semaphore).""" stmt = select(Paper).where(Paper.id.in_(paper_ids)) result = await self.db.execute(stmt) papers = {p.id: p for p in result.scalars().all()} - summaries = [] - for paper_id in paper_ids: - paper = papers.get(paper_id) - if not paper: - continue - + async def _summarize_one(paper: Paper) -> dict: prompt = f"""Summarize this scientific paper in {language}: Title: {paper.title} Abstract: {paper.abstract} @@ -59,27 +57,21 @@ async def summarize_papers(self, paper_ids: list[int], language: str = "en") -> 3. Innovation points 4. Limitations (if apparent from abstract)""" - summary = await self.llm.chat( - messages=[ - { - "role": "system", - "content": "You are a scientific paper analyst. Provide concise, accurate summaries.", - }, - {"role": "user", "content": prompt}, - ], - temperature=0.3, - task_type="summarize", - ) + async with self._summarize_semaphore: + summary = await self.llm.chat( + messages=[ + {"role": "system", "content": WRITING_SUMMARIZE_SYSTEM}, + {"role": "user", "content": prompt}, + ], + temperature=0.3, + task_type="summarize", + ) - summaries.append( - { - "paper_id": paper.id, - "title": paper.title, - "summary": summary, - } - ) + return {"paper_id": paper.id, "title": paper.title, "summary": summary} - return summaries + tasks = [_summarize_one(papers[pid]) for pid in paper_ids if pid in papers] + results = await asyncio.gather(*tasks, return_exceptions=True) + return [r for r in results if isinstance(r, dict)] async def generate_citations(self, paper_ids: list[int], style: str = "gb_t_7714") -> list[dict]: """Generate formatted citations for papers.""" @@ -150,10 +142,7 @@ async def generate_review_outline(self, project_id: int, topic: str, language: s outline = await self.llm.chat( messages=[ - { - "role": "system", - "content": "You are a scientific writing expert. Generate well-structured review outlines.", - }, + {"role": "system", "content": WRITING_OUTLINE_SYSTEM}, {"role": "user", "content": prompt}, ], temperature=0.5, @@ -188,10 +177,7 @@ async def analyze_gaps(self, project_id: int, research_topic: str) -> dict: analysis = await self.llm.chat( messages=[ - { - "role": "system", - "content": "You are a research gap analyst. Identify unexplored areas and innovation opportunities.", - }, + {"role": "system", "content": WRITING_GAP_SYSTEM}, {"role": "user", "content": prompt}, ], temperature=0.5, @@ -222,7 +208,7 @@ async def generate_literature_review( papers = result.scalars().all() if not papers: - yield _sse("error", {"message": "知识库中暂无文献，请先添加文献后再生成综述"}) + yield format_sse_error("知识库中暂无文献，请先添加文献后再生成综述", code=400) return yield _sse("progress", {"step": "outline", "message": "正在生成综述提纲..."}) @@ -270,7 +256,7 @@ async def generate_literature_review( async for chunk in self.llm.chat_stream( messages=[ - {"role": "system", "content": SECTION_SYSTEM_PROMPT}, + {"role": "system", "content": WRITING_SECTION_SYSTEM}, {"role": "user", "content": prompt}, ], temperature=0.5, @@ -314,10 +300,7 @@ async def _generate_review_outline_for_draft( return await self.llm.chat( messages=[ - { - "role": "system", - "content": "You are a scientific writing expert. Generate well-structured review outlines.", - }, + {"role": "system", "content": WRITING_OUTLINE_SYSTEM}, {"role": "user", "content": prompt}, ], temperature=0.5, diff --git a/backend/app/utils/sse.py b/backend/app/utils/sse.py new file mode 100644 index 0000000..6b12b51 --- /dev/null +++ b/backend/app/utils/sse.py @@ -0,0 +1,11 @@ +"""SSE (Server-Sent Events) formatting utilities.""" + +import json + + +def format_sse_error(message: str, code: int = 500) -> str: + """Format a standardized SSE error event. + + Unified format: event: error\\ndata: {"code": status_code, "message": error_msg}\\n\\n + """ + return f"event: error\ndata: {json.dumps({'code': code, 'message': message})}\n\n" diff --git a/backend/app/websocket/__init__.py b/backend/app/websocket/__init__.py new file mode 100644 index 0000000..b997b06 --- /dev/null +++ b/backend/app/websocket/__init__.py @@ -0,0 +1,5 @@ +"""WebSocket connection management for real-time pipeline status updates.""" + +from app.websocket.manager import PipelineConnectionManager, pipeline_manager + +__all__ = ["PipelineConnectionManager", "pipeline_manager"] diff --git a/backend/app/websocket/manager.py b/backend/app/websocket/manager.py new file mode 100644 index 0000000..69d3ae2 --- /dev/null +++ b/backend/app/websocket/manager.py @@ -0,0 +1,43 @@ +"""Room-based WebSocket connection manager for pipeline status broadcasts.""" + +import logging +from collections import defaultdict + +from fastapi import WebSocket + +logger = logging.getLogger(__name__) + + +class PipelineConnectionManager: + """Manages WebSocket connections grouped by pipeline thread_id (room).""" + + def __init__(self) -> None: + self.rooms: dict[str, set[WebSocket]] = defaultdict(set) + + async def connect(self, websocket: WebSocket, thread_id: str) -> None: + await websocket.accept() + self.rooms[thread_id].add(websocket) + logger.debug("WS connected to room %s (%d clients)", thread_id, len(self.rooms[thread_id])) + + def disconnect(self, websocket: WebSocket, thread_id: str) -> None: + if thread_id in self.rooms: + self.rooms[thread_id].discard(websocket) + if not self.rooms[thread_id]: + del self.rooms[thread_id] + + async def broadcast_to_room(self, thread_id: str, message: dict) -> None: + if thread_id not in self.rooms: + return + dead: list[WebSocket] = [] + for conn in list(self.rooms[thread_id]): + try: + await conn.send_json(message) + except Exception: + dead.append(conn) + for conn in dead: + self.rooms[thread_id].discard(conn) + if thread_id in self.rooms and not self.rooms[thread_id]: + del self.rooms[thread_id] + + +pipeline_manager = PipelineConnectionManager() diff --git a/backend/conftest.py b/backend/conftest.py index 93c815f..1c09e03 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -3,6 +3,9 @@ import os import tempfile +import pytest +from sqlalchemy import UniqueConstraint + _test_data_dir = tempfile.mkdtemp(prefix="omelette_test_") _test_db_path = os.path.join(_test_data_dir, "test_omelette.db") @@ -10,3 +13,61 @@ os.environ.setdefault("LLM_PROVIDER", "mock") os.environ.setdefault("DATABASE_URL", f"sqlite:///{_test_db_path}") os.environ.setdefault("DATA_DIR", _test_data_dir) + +REAL_LLM_AVAILABLE = os.environ.get("LLM_PROVIDER", "mock") != "mock" + +real_llm = pytest.mark.skipif( + not REAL_LLM_AVAILABLE, + reason="Real LLM not configured (set LLM_PROVIDER=volcengine)", +) + + +def remove_paper_doi_unique_constraint(): + """Remove (project_id, doi) unique constraint so tests can insert duplicate DOIs for dedup.""" + from app.database import Base + + table = Base.metadata.tables.get("papers") + if table is not None: + for c in list(table.constraints): + if isinstance(c, UniqueConstraint) and getattr(c, "name", None) == "uq_paper_project_doi": + table.constraints.discard(c) + break + + +# --------------------------------------------------------------------------- +# Shared fixtures (for tests that need DB + HTTP client) +# Tests with local fixtures of the same name will use their own (no override). +# --------------------------------------------------------------------------- + + +@pytest.fixture +async def setup_db(): + """Create tables before each test, drop after. Request explicitly or use local override.""" + from app.database import Base, engine + + remove_paper_doi_unique_constraint() + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def client(): + """Async HTTP client for in-process testing.""" + from httpx import ASGITransport, AsyncClient + + from app.main import app + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.fixture +async def project_id(client): + """Create a project and return its ID. Depends on client.""" + resp = await client.post("/api/v1/projects", json={"name": "Test Project", "domain": "optics"}) + assert resp.status_code == 201 + return resp.json()["data"]["id"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 45d2c95..73551bb 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -19,7 +19,6 @@ dependencies = [ "pydantic-settings>=2.7.0", "python-dotenv>=1.0.0", "httpx>=0.28.0", - "aiohttp>=3.11.0", "chromadb>=0.6.0", "openai>=1.60.0", "pdfplumber>=0.11.0", @@ -38,6 +37,7 @@ dependencies = [ "langchain-ollama>=0.3", "llama-index-core>=0.12", "llama-index-vector-stores-chroma>=0.4", + "huggingface-hub>=0.28", "llama-index-embeddings-huggingface>=0.5", "llama-index-embeddings-openai>=0.4", "mcp>=1.26", @@ -62,6 +62,8 @@ ocr = [ ml = [ "sentence-transformers>=4.0.0", "torch>=2.6.0", + "transformers>=4.51.0", + "llama-index-postprocessor-sbert-rerank>=0.4.0", ] [build-system] @@ -97,6 +99,10 @@ indent-style = "space" testpaths = ["tests"] asyncio_mode = "auto" addopts = "-v --tb=short" +markers = [ + "real_llm: marks tests requiring real LLM (deselect with -m 'not real_llm')", + "e2e: marks end-to-end tests requiring a live server (deselect with -m 'not e2e')", +] [tool.mypy] python_version = "3.12" diff --git a/backend/scripts/gpu_watchdog.py b/backend/scripts/gpu_watchdog.py new file mode 100755 index 0000000..b8fc999 --- /dev/null +++ b/backend/scripts/gpu_watchdog.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""GPU resource watchdog — monitors Omelette and cleans up GPU resources on exit. + +Runs as an independent process. When the monitored Omelette process dies +(including kill -9, OOM, crash), this script kills MinerU and clears GPU caches. + +Usage: + python scripts/gpu_watchdog.py # foreground + python scripts/gpu_watchdog.py --daemon # background (detach) + python scripts/gpu_watchdog.py --pid-file /path.pid # custom PID file +""" + +from __future__ import annotations + +import argparse +import logging +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)-8s | gpu_watchdog | %(message)s", +) +logger = logging.getLogger("gpu_watchdog") + + +def pid_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + return True + except (OSError, ProcessLookupError): + return False + + +def find_pid_by_port(port: int) -> int | None: + """Find PID listening on a TCP port.""" + try: + with open("/proc/net/tcp") as f: + hex_port = f":{port:04X}" + for line in f: + parts = line.strip().split() + if len(parts) >= 10 and hex_port in parts[1] and parts[3] == "0A": + inode = parts[9] + for pid_dir in os.listdir("/proc"): + if not pid_dir.isdigit(): + continue + try: + fd_dir = f"/proc/{pid_dir}/fd" + for fd in os.listdir(fd_dir): + link = os.readlink(f"{fd_dir}/{fd}") + if f"socket:[{inode}]" in link: + return int(pid_dir) + except (OSError, PermissionError): + continue + except (OSError, PermissionError): + pass + + try: + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0 and result.stdout.strip(): + return int(result.stdout.strip().split("\n")[0]) + except (subprocess.TimeoutExpired, ValueError, OSError, FileNotFoundError): + pass + return None + + +def is_mineru_process(pid: int) -> bool: + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + return "mineru" in f.read().decode(errors="replace").lower() + except (OSError, PermissionError): + return False + + +def kill_mineru(port: int) -> None: + pid = find_pid_by_port(port) + if pid and is_mineru_process(pid): + logger.info("Killing MinerU pid=%d on port %d", pid, port) + try: + os.kill(pid, signal.SIGTERM) + for _ in range(10): + time.sleep(1) + if not pid_alive(pid): + logger.info("MinerU pid=%d terminated", pid) + return + os.kill(pid, signal.SIGKILL) + logger.info("Force-killed MinerU pid=%d", pid) + except (OSError, ProcessLookupError): + pass + else: + logger.info("No MinerU found on port %d", port) + + +def cleanup(pid_file: Path, mineru_port: int) -> None: + logger.info("Omelette process gone — running cleanup") + kill_mineru(mineru_port) + if pid_file.exists(): + try: + pid_file.unlink() + logger.info("Removed PID file: %s", pid_file) + except OSError: + pass + logger.info("Cleanup complete") + + +def wait_for_pid_file(pid_file: Path, timeout: int = 60) -> int | None: + """Wait for PID file to appear and return the PID.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pid_file.exists(): + try: + pid = int(pid_file.read_text().strip()) + if pid_alive(pid): + return pid + except (ValueError, OSError): + pass + time.sleep(2) + return None + + +def daemonize() -> None: + """Double-fork to detach from terminal.""" + if os.fork() > 0: + sys.exit(0) + os.setsid() + if os.fork() > 0: + sys.exit(0) + devnull_r = open(os.devnull) # noqa: SIM115 + devnull_w = open(os.devnull, "w") # noqa: SIM115 + sys.stdin = devnull_r + sys.stdout = devnull_w + sys.stderr = devnull_w + + +def main() -> None: + parser = argparse.ArgumentParser(description="GPU watchdog for Omelette") + parser.add_argument("--pid-file", default="./data/omelette.pid", help="Path to PID file") + parser.add_argument("--interval", type=int, default=5, help="Check interval in seconds") + parser.add_argument("--mineru-port", type=int, default=8010, help="MinerU port") + parser.add_argument("--daemon", action="store_true", help="Run as daemon") + args = parser.parse_args() + + pid_file = Path(args.pid_file) + + if args.daemon: + daemonize() + + logger.info( + "GPU watchdog started (pid_file=%s, interval=%ds, mineru_port=%d)", pid_file, args.interval, args.mineru_port + ) + + target_pid = wait_for_pid_file(pid_file, timeout=120) + if target_pid is None: + logger.warning("No Omelette process found within timeout, exiting") + return + logger.info("Monitoring Omelette pid=%d", target_pid) + + try: + while True: + time.sleep(args.interval) + if not pid_alive(target_pid): + cleanup(pid_file, args.mineru_port) + return + except KeyboardInterrupt: + logger.info("Watchdog stopped by user") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_api_chat_rag_writing.py b/backend/tests/test_api_chat_rag_writing.py new file mode 100644 index 0000000..8492c6a --- /dev/null +++ b/backend/tests/test_api_chat_rag_writing.py @@ -0,0 +1,579 @@ +"""Comprehensive API tests for Chat, RAG, Writing, Completion, and Rewrite modules.""" + +from __future__ import annotations + +import json + +import chromadb +import pytest +from httpx import ASGITransport, AsyncClient +from llama_index.core.embeddings import MockEmbedding + +from app.api.v1.rag import get_rag_service +from app.database import Base, async_session_factory, engine +from app.main import app +from app.models import Paper, PaperChunk, PaperStatus, Project +from app.services.rag_service import RAGService + +MOCK_EMBED = MockEmbedding(embed_dim=128) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +async def setup_db(): + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.fixture +def rag_service(): + """RAGService with ephemeral ChromaDB and mock embedding for fast tests.""" + return RAGService( + chroma_client=chromadb.EphemeralClient(), + embed_model=MOCK_EMBED, + ) + + +@pytest.fixture(autouse=True) +def override_rag_dependency(rag_service): + """Override RAG dependency to use ephemeral ChromaDB.""" + app.dependency_overrides[get_rag_service] = lambda: rag_service + yield + app.dependency_overrides.pop(get_rag_service, None) + + +@pytest.fixture(autouse=True) +def mock_chat_services(monkeypatch): + """Mock _init_services so Chat stream uses mock LLM/RAG without DB lookups.""" + import app.api.v1.chat as chat_module + from app.services.llm.client import LLMClient + + async def _mock_init_services(db): + from app.services.rag_service import RAGService + + llm = LLMClient(provider="mock") + rag = RAGService(llm=llm, embed_model=MockEmbedding(embed_dim=128)) + return {"llm": llm, "rag": rag} + + monkeypatch.setattr(chat_module, "_init_services", _mock_init_services) + + +@pytest.fixture +async def project_with_chunks(): + """Create a project with OCR-complete papers and chunks for RAG tests.""" + async with async_session_factory() as session: + project = Project(name="RAG Test Project", domain="optics") + session.add(project) + await session.flush() + + paper = Paper( + project_id=project.id, + title="Super-Resolution Microscopy Review", + abstract="A review of super-resolution techniques.", + journal="Nature", + year=2023, + status=PaperStatus.OCR_COMPLETE, + ) + session.add(paper) + await session.flush() + + chunk1 = PaperChunk( + paper_id=paper.id, + content="Super-resolution microscopy enables imaging beyond the diffraction limit.", + chunk_type="text", + page_number=1, + chunk_index=0, + ) + chunk2 = PaperChunk( + paper_id=paper.id, + content="STED and STORM are two major techniques for nanoscale imaging.", + chunk_type="text", + page_number=2, + chunk_index=1, + ) + session.add(chunk1) + session.add(chunk2) + await session.commit() + return project.id + + +@pytest.fixture +async def project_with_papers(): + """Create a project with papers for writing tests.""" + async with async_session_factory() as session: + project = Project(name="Writing Test Project", domain="optics") + session.add(project) + await session.flush() + + paper1 = Paper( + project_id=project.id, + title="Super-Resolution Microscopy", + abstract="A comprehensive review of super-resolution techniques.", + journal="Nature", + year=2023, + authors=[{"name": "Alice Smith"}, {"name": "Bob Jones"}], + citation_count=100, + status=PaperStatus.INDEXED, + ) + paper2 = Paper( + project_id=project.id, + title="STED Imaging Methods", + abstract="Stimulated emission depletion microscopy for nanoscale imaging.", + journal="Science", + year=2022, + authors=[{"name": "Carol Lee"}], + doi="10.1234/test", + citation_count=50, + status=PaperStatus.INDEXED, + ) + session.add(paper1) + session.add(paper2) + await session.flush() + paper_ids = [paper1.id, paper2.id] + await session.commit() + return project.id, paper_ids + + +# --------------------------------------------------------------------------- +# Chat API tests +# --------------------------------------------------------------------------- + + +class TestChatStream: + """Tests for POST /api/v1/chat/stream (SSE).""" + + @pytest.mark.asyncio + async def test_stream_returns_sse(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/stream", + json={"message": "Hello", "knowledge_base_ids": []}, + ) + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("text/event-stream") + + text = resp.text + lines = [line for line in text.split("\n") if line.startswith("data: ")] + + event_types = [] + for line in lines: + payload = line.removeprefix("data: ").strip() + if payload == "[DONE]": + event_types.append("[DONE]") + continue + try: + parsed = json.loads(payload) + event_types.append(parsed.get("type", "unknown")) + except json.JSONDecodeError: + pass + + assert "start" in event_types + assert "text-delta" in event_types + assert "finish" in event_types + assert "[DONE]" in event_types + + @pytest.mark.asyncio + async def test_stream_with_rag_top_k_and_use_reranker(self, client: AsyncClient): + """Chat stream accepts rag_top_k (1-50) and use_reranker.""" + resp = await client.post( + "/api/v1/chat/stream", + json={ + "message": "What is super-resolution?", + "knowledge_base_ids": [1], + "rag_top_k": 15, + "use_reranker": True, + }, + ) + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("text/event-stream") + assert "data:" in resp.text + + @pytest.mark.asyncio + async def test_stream_rag_top_k_validation_min_fails(self, client: AsyncClient): + """rag_top_k=0 should fail validation.""" + resp = await client.post( + "/api/v1/chat/stream", + json={"message": "Hello", "knowledge_base_ids": [], "rag_top_k": 0}, + ) + assert resp.status_code == 422 + + @pytest.mark.asyncio + async def test_stream_rag_top_k_validation_max_fails(self, client: AsyncClient): + """rag_top_k=51 should fail validation.""" + resp = await client.post( + "/api/v1/chat/stream", + json={"message": "Hello", "knowledge_base_ids": [], "rag_top_k": 51}, + ) + assert resp.status_code == 422 + + @pytest.mark.asyncio + async def test_stream_message_required(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/stream", + json={"message": "", "knowledge_base_ids": []}, + ) + assert resp.status_code == 422 + + +class TestChatComplete: + """Tests for POST /api/v1/chat/complete (Completion).""" + + @pytest.mark.asyncio + async def test_complete_success(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/complete", + json={ + "prefix": "深度学习在自然语言处理领域", + "knowledge_base_ids": [], + }, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert "completion" in data + assert "confidence" in data + + @pytest.mark.asyncio + async def test_complete_prefix_too_short_fails(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/complete", + json={"prefix": "short"}, + ) + assert resp.status_code == 422 + + +# --------------------------------------------------------------------------- +# RAG API tests +# --------------------------------------------------------------------------- + + +class TestRAGQuery: + """Tests for POST /api/v1/projects/{project_id}/rag/query.""" + + @pytest.mark.asyncio + async def test_query_empty_index(self, client: AsyncClient, project_with_chunks: int): + resp = await client.post( + f"/api/v1/projects/{project_with_chunks}/rag/query", + json={"question": "What is super-resolution?", "top_k": 5}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert "answer" in body["data"] + assert "sources" in body["data"] + assert "confidence" in body["data"] + + @pytest.mark.asyncio + async def test_query_with_use_reranker(self, client: AsyncClient, project_with_chunks: int): + resp = await client.post( + f"/api/v1/projects/{project_with_chunks}/rag/query", + json={ + "question": "What is super-resolution?", + "top_k": 5, + "use_reranker": True, + }, + ) + assert resp.status_code == 200 + body = resp.json() + assert "answer" in body["data"] + + @pytest.mark.asyncio + async def test_query_without_reranker(self, client: AsyncClient, project_with_chunks: int): + resp = await client.post( + f"/api/v1/projects/{project_with_chunks}/rag/query", + json={ + "question": "What is super-resolution?", + "top_k": 5, + "use_reranker": False, + }, + ) + assert resp.status_code == 200 + body = resp.json() + assert "answer" in body["data"] + + @pytest.mark.asyncio + async def test_query_top_k_validation_min_fails(self, client: AsyncClient, project_with_chunks: int): + """top_k=0 should fail validation.""" + resp = await client.post( + f"/api/v1/projects/{project_with_chunks}/rag/query", + json={"question": "test", "top_k": 0}, + ) + assert resp.status_code == 422 + + @pytest.mark.asyncio + async def test_query_top_k_validation_max_fails(self, client: AsyncClient, project_with_chunks: int): + """top_k=51 should fail validation.""" + resp = await client.post( + f"/api/v1/projects/{project_with_chunks}/rag/query", + json={"question": "test", "top_k": 51}, + ) + assert resp.status_code == 422 + + @pytest.mark.asyncio + async def test_query_after_index(self, client: AsyncClient, project_with_chunks: int): + await client.post(f"/api/v1/projects/{project_with_chunks}/rag/index") + + resp = await client.post( + f"/api/v1/projects/{project_with_chunks}/rag/query", + json={"question": "What is super-resolution microscopy?", "top_k": 5}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert "answer" in body["data"] + assert "sources" in body["data"] + assert "confidence" in body["data"] + + +class TestRAGIndex: + """Tests for POST /api/v1/projects/{project_id}/rag/index.""" + + @pytest.mark.asyncio + async def test_build_index(self, client: AsyncClient, project_with_chunks: int): + resp = await client.post(f"/api/v1/projects/{project_with_chunks}/rag/index") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert "indexed" in body["data"] + assert body["data"]["indexed"] >= 0 + + +class TestRAGIndexStream: + """Tests for POST /api/v1/projects/{project_id}/rag/index/stream (SSE).""" + + @pytest.mark.asyncio + async def test_index_stream_returns_sse(self, client: AsyncClient, project_with_chunks: int): + resp = await client.post(f"/api/v1/projects/{project_with_chunks}/rag/index/stream") + assert resp.status_code == 200 + assert resp.headers.get("content-type", "").startswith("text/event-stream") + + text = resp.text + assert "event:" in text + assert "data:" in text + + # Should have progress and complete events + lines = text.split("\n") + event_lines = [line for line in lines if line.startswith("event:")] + assert len(event_lines) >= 1 + + +class TestRAGStats: + """Tests for GET /api/v1/projects/{project_id}/rag/stats.""" + + @pytest.mark.asyncio + async def test_stats(self, client: AsyncClient, project_with_chunks: int): + resp = await client.get(f"/api/v1/projects/{project_with_chunks}/rag/stats") + assert resp.status_code == 200 + body = resp.json() + assert "total_chunks" in body["data"] + assert "collection_name" in body["data"] + + +class TestRAGDeleteIndex: + """Tests for DELETE /api/v1/projects/{project_id}/rag/index.""" + + @pytest.mark.asyncio + async def test_delete_index(self, client: AsyncClient, project_with_chunks: int): + resp = await client.delete(f"/api/v1/projects/{project_with_chunks}/rag/index") + assert resp.status_code == 200 + body = resp.json() + assert "deleted" in body["data"] + + +# --------------------------------------------------------------------------- +# Writing API tests +# --------------------------------------------------------------------------- + + +class TestWritingSummarize: + """Tests for POST /api/v1/projects/{project_id}/writing/summarize.""" + + @pytest.mark.asyncio + async def test_summarize(self, client: AsyncClient, project_with_papers): + project_id, paper_ids = project_with_papers + resp = await client.post( + f"/api/v1/projects/{project_id}/writing/summarize", + json={"paper_ids": paper_ids, "language": "en"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert "summaries" in body["data"] + assert len(body["data"]["summaries"]) == 2 + + +class TestWritingCitations: + """Tests for POST /api/v1/projects/{project_id}/writing/citations.""" + + @pytest.mark.asyncio + async def test_citations(self, client: AsyncClient, project_with_papers): + project_id, paper_ids = project_with_papers + resp = await client.post( + f"/api/v1/projects/{project_id}/writing/citations", + json={"paper_ids": paper_ids, "style": "gb_t_7714"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert "citations" in body["data"] + assert body["data"]["style"] == "gb_t_7714" + assert len(body["data"]["citations"]) == 2 + + +class TestWritingReviewOutline: + """Tests for POST /api/v1/projects/{project_id}/writing/review-outline.""" + + @pytest.mark.asyncio + async def test_review_outline(self, client: AsyncClient, project_with_papers): + project_id, _ = project_with_papers + resp = await client.post( + f"/api/v1/projects/{project_id}/writing/review-outline", + json={"topic": "Super-resolution imaging", "language": "en"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert "outline" in body["data"] + assert "paper_count" in body["data"] + + +class TestWritingGapAnalysis: + """Tests for POST /api/v1/projects/{project_id}/writing/gap-analysis.""" + + @pytest.mark.asyncio + async def test_gap_analysis(self, client: AsyncClient, project_with_papers): + project_id, _ = project_with_papers + resp = await client.post( + f"/api/v1/projects/{project_id}/writing/gap-analysis", + json={"research_topic": "Nanoscale microscopy"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert "analysis" in body["data"] + assert "papers_analyzed" in body["data"] + + +class TestWritingReviewDraftStream: + """Tests for POST /api/v1/projects/{project_id}/writing/review-draft/stream (SSE).""" + + @pytest.mark.asyncio + async def test_review_draft_stream_returns_sse(self, client: AsyncClient, project_with_papers): + project_id, _ = project_with_papers + resp = await client.post( + f"/api/v1/projects/{project_id}/writing/review-draft/stream", + json={ + "topic": "Super-resolution microscopy", + "style": "narrative", + "citation_format": "numbered", + "language": "en", + }, + ) + assert resp.status_code == 200 + assert resp.headers.get("content-type", "").startswith("text/event-stream") + + text = resp.text + assert "event:" in text + assert "data:" in text + + @pytest.mark.asyncio + async def test_review_draft_stream_invalid_style_fails(self, client: AsyncClient, project_with_papers): + project_id, _ = project_with_papers + resp = await client.post( + f"/api/v1/projects/{project_id}/writing/review-draft/stream", + json={"topic": "test", "style": "invalid_style"}, + ) + assert resp.status_code == 422 + + +# --------------------------------------------------------------------------- +# Rewrite API tests (POST /api/v1/chat/rewrite) +# --------------------------------------------------------------------------- + + +class TestRewrite: + """Tests for POST /api/v1/chat/rewrite (SSE).""" + + @pytest.mark.asyncio + async def test_rewrite_stream_returns_sse(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/rewrite", + json={ + "excerpt": "This is a sample excerpt to simplify for testing.", + "style": "simplify", + }, + ) + assert resp.status_code == 200 + assert resp.headers.get("content-type", "").startswith("text/event-stream") + + text = resp.text + assert "event:" in text + assert "data:" in text + + # Parse SSE events + lines = text.split("\n") + event_types = [] + for line in lines: + if line.startswith("event:"): + event_types.append(line.replace("event:", "").strip()) + + assert "rewrite_delta" in event_types or "rewrite_end" in event_types or "error" in event_types + + @pytest.mark.asyncio + async def test_rewrite_academic_style(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/rewrite", + json={ + "excerpt": "This is a simple sentence.", + "style": "academic", + }, + ) + assert resp.status_code == 200 + assert "data:" in resp.text + + @pytest.mark.asyncio + async def test_rewrite_excerpt_too_long_fails(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/rewrite", + json={ + "excerpt": "x" * 2001, + "style": "simplify", + }, + ) + assert resp.status_code == 422 + + @pytest.mark.asyncio + async def test_rewrite_custom_requires_prompt(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/rewrite", + json={ + "excerpt": "Sample text", + "style": "custom", + }, + ) + assert resp.status_code == 422 + + @pytest.mark.asyncio + async def test_rewrite_custom_with_prompt(self, client: AsyncClient): + resp = await client.post( + "/api/v1/chat/rewrite", + json={ + "excerpt": "Sample text to rewrite.", + "style": "custom", + "custom_prompt": "Rewrite this in a formal tone.", + }, + ) + assert resp.status_code == 200 + assert "data:" in resp.text diff --git a/backend/tests/test_api_convos_subs_tasks_settings.py b/backend/tests/test_api_convos_subs_tasks_settings.py new file mode 100644 index 0000000..03153f3 --- /dev/null +++ b/backend/tests/test_api_convos_subs_tasks_settings.py @@ -0,0 +1,669 @@ +"""Comprehensive API tests for Conversations, Subscriptions, Tasks, Settings, and Pipelines.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.database import Base, async_session_factory, engine +from app.main import app +from app.models import Message, Project, Task + + +@pytest.fixture(autouse=True) +async def setup_db(): + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.fixture +async def project(setup_db): + async with async_session_factory() as db: + p = Project(name="Test Project", description="For API tests") + db.add(p) + await db.commit() + await db.refresh(p) + return p + + +# ── Conversations ── + + +class TestConversationsAPI: + """Tests for /api/v1/conversations.""" + + @pytest.mark.asyncio + async def test_list_conversations_empty(self, client): + resp = await client.get("/api/v1/conversations") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["items"] == [] + assert data["total"] == 0 + assert "page" in data + assert "page_size" in data + + @pytest.mark.asyncio + async def test_list_conversations_paginated(self, client): + for i in range(5): + await client.post("/api/v1/conversations", json={"title": f"Conv {i}"}) + resp = await client.get("/api/v1/conversations", params={"page": 1, "page_size": 2}) + assert resp.status_code == 200 + data = resp.json()["data"] + assert len(data["items"]) == 2 + assert data["total"] == 5 + assert data["page"] == 1 + assert data["page_size"] == 2 + assert data["total_pages"] == 3 + + @pytest.mark.asyncio + async def test_list_conversations_filter_by_knowledge_base_id(self, client): + await client.post( + "/api/v1/conversations", + json={"title": "KB1", "knowledge_base_ids": [1, 2]}, + ) + await client.post( + "/api/v1/conversations", + json={"title": "KB2", "knowledge_base_ids": [3, 4]}, + ) + resp = await client.get("/api/v1/conversations", params={"knowledge_base_id": 1}) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["total"] == 1 + assert data["items"][0]["knowledge_base_ids"] == [1, 2] + + @pytest.mark.asyncio + async def test_create_conversation(self, client): + resp = await client.post( + "/api/v1/conversations", + json={ + "title": "New Chat", + "knowledge_base_ids": [1, 2], + "model": "gpt-4o", + "tool_mode": "citation_lookup", + }, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["title"] == "New Chat" + assert data["knowledge_base_ids"] == [1, 2] + assert data["model"] == "gpt-4o" + assert data["tool_mode"] == "citation_lookup" + assert data["messages"] == [] + assert "id" in data + assert "created_at" in data + + @pytest.mark.asyncio + async def test_create_conversation_default_title(self, client): + resp = await client.post("/api/v1/conversations", json={}) + assert resp.status_code == 200 + assert resp.json()["data"]["title"] == "新对话" + + @pytest.mark.asyncio + async def test_get_conversation_with_messages(self, client): + create_resp = await client.post( + "/api/v1/conversations", + json={"title": "With Messages", "knowledge_base_ids": [1]}, + ) + conv_id = create_resp.json()["data"]["id"] + async with async_session_factory() as db: + msg = Message( + conversation_id=conv_id, + role="user", + content="Hello", + ) + db.add(msg) + await db.commit() + + resp = await client.get(f"/api/v1/conversations/{conv_id}") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["title"] == "With Messages" + assert len(data["messages"]) == 1 + assert data["messages"][0]["role"] == "user" + assert data["messages"][0]["content"] == "Hello" + + @pytest.mark.asyncio + async def test_get_conversation_not_found(self, client): + resp = await client.get("/api/v1/conversations/99999") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_update_conversation(self, client): + create_resp = await client.post( + "/api/v1/conversations", + json={"title": "Old", "tool_mode": "qa"}, + ) + conv_id = create_resp.json()["data"]["id"] + resp = await client.put( + f"/api/v1/conversations/{conv_id}", + json={"title": "Updated", "tool_mode": "review_outline"}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["title"] == "Updated" + assert data["tool_mode"] == "review_outline" + + @pytest.mark.asyncio + async def test_update_conversation_not_found(self, client): + resp = await client.put( + "/api/v1/conversations/99999", + json={"title": "X"}, + ) + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_conversation(self, client): + create_resp = await client.post( + "/api/v1/conversations", + json={"title": "To Delete"}, + ) + conv_id = create_resp.json()["data"]["id"] + resp = await client.delete(f"/api/v1/conversations/{conv_id}") + assert resp.status_code == 200 + assert resp.json()["data"]["deleted"] is True + assert resp.json()["data"]["id"] == conv_id + + resp2 = await client.get(f"/api/v1/conversations/{conv_id}") + assert resp2.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_conversation_not_found(self, client): + resp = await client.delete("/api/v1/conversations/99999") + assert resp.status_code == 404 + + +# ── Subscriptions ── + + +class TestSubscriptionsAPI: + """Tests for /api/v1/projects/{project_id}/subscriptions.""" + + @pytest.mark.asyncio + async def test_list_subscriptions_empty(self, client, project): + resp = await client.get(f"/api/v1/projects/{project.id}/subscriptions") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["items"] == [] + assert data["total"] == 0 + + @pytest.mark.asyncio + async def test_create_subscription_api_type(self, client, project): + resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions", + json={ + "name": "API Sub", + "query": "machine learning", + "sources": ["semantic_scholar", "arxiv"], + "frequency": "weekly", + "max_results": 50, + }, + ) + assert resp.status_code == 201 + data = resp.json()["data"] + assert data["name"] == "API Sub" + assert data["query"] == "machine learning" + assert data["sources"] == ["semantic_scholar", "arxiv"] + assert data["frequency"] == "weekly" + assert data["max_results"] == 50 + assert data["project_id"] == project.id + assert data["is_active"] is True + + @pytest.mark.asyncio + async def test_create_subscription_minimal(self, client, project): + resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions", + json={"name": "Minimal Sub"}, + ) + assert resp.status_code == 201 + data = resp.json()["data"] + assert data["name"] == "Minimal Sub" + assert data["query"] == "" + assert data["sources"] == [] + assert data["frequency"] == "weekly" + assert data["max_results"] == 50 + + @pytest.mark.asyncio + async def test_get_subscription(self, client, project): + create_resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions", + json={"name": "Get Me"}, + ) + sub_id = create_resp.json()["data"]["id"] + resp = await client.get(f"/api/v1/projects/{project.id}/subscriptions/{sub_id}") + assert resp.status_code == 200 + assert resp.json()["data"]["name"] == "Get Me" + + @pytest.mark.asyncio + async def test_get_subscription_not_found(self, client, project): + resp = await client.get(f"/api/v1/projects/{project.id}/subscriptions/99999") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_get_subscription_wrong_project(self, client, project): + create_resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions", + json={"name": "Sub"}, + ) + sub_id = create_resp.json()["data"]["id"] + resp = await client.get(f"/api/v1/projects/99999/subscriptions/{sub_id}") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_update_subscription(self, client, project): + create_resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions", + json={"name": "Old", "query": "old query"}, + ) + sub_id = create_resp.json()["data"]["id"] + resp = await client.put( + f"/api/v1/projects/{project.id}/subscriptions/{sub_id}", + json={"name": "New Name", "query": "new query", "is_active": False}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["name"] == "New Name" + assert data["query"] == "new query" + assert data["is_active"] is False + + @pytest.mark.asyncio + async def test_delete_subscription(self, client, project): + create_resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions", + json={"name": "To Delete"}, + ) + sub_id = create_resp.json()["data"]["id"] + resp = await client.delete(f"/api/v1/projects/{project.id}/subscriptions/{sub_id}") + assert resp.status_code == 200 + + resp2 = await client.get(f"/api/v1/projects/{project.id}/subscriptions/{sub_id}") + assert resp2.status_code == 404 + + @pytest.mark.asyncio + async def test_trigger_subscription(self, client, project): + create_resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions", + json={"name": "Trigger Sub", "query": "test", "max_results": 10}, + ) + sub_id = create_resp.json()["data"]["id"] + resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions/{sub_id}/trigger", + params={"since_days": 7}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert "new_papers" in data + assert "total_checked" in data + assert "sources_searched" in data + + @pytest.mark.asyncio + async def test_check_rss(self, client, project): + mock_rss = """ +Test +Paperhttps://example.comhttps://doi.org/10.1234/test +""" + mock_resp = MagicMock() + mock_resp.text = mock_rss + mock_resp.raise_for_status = MagicMock() + + with ( + patch("app.services.url_validator.validate_url_safe", return_value="https://example.com/feed.xml"), + patch("httpx.AsyncClient") as mock_client_cls, + ): + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(return_value=mock_resp) + mock_client_cls.return_value = mock_client + + resp = await client.post( + f"/api/v1/projects/{project.id}/subscriptions/check-rss", + params={"feed_url": "https://example.com/feed.xml", "since_days": 7}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert "entries" in data + assert "count" in data + + @pytest.mark.asyncio + async def test_list_common_feeds(self, client, project): + resp = await client.get(f"/api/v1/projects/{project.id}/subscriptions/feeds") + assert resp.status_code == 200 + data = resp.json()["data"] + assert isinstance(data, list) + assert len(data) >= 4 + assert all("name" in f and "url" in f for f in data) + + +# ── Tasks ── + + +class TestTasksAPI: + """Tests for /api/v1/tasks.""" + + @pytest.mark.asyncio + async def test_list_tasks_empty(self, client): + resp = await client.get("/api/v1/tasks") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["items"] == [] + assert data["total"] == 0 + assert "page" in data + assert "page_size" in data + + @pytest.mark.asyncio + async def test_list_tasks_paginated(self, client, project): + async with async_session_factory() as db: + for _ in range(5): + t = Task( + project_id=project.id, + task_type="search", + status="pending", + ) + db.add(t) + await db.commit() + + resp = await client.get("/api/v1/tasks", params={"page": 1, "page_size": 2}) + assert resp.status_code == 200 + data = resp.json()["data"] + assert len(data["items"]) == 2 + assert data["total"] == 5 + assert data["page"] == 1 + assert data["page_size"] == 2 + + @pytest.mark.asyncio + async def test_list_tasks_filter_by_status(self, client, project): + async with async_session_factory() as db: + db.add(Task(project_id=project.id, task_type="search", status="pending")) + db.add(Task(project_id=project.id, task_type="search", status="running")) + db.add(Task(project_id=project.id, task_type="search", status="completed")) + await db.commit() + + resp = await client.get("/api/v1/tasks", params={"status": "pending"}) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["total"] == 1 + assert data["items"][0]["status"] == "pending" + + @pytest.mark.asyncio + async def test_list_tasks_filter_by_project_id(self, client, project): + async with async_session_factory() as db: + db.add(Task(project_id=project.id, task_type="search", status="pending")) + await db.commit() + + resp = await client.get("/api/v1/tasks", params={"project_id": project.id}) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["total"] == 1 + assert data["items"][0]["project_id"] == project.id + + @pytest.mark.asyncio + async def test_get_task(self, client, project): + async with async_session_factory() as db: + t = Task( + project_id=project.id, + task_type="search", + status="running", + progress=50, + total=100, + ) + db.add(t) + await db.commit() + await db.refresh(t) + task_id = t.id + + resp = await client.get(f"/api/v1/tasks/{task_id}") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["id"] == task_id + assert data["task_type"] == "search" + assert data["status"] == "running" + assert data["progress"] == 50 + assert data["total"] == 100 + + @pytest.mark.asyncio + async def test_get_task_not_found(self, client): + resp = await client.get("/api/v1/tasks/99999") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_cancel_task(self, client, project): + async with async_session_factory() as db: + t = Task( + project_id=project.id, + task_type="search", + status="running", + ) + db.add(t) + await db.commit() + await db.refresh(t) + task_id = t.id + + resp = await client.post(f"/api/v1/tasks/{task_id}/cancel") + assert resp.status_code == 200 + assert resp.json()["message"] == "Task cancelled" + + resp2 = await client.get(f"/api/v1/tasks/{task_id}") + assert resp2.json()["data"]["status"] == "cancelled" + + @pytest.mark.asyncio + async def test_cancel_task_already_completed_fails(self, client, project): + async with async_session_factory() as db: + t = Task( + project_id=project.id, + task_type="search", + status="completed", + ) + db.add(t) + await db.commit() + await db.refresh(t) + task_id = t.id + + resp = await client.post(f"/api/v1/tasks/{task_id}/cancel") + assert resp.status_code == 400 + assert "Cannot cancel" in resp.json()["message"] + + @pytest.mark.asyncio + async def test_cancel_task_not_found(self, client): + resp = await client.post("/api/v1/tasks/99999/cancel") + assert resp.status_code == 404 + + +# ── Settings ── + + +class TestSettingsAPI: + """Tests for /api/v1/settings.""" + + @pytest.mark.asyncio + async def test_get_settings(self, client): + resp = await client.get("/api/v1/settings") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["llm_provider"] == "mock" + for key in ["openai_api_key", "anthropic_api_key"]: + val = data.get(key, "") + assert "***" in val or val == "" + + @pytest.mark.asyncio + async def test_put_settings(self, client): + resp = await client.put( + "/api/v1/settings", + json={"llm_provider": "openai", "llm_temperature": 0.7}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["llm_provider"] == "openai" + assert data["llm_temperature"] == 0.7 + + resp2 = await client.get("/api/v1/settings") + assert resp2.json()["data"]["llm_provider"] == "openai" + + @pytest.mark.asyncio + async def test_list_models(self, client): + resp = await client.get("/api/v1/settings/models") + assert resp.status_code == 200 + data = resp.json()["data"] + providers = [p["provider"] for p in data] + assert "openai" in providers + assert "anthropic" in providers + assert "mock" in providers + + @pytest.mark.asyncio + async def test_test_connection_mock(self, client): + resp = await client.post("/api/v1/settings/test-connection") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["success"] is True + assert "response" in data + + @pytest.mark.asyncio + async def test_health_check_unauthenticated(self, client): + """Health endpoint is auth-exempt and returns 200.""" + resp = await client.get("/api/v1/settings/health") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["status"] == "healthy" + assert "version" in data + + +# ── Pipelines ── + + +class TestPipelinesAPI: + """Tests for /api/v1/pipelines.""" + + @pytest.mark.asyncio + async def test_start_search_pipeline(self, client, project): + with patch("app.services.search_service.SearchService.search", new_callable=AsyncMock) as mock_search: + mock_search.return_value = {"papers": [], "total": 0} + + resp = await client.post( + "/api/v1/pipelines/search", + json={ + "project_id": project.id, + "query": "machine learning", + "max_results": 10, + }, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert "thread_id" in data + assert data["status"] == "running" + assert data["project_id"] == project.id + + @pytest.mark.asyncio + async def test_start_search_pipeline_project_not_found(self, client): + resp = await client.post( + "/api/v1/pipelines/search", + json={ + "project_id": 99999, + "query": "test", + "max_results": 10, + }, + ) + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_get_pipeline_status(self, client, project): + with patch("app.services.search_service.SearchService.search", new_callable=AsyncMock) as mock_search: + mock_search.return_value = {"papers": [], "total": 0} + + start_resp = await client.post( + "/api/v1/pipelines/search", + json={ + "project_id": project.id, + "query": "test", + "max_results": 5, + }, + ) + thread_id = start_resp.json()["data"]["thread_id"] + + # Get status immediately before pipeline completes and removes itself + status_resp = await client.get(f"/api/v1/pipelines/{thread_id}/status") + assert status_resp.status_code == 200 + data = status_resp.json()["data"] + assert data["thread_id"] == thread_id + assert "status" in data + + @pytest.mark.asyncio + async def test_get_pipeline_status_not_found(self, client): + resp = await client.get("/api/v1/pipelines/nonexistent_thread/status") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_resume_pipeline_not_found(self, client): + resp = await client.post( + "/api/v1/pipelines/nonexistent/resume", + json={"resolved_conflicts": []}, + ) + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_resume_pipeline_not_interrupted(self, client, project): + """Resume returns 400 when pipeline is completed, not interrupted.""" + + async def slow_search(*args, **kwargs): + import asyncio + + await asyncio.sleep(3) + return {"papers": [], "total": 0} + + with patch("app.services.search_service.SearchService.search", new_callable=AsyncMock) as mock_search: + mock_search.side_effect = slow_search + + start_resp = await client.post( + "/api/v1/pipelines/search", + json={ + "project_id": project.id, + "query": "test", + "max_results": 5, + }, + ) + thread_id = start_resp.json()["data"]["thread_id"] + + import asyncio + + await asyncio.sleep(1) + + resp = await client.post( + f"/api/v1/pipelines/{thread_id}/resume", + json={"resolved_conflicts": []}, + ) + assert resp.status_code == 400 + assert "not interrupted" in resp.json()["message"].lower() + + @pytest.mark.asyncio + async def test_cancel_pipeline(self, client, project): + with patch("app.services.search_service.SearchService.search", new_callable=AsyncMock) as mock_search: + + async def slow_search(*args, **kwargs): + import asyncio + + await asyncio.sleep(10) + return {"papers": [], "total": 0} + + mock_search.side_effect = slow_search + + start_resp = await client.post( + "/api/v1/pipelines/search", + json={ + "project_id": project.id, + "query": "test", + "max_results": 5, + }, + ) + thread_id = start_resp.json()["data"]["thread_id"] + + cancel_resp = await client.post(f"/api/v1/pipelines/{thread_id}/cancel") + assert cancel_resp.status_code == 200 + assert cancel_resp.json()["data"]["status"] == "cancelled" diff --git a/backend/tests/test_api_keywords_search_dedup.py b/backend/tests/test_api_keywords_search_dedup.py new file mode 100644 index 0000000..69039f3 --- /dev/null +++ b/backend/tests/test_api_keywords_search_dedup.py @@ -0,0 +1,572 @@ +"""Comprehensive API tests for Keywords, Search, and Dedup modules.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from conftest import real_llm, remove_paper_doi_unique_constraint +from httpx import ASGITransport, AsyncClient + +from app.database import Base, engine +from app.main import app + +# --- Fixtures --- + + +@pytest.fixture(autouse=True) +async def setup_db(): + remove_paper_doi_unique_constraint() + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.fixture +async def project_id(client: AsyncClient) -> int: + """Create a project and return its ID.""" + resp = await client.post("/api/v1/projects", json={"name": "Test Project", "domain": "optics"}) + assert resp.status_code == 201 + return resp.json()["data"]["id"] + + +# ============================================================================= +# KEYWORDS API +# ============================================================================= + + +class TestKeywordsAPI: + """Tests for /api/v1/projects/{project_id}/keywords endpoints.""" + + @pytest.mark.asyncio + async def test_list_keywords_empty(self, client: AsyncClient, project_id: int): + resp = await client.get(f"/api/v1/projects/{project_id}/keywords") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert body["data"]["items"] == [] + assert body["data"]["total"] == 0 + assert body["data"]["page"] == 1 + assert body["data"]["page_size"] in (20, 50) + + @pytest.mark.asyncio + async def test_create_keyword(self, client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={ + "term": "超分辨率显微", + "term_en": "super-resolution microscopy", + "level": 1, + "category": "technique", + "synonyms": "SRM, nanoscopy", + }, + ) + assert resp.status_code == 201 + body = resp.json() + assert body["code"] == 201 + assert body["data"]["term"] == "超分辨率显微" + assert body["data"]["term_en"] == "super-resolution microscopy" + assert body["data"]["level"] == 1 + assert body["data"]["id"] > 0 + + @pytest.mark.asyncio + async def test_list_keywords_paginated(self, client: AsyncClient, project_id: int): + for i in range(5): + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": f"Term {i}", "term_en": f"term{i}", "level": 1}, + ) + resp = await client.get(f"/api/v1/projects/{project_id}/keywords?page=1&page_size=2") + assert resp.status_code == 200 + body = resp.json() + assert len(body["data"]["items"]) == 2 + assert body["data"]["total"] == 5 + assert body["data"]["page"] == 1 + assert body["data"]["page_size"] == 2 + + @pytest.mark.asyncio + async def test_list_keywords_by_level(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "Core Term", "term_en": "core", "level": 1}, + ) + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "Sub Term", "term_en": "sub", "level": 2}, + ) + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "Another Core", "term_en": "core2", "level": 1}, + ) + resp = await client.get(f"/api/v1/projects/{project_id}/keywords?level=1") + assert resp.status_code == 200 + body = resp.json() + assert len(body["data"]["items"]) == 2 + assert all(k["level"] == 1 for k in body["data"]["items"]) + + @pytest.mark.asyncio + async def test_bulk_create_keywords(self, client: AsyncClient, project_id: int): + keywords = [ + {"term": "Term 1", "term_en": "term1", "level": 1}, + {"term": "Term 2", "term_en": "term2", "level": 2}, + {"term": "Term 3", "term_en": "term3", "level": 3}, + ] + resp = await client.post(f"/api/v1/projects/{project_id}/keywords/bulk", json=keywords) + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["created"] == 3 + list_resp = await client.get(f"/api/v1/projects/{project_id}/keywords") + assert len(list_resp.json()["data"]["items"]) == 3 + + @pytest.mark.asyncio + async def test_update_keyword(self, client: AsyncClient, project_id: int): + create_resp = await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "Original", "term_en": "original", "level": 1}, + ) + keyword_id = create_resp.json()["data"]["id"] + resp = await client.put( + f"/api/v1/projects/{project_id}/keywords/{keyword_id}", + json={"term": "Updated", "term_en": "updated"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["term"] == "Updated" + assert body["data"]["term_en"] == "updated" + + @pytest.mark.asyncio + async def test_delete_keyword(self, client: AsyncClient, project_id: int): + create_resp = await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "To Delete", "term_en": "delete", "level": 1}, + ) + keyword_id = create_resp.json()["data"]["id"] + resp = await client.delete(f"/api/v1/projects/{project_id}/keywords/{keyword_id}") + assert resp.status_code == 200 + list_resp = await client.get(f"/api/v1/projects/{project_id}/keywords") + assert len(list_resp.json()["data"]["items"]) == 0 + + @pytest.mark.asyncio + async def test_search_formula_empty(self, client: AsyncClient, project_id: int): + resp = await client.get(f"/api/v1/projects/{project_id}/keywords/search-formula?database=wos") + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["formula"] == "" + assert body["data"]["database"] == "wos" + assert body["data"]["keyword_count"] == 0 + + @pytest.mark.asyncio + async def test_search_formula_with_keywords(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={ + "term": "超分辨率", + "term_en": "super-resolution", + "level": 1, + "synonyms": "SRM, nanoscopy", + }, + ) + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "STED", "term_en": "STED", "level": 2, "synonyms": "STED microscopy"}, + ) + resp = await client.get(f"/api/v1/projects/{project_id}/keywords/search-formula?database=wos") + assert resp.status_code == 200 + body = resp.json() + assert "super-resolution" in body["data"]["formula"] + assert "STED" in body["data"]["formula"] + assert body["data"]["database"] == "wos" + assert body["data"]["keyword_count"] == 2 + assert "core_terms" in body["data"] + assert "sub_terms" in body["data"] + + @pytest.mark.asyncio + async def test_search_formula_scopus(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "microscopy", "term_en": "microscopy", "level": 1}, + ) + resp = await client.get(f"/api/v1/projects/{project_id}/keywords/search-formula?database=scopus") + assert resp.status_code == 200 + body = resp.json() + assert "TITLE-ABS-KEY" in body["data"]["formula"] + assert "microscopy" in body["data"]["formula"] + + @pytest.mark.asyncio + async def test_expand_keywords_mock(self, client: AsyncClient, project_id: int): + """Test keyword expansion with mock LLM.""" + resp = await client.post( + f"/api/v1/projects/{project_id}/keywords/expand", + json={ + "seed_terms": ["super-resolution microscopy"], + "language": "en", + "max_results": 10, + }, + ) + assert resp.status_code == 200 + body = resp.json() + assert "expanded_terms" in body["data"] + assert len(body["data"]["expanded_terms"]) > 0 + assert "term" in body["data"]["expanded_terms"][0] + assert "source" in body["data"] + + @pytest.mark.asyncio + async def test_keywords_nonexistent_project(self, client: AsyncClient): + resp = await client.get("/api/v1/projects/99999/keywords") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_update_nonexistent_keyword(self, client: AsyncClient, project_id: int): + resp = await client.put( + f"/api/v1/projects/{project_id}/keywords/99999", + json={"term": "Updated"}, + ) + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_nonexistent_keyword(self, client: AsyncClient, project_id: int): + resp = await client.delete(f"/api/v1/projects/{project_id}/keywords/99999") + assert resp.status_code == 404 + + +@real_llm +@pytest.mark.asyncio +async def test_expand_keywords_real_llm(client: AsyncClient, project_id: int): + """Test keyword expansion with real LLM — verifies non-empty content.""" + resp = await client.post( + f"/api/v1/projects/{project_id}/keywords/expand", + json={ + "seed_terms": ["machine learning"], + "language": "en", + "max_results": 5, + }, + ) + assert resp.status_code == 200 + body = resp.json() + assert len(body["data"]["expanded_terms"]) > 0 + assert all("term" in t for t in body["data"]["expanded_terms"]) + + +# ============================================================================= +# SEARCH API +# ============================================================================= + + +class TestSearchAPI: + """Tests for /api/v1/projects/{project_id}/search endpoints.""" + + @pytest.mark.asyncio + async def test_execute_search_with_query(self, client: AsyncClient, project_id: int): + mock_results = { + "papers": [ + { + "title": "Test Paper", + "doi": "10.1234/test", + "abstract": "Abstract", + "source": "openalex", + } + ], + "total": 1, + "source_stats": {"openalex": {"count": 1}}, + } + + with patch("app.api.v1.search.SearchService") as mock_svc_cls: + mock_svc = MagicMock() + mock_svc.search = AsyncMock(return_value=mock_results) + mock_svc_cls.return_value = mock_svc + + resp = await client.post( + f"/api/v1/projects/{project_id}/search/execute", + json={"query": "machine learning"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert body["data"]["total"] == 1 + assert body["data"]["papers"][0]["title"] == "Test Paper" + + @pytest.mark.asyncio + async def test_execute_search_from_keywords(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "microscopy", "term_en": "microscopy", "level": 1}, + ) + mock_results = { + "papers": [{"title": "Paper", "doi": "10.1/a", "abstract": ""}], + "total": 1, + "source_stats": {}, + } + with patch("app.api.v1.search.SearchService") as mock_svc_cls: + mock_svc = MagicMock() + mock_svc.search = AsyncMock(return_value=mock_results) + mock_svc_cls.return_value = mock_svc + + resp = await client.post( + f"/api/v1/projects/{project_id}/search/execute", + json={"query": ""}, + ) + assert resp.status_code == 200 + assert resp.json()["data"]["total"] == 1 + + @pytest.mark.asyncio + async def test_execute_search_no_query_no_keywords(self, client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/search/execute", + json={"query": ""}, + ) + assert resp.status_code == 400 + assert "no keywords" in resp.json()["message"].lower() + + @pytest.mark.asyncio + async def test_execute_search_with_sources(self, client: AsyncClient, project_id: int): + mock_results = { + "papers": [], + "total": 0, + "source_stats": {"semantic_scholar": {"count": 0}, "arxiv": {"count": 0}}, + } + with patch("app.api.v1.search.SearchService") as mock_svc_cls: + mock_svc = MagicMock() + mock_svc.search = AsyncMock(return_value=mock_results) + mock_svc_cls.return_value = mock_svc + + resp = await client.post( + f"/api/v1/projects/{project_id}/search/execute", + json={"query": "test", "sources": ["semantic_scholar", "arxiv"]}, + ) + assert resp.status_code == 200 + body = resp.json() + assert "source_stats" in body["data"] + + @pytest.mark.asyncio + async def test_list_search_sources(self, client: AsyncClient, project_id: int): + resp = await client.get(f"/api/v1/projects/{project_id}/search/sources") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + sources = body["data"] + assert len(sources) >= 4 + ids = [s["id"] for s in sources] + assert "semantic_scholar" in ids + assert "openalex" in ids + assert "arxiv" in ids + assert "crossref" in ids + + @pytest.mark.asyncio + async def test_search_nonexistent_project(self, client: AsyncClient): + resp = await client.post( + "/api/v1/projects/99999/search/execute", + json={"query": "test"}, + ) + assert resp.status_code == 404 + + +# ============================================================================= +# DEDUP API +# ============================================================================= + + +class TestDedupAPI: + """Tests for /api/v1/projects/{project_id}/dedup endpoints.""" + + @pytest.mark.asyncio + async def test_run_dedup_doi_only(self, client: AsyncClient, project_id: int): + for i in range(3): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": f"Paper {i}", "doi": "10.1234/same-doi"}, + ) + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/run", + params={"strategy": "doi_only"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["removed"] == 2 + assert body["data"]["remaining"] == 1 + + @pytest.mark.asyncio + async def test_run_dedup_title_only(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Exact Same Title"}, + ) + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Exact Same Title"}, + ) + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/run", + params={"strategy": "title_only"}, + ) + assert resp.status_code == 200 + assert resp.json()["data"]["removed"] == 1 + + @pytest.mark.asyncio + async def test_run_dedup_full(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "A", "doi": "10.1/dup"}, + ) + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "A", "doi": "10.1/dup"}, + ) + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Machine Learning"}, + ) + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Machine Learning Methods"}, + ) + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/run", + params={"strategy": "full"}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert "stage1_doi_removed" in data + assert "stage2_title_removed" in data + assert "stage3_candidates" in data + assert "total_remaining" in data + assert data["stage1_doi_removed"] == 1 + + @pytest.mark.asyncio + async def test_list_candidates_empty(self, client: AsyncClient, project_id: int): + resp = await client.get(f"/api/v1/projects/{project_id}/dedup/candidates") + assert resp.status_code == 200 + assert resp.json()["data"]["items"] == [] + + @pytest.mark.asyncio + async def test_list_candidates_with_similar_titles(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Machine Learning"}, + ) + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Machine Learning Methods"}, + ) + resp = await client.get(f"/api/v1/projects/{project_id}/dedup/candidates") + assert resp.status_code == 200 + candidates = resp.json()["data"]["items"] + assert len(candidates) >= 1 + assert "paper_a_id" in candidates[0] + assert "paper_b_id" in candidates[0] + assert "similarity" in candidates[0] + assert 0.80 <= candidates[0]["similarity"] < 0.90 + + @pytest.mark.asyncio + async def test_verify_duplicate_mock(self, client: AsyncClient, project_id: int): + p1 = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Paper One", "doi": "10.1111/a"}, + ) + p2 = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Paper Two", "doi": "10.2222/b"}, + ) + id_a = p1.json()["data"]["id"] + id_b = p2.json()["data"]["id"] + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/verify", + params={"paper_a_id": id_a, "paper_b_id": id_b}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert "is_duplicate" in data + assert "confidence" in data + assert "reason" in data + + @pytest.mark.asyncio + async def test_verify_duplicate_paper_not_found(self, client: AsyncClient, project_id: int): + p1 = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Only One"}, + ) + id_a = p1.json()["data"]["id"] + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/verify", + params={"paper_a_id": id_a, "paper_b_id": 99999}, + ) + assert resp.status_code == 200 + assert "error" in resp.json()["data"] + assert resp.json()["data"]["error"] == "Paper not found" + + @pytest.mark.asyncio + async def test_resolve_keep_old(self, client: AsyncClient, project_id: int): + paper_resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Test Paper"}, + ) + paper_id = paper_resp.json()["data"]["id"] + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/resolve", + json={"conflict_id": f"{paper_id}:dummy.pdf", "action": "keep_old"}, + ) + assert resp.status_code == 200 + assert resp.json()["data"]["action"] == "keep_old" + + @pytest.mark.asyncio + async def test_resolve_invalid_conflict_id(self, client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/resolve", + json={"conflict_id": "invalid", "action": "keep_old"}, + ) + assert resp.status_code == 400 + + @pytest.mark.asyncio + async def test_resolve_nonexistent_paper(self, client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/resolve", + json={"conflict_id": "999:nonexistent.pdf", "action": "keep_old"}, + ) + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_auto_resolve_empty(self, client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/auto-resolve", + json={"conflict_ids": []}, + ) + assert resp.status_code == 200 + assert resp.json()["data"] == [] + + @pytest.mark.asyncio + async def test_dedup_nonexistent_project(self, client: AsyncClient): + resp = await client.post("/api/v1/projects/99999/dedup/run") + assert resp.status_code == 404 + + resp = await client.get("/api/v1/projects/99999/dedup/candidates") + assert resp.status_code == 404 + + +@real_llm +@pytest.mark.asyncio +async def test_verify_duplicate_real_llm(client: AsyncClient, project_id: int): + """Test verify with real LLM — verifies non-empty reason.""" + p1 = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Deep Learning", "doi": "10.1/a"}, + ) + p2 = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Deep Learning", "doi": "10.1/a"}, + ) + id_a = p1.json()["data"]["id"] + id_b = p2.json()["data"]["id"] + resp = await client.post( + f"/api/v1/projects/{project_id}/dedup/verify", + params={"paper_a_id": id_a, "paper_b_id": id_b}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert "reason" in data + assert len(data["reason"]) > 0 diff --git a/backend/tests/test_api_projects_papers.py b/backend/tests/test_api_projects_papers.py new file mode 100644 index 0000000..be31974 --- /dev/null +++ b/backend/tests/test_api_projects_papers.py @@ -0,0 +1,584 @@ +"""Comprehensive API tests for Projects, Papers, and Upload modules.""" + +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.config import settings +from app.database import Base, engine +from app.main import app + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +async def setup_db(): + """Create tables before each test, drop after.""" + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def client(): + """Async HTTP client for in-process testing.""" + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.fixture +async def project_id(client: AsyncClient) -> int: + """Create a project and return its ID.""" + resp = await client.post("/api/v1/projects", json={"name": "Test Project", "domain": "optics"}) + assert resp.status_code == 201 + return resp.json()["data"]["id"] + + +def make_minimal_pdf() -> bytes: + """Create a minimal valid PDF that PyMuPDF can open.""" + import fitz + + doc = fitz.open() + doc.new_page() + pdf_bytes = doc.tobytes() + doc.close() + return pdf_bytes + + +@pytest.fixture +def minimal_pdf_bytes() -> bytes: + """Minimal valid PDF bytes for upload tests.""" + return make_minimal_pdf() + + +# --------------------------------------------------------------------------- +# Projects API +# --------------------------------------------------------------------------- + + +class TestProjectsAPI: + """Tests for /api/v1/projects endpoints.""" + + @pytest.mark.asyncio + async def test_list_projects_empty(self, client: AsyncClient): + resp = await client.get("/api/v1/projects") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert body["data"]["items"] == [] + assert body["data"]["total"] == 0 + assert body["data"]["page"] == 1 + assert body["data"]["page_size"] == 20 + + @pytest.mark.asyncio + async def test_list_projects_paginated(self, client: AsyncClient): + for i in range(5): + await client.post("/api/v1/projects", json={"name": f"Project {i}"}) + + resp = await client.get("/api/v1/projects?page=1&page_size=2") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert len(body["data"]["items"]) == 2 + assert body["data"]["total"] == 5 + assert body["data"]["page"] == 1 + assert body["data"]["page_size"] == 2 + assert body["data"]["total_pages"] == 3 + + @pytest.mark.asyncio + async def test_list_projects_page_2(self, client: AsyncClient): + for i in range(5): + await client.post("/api/v1/projects", json={"name": f"Project {i}"}) + + resp = await client.get("/api/v1/projects?page=2&page_size=2") + assert resp.status_code == 200 + body = resp.json() + assert len(body["data"]["items"]) == 2 + assert body["data"]["page"] == 2 + + @pytest.mark.asyncio + async def test_create_project(self, client: AsyncClient): + resp = await client.post( + "/api/v1/projects", + json={ + "name": "Super-Resolution Microscopy", + "description": "Literature review for SRM techniques", + "domain": "optics", + }, + ) + assert resp.status_code == 201 + body = resp.json() + assert body["code"] == 201 + assert body["data"]["name"] == "Super-Resolution Microscopy" + assert body["data"]["description"] == "Literature review for SRM techniques" + assert body["data"]["domain"] == "optics" + assert body["data"]["id"] > 0 + + @pytest.mark.asyncio + async def test_create_project_validation_error_empty_name(self, client: AsyncClient): + resp = await client.post("/api/v1/projects", json={"name": ""}) + assert resp.status_code == 422 + + @pytest.mark.asyncio + async def test_get_project(self, client: AsyncClient): + create_resp = await client.post("/api/v1/projects", json={"name": "Test Project"}) + project_id = create_resp.json()["data"]["id"] + + resp = await client.get(f"/api/v1/projects/{project_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["name"] == "Test Project" + assert body["data"]["paper_count"] == 0 + assert body["data"]["keyword_count"] == 0 + + @pytest.mark.asyncio + async def test_get_project_404(self, client: AsyncClient): + resp = await client.get("/api/v1/projects/99999") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_update_project(self, client: AsyncClient): + create_resp = await client.post("/api/v1/projects", json={"name": "Old Name"}) + project_id = create_resp.json()["data"]["id"] + + resp = await client.put( + f"/api/v1/projects/{project_id}", + json={"name": "New Name", "description": "Updated desc"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["name"] == "New Name" + assert body["data"]["description"] == "Updated desc" + + @pytest.mark.asyncio + async def test_update_project_partial(self, client: AsyncClient): + create_resp = await client.post( + "/api/v1/projects", + json={"name": "Original", "description": "Keep this"}, + ) + project_id = create_resp.json()["data"]["id"] + + resp = await client.put(f"/api/v1/projects/{project_id}", json={"name": "Only Name Changed"}) + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["name"] == "Only Name Changed" + assert body["data"]["description"] == "Keep this" + + @pytest.mark.asyncio + async def test_update_project_404(self, client: AsyncClient): + resp = await client.put("/api/v1/projects/99999", json={"name": "New Name"}) + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_project(self, client: AsyncClient): + create_resp = await client.post("/api/v1/projects", json={"name": "To Delete"}) + project_id = create_resp.json()["data"]["id"] + + resp = await client.delete(f"/api/v1/projects/{project_id}") + assert resp.status_code == 200 + assert resp.json().get("message") == "Project deleted" + + resp = await client.get(f"/api/v1/projects/{project_id}") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_project_404(self, client: AsyncClient): + resp = await client.delete("/api/v1/projects/99999") + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# Papers API +# --------------------------------------------------------------------------- + + +class TestPapersAPI: + """Tests for /api/v1/projects/{project_id}/papers endpoints.""" + + @pytest.mark.asyncio + async def test_list_papers_empty(self, client: AsyncClient, project_id: int): + resp = await client.get(f"/api/v1/projects/{project_id}/papers") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert body["data"]["items"] == [] + assert body["data"]["total"] == 0 + + @pytest.mark.asyncio + async def test_list_papers_paginated(self, client: AsyncClient, project_id: int): + for i in range(5): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": f"Paper {i}", "abstract": f"Abstract {i}"}, + ) + + resp = await client.get(f"/api/v1/projects/{project_id}/papers?page=1&page_size=2") + assert resp.status_code == 200 + body = resp.json() + assert len(body["data"]["items"]) == 2 + assert body["data"]["total"] == 5 + assert body["data"]["page"] == 1 + assert body["data"]["page_size"] == 2 + + @pytest.mark.asyncio + async def test_list_papers_search(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Machine Learning in Biology", "abstract": "ML techniques"}, + ) + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Deep Learning", "abstract": "Neural networks"}, + ) + + resp = await client.get(f"/api/v1/projects/{project_id}/papers?q=Biology") + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["total"] == 1 + assert "Biology" in body["data"]["items"][0]["title"] + + @pytest.mark.asyncio + async def test_list_papers_filter_status(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Paper 1", "abstract": "A1"}, + ) + resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Paper 2", "abstract": "A2"}, + ) + paper_id = resp.json()["data"]["id"] + + await client.put( + f"/api/v1/projects/{project_id}/papers/{paper_id}", + json={"status": "indexed"}, + ) + + resp = await client.get(f"/api/v1/projects/{project_id}/papers?status=indexed") + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["total"] == 1 + assert body["data"]["items"][0]["status"] == "indexed" + + @pytest.mark.asyncio + async def test_list_papers_filter_year(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Paper 2020", "abstract": "A", "year": 2020}, + ) + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Paper 2021", "abstract": "A", "year": 2021}, + ) + + resp = await client.get(f"/api/v1/projects/{project_id}/papers?year=2020") + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["total"] == 1 + assert body["data"]["items"][0]["year"] == 2020 + + @pytest.mark.asyncio + async def test_list_papers_sort(self, client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Alpha", "abstract": "A"}, + ) + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Beta", "abstract": "B"}, + ) + + resp = await client.get(f"/api/v1/projects/{project_id}/papers?sort_by=title&order=asc") + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["items"][0]["title"] == "Alpha" + + @pytest.mark.asyncio + async def test_create_paper(self, client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={ + "title": "Deep Learning for Microscopy", + "abstract": "We present a novel approach.", + "doi": "10.1234/test", + "year": 2024, + "journal": "Nature Methods", + }, + ) + assert resp.status_code == 201 + body = resp.json() + assert body["code"] == 201 + assert body["data"]["title"] == "Deep Learning for Microscopy" + assert body["data"]["doi"] == "10.1234/test" + assert body["data"]["year"] == 2024 + assert body["data"]["project_id"] == project_id + + @pytest.mark.asyncio + async def test_create_paper_validation_error_empty_title(self, client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "", "abstract": "A"}, + ) + assert resp.status_code == 422 + + @pytest.mark.asyncio + async def test_get_paper(self, client: AsyncClient, project_id: int): + create_resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Test Paper", "abstract": "Abstract"}, + ) + paper_id = create_resp.json()["data"]["id"] + + resp = await client.get(f"/api/v1/projects/{project_id}/papers/{paper_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["title"] == "Test Paper" + + @pytest.mark.asyncio + async def test_get_paper_404(self, client: AsyncClient, project_id: int): + resp = await client.get(f"/api/v1/projects/{project_id}/papers/99999") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_get_nonexistent_paper_404(self, client: AsyncClient, project_id: int): + """Request non-existent paper returns 404.""" + resp = await client.get(f"/api/v1/projects/{project_id}/papers/99999") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_list_paper_chunks_empty(self, client: AsyncClient, project_id: int): + """Get chunks for paper with no chunks returns empty list.""" + create_resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Paper Without Chunks", "abstract": "A"}, + ) + paper_id = create_resp.json()["data"]["id"] + + resp = await client.get(f"/api/v1/projects/{project_id}/papers/{paper_id}/chunks") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert body["data"]["items"] == [] + assert body["data"]["total"] == 0 + + @pytest.mark.asyncio + async def test_get_paper_wrong_project(self, client: AsyncClient, project_id: int): + other_resp = await client.post("/api/v1/projects", json={"name": "Other Project"}) + other_project_id = other_resp.json()["data"]["id"] + + create_resp = await client.post( + f"/api/v1/projects/{other_project_id}/papers", + json={"title": "Other Paper", "abstract": "A"}, + ) + paper_id = create_resp.json()["data"]["id"] + + resp = await client.get(f"/api/v1/projects/{project_id}/papers/{paper_id}") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_update_paper(self, client: AsyncClient, project_id: int): + create_resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Original", "abstract": "A"}, + ) + paper_id = create_resp.json()["data"]["id"] + + resp = await client.put( + f"/api/v1/projects/{project_id}/papers/{paper_id}", + json={"title": "Updated Title", "notes": "My notes"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["title"] == "Updated Title" + assert body["data"]["notes"] == "My notes" + + @pytest.mark.asyncio + async def test_update_paper_404(self, client: AsyncClient, project_id: int): + resp = await client.put( + f"/api/v1/projects/{project_id}/papers/99999", + json={"title": "Updated"}, + ) + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_paper(self, client: AsyncClient, project_id: int): + create_resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "To Delete", "abstract": "A"}, + ) + paper_id = create_resp.json()["data"]["id"] + + resp = await client.delete(f"/api/v1/projects/{project_id}/papers/{paper_id}") + assert resp.status_code == 200 + + resp = await client.get(f"/api/v1/projects/{project_id}/papers/{paper_id}") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_paper_404(self, client: AsyncClient, project_id: int): + resp = await client.delete(f"/api/v1/projects/{project_id}/papers/99999") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_serve_pdf_success(self, client: AsyncClient, project_id: int, minimal_pdf_bytes: bytes): + pdf_dir = Path(settings.pdf_dir) + project_pdf_dir = pdf_dir / str(project_id) + project_pdf_dir.mkdir(parents=True, exist_ok=True) + pdf_path = project_pdf_dir / "test_paper.pdf" + pdf_path.write_bytes(minimal_pdf_bytes) + + create_resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={ + "title": "Paper With PDF", + "abstract": "A", + }, + ) + paper_id = create_resp.json()["data"]["id"] + + from app.database import async_session_factory + from app.models import Paper + + async with async_session_factory() as session: + from sqlalchemy import select + + result = await session.execute(select(Paper).where(Paper.id == paper_id)) + paper = result.scalar_one() + paper.pdf_path = str(pdf_path) + await session.commit() + + resp = await client.get(f"/api/v1/projects/{project_id}/papers/{paper_id}/pdf") + assert resp.status_code == 200 + assert resp.headers.get("content-type") == "application/pdf" + assert resp.content == minimal_pdf_bytes + + @pytest.mark.asyncio + async def test_serve_pdf_not_found(self, client: AsyncClient, project_id: int): + create_resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "No PDF", "abstract": "A"}, + ) + paper_id = create_resp.json()["data"]["id"] + + resp = await client.get(f"/api/v1/projects/{project_id}/papers/{paper_id}/pdf") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_serve_pdf_paper_404(self, client: AsyncClient, project_id: int): + resp = await client.get(f"/api/v1/projects/{project_id}/papers/99999/pdf") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_list_papers_project_404(self, client: AsyncClient): + resp = await client.get("/api/v1/projects/99999/papers") + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# Upload API +# --------------------------------------------------------------------------- + + +class TestUploadAPI: + """Tests for /api/v1/projects/{project_id}/papers/upload endpoint.""" + + @pytest.mark.asyncio + async def test_upload_single_pdf(self, client: AsyncClient, project_id: int, minimal_pdf_bytes: bytes): + files = [("files", ("test.pdf", minimal_pdf_bytes, "application/pdf"))] + resp = await client.post( + f"/api/v1/projects/{project_id}/papers/upload", + files=files, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert body["data"]["total_uploaded"] == 1 + assert len(body["data"]["papers"]) == 1 + assert body["data"]["conflicts"] == [] + + @pytest.mark.asyncio + async def test_upload_multiple_pdfs(self, client: AsyncClient, project_id: int, minimal_pdf_bytes: bytes): + files = [ + ("files", ("paper1.pdf", minimal_pdf_bytes, "application/pdf")), + ("files", ("paper2.pdf", minimal_pdf_bytes, "application/pdf")), + ("files", ("paper3.pdf", minimal_pdf_bytes, "application/pdf")), + ] + resp = await client.post( + f"/api/v1/projects/{project_id}/papers/upload", + files=files, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert body["data"]["total_uploaded"] == 3 + assert len(body["data"]["papers"]) == 3 + + @pytest.mark.asyncio + async def test_upload_empty_file_422(self, client: AsyncClient, project_id: int): + files = [("files", ("empty.pdf", b"", "application/pdf"))] + resp = await client.post( + f"/api/v1/projects/{project_id}/papers/upload", + files=files, + ) + assert resp.status_code == 422 + body = resp.json() + assert "empty" in body.get("message", "").lower() + + @pytest.mark.asyncio + async def test_upload_file_exceeds_size_limit_413( + self, client: AsyncClient, project_id: int, minimal_pdf_bytes: bytes + ): + # Upload endpoint limits to 50MB + oversized = minimal_pdf_bytes + b"x" * (51 * 1024 * 1024) + files = [("files", ("huge.pdf", oversized, "application/pdf"))] + resp = await client.post( + f"/api/v1/projects/{project_id}/papers/upload", + files=files, + ) + assert resp.status_code == 413 + + @pytest.mark.asyncio + async def test_upload_non_pdf_skipped(self, client: AsyncClient, project_id: int, minimal_pdf_bytes: bytes): + files = [ + ("files", ("paper.pdf", minimal_pdf_bytes, "application/pdf")), + ("files", ("readme.txt", b"not a pdf", "text/plain")), + ] + resp = await client.post( + f"/api/v1/projects/{project_id}/papers/upload", + files=files, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["total_uploaded"] == 1 + assert len(body["data"]["papers"]) == 1 + + @pytest.mark.asyncio + async def test_upload_project_404(self, minimal_pdf_bytes: bytes): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + files = [("files", ("test.pdf", minimal_pdf_bytes, "application/pdf"))] + resp = await client.post( + "/api/v1/projects/99999/papers/upload", + files=files, + ) + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_upload_creates_papers_in_db(self, client: AsyncClient, project_id: int, minimal_pdf_bytes: bytes): + files = [("files", ("test.pdf", minimal_pdf_bytes, "application/pdf"))] + await client.post( + f"/api/v1/projects/{project_id}/papers/upload", + files=files, + ) + + resp = await client.get(f"/api/v1/projects/{project_id}/papers") + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["total"] == 1 + assert body["data"]["items"][0]["status"] == "pdf_downloaded" diff --git a/backend/tests/test_chat_pipeline.py b/backend/tests/test_chat_pipeline.py index c7f7f5c..1a9e5e6 100644 --- a/backend/tests/test_chat_pipeline.py +++ b/backend/tests/test_chat_pipeline.py @@ -319,3 +319,65 @@ async def test_stream_endpoint_persists_conversation(client): assert len(messages) == 2 assert messages[0]["role"] == "user" assert messages[1]["role"] == "assistant" + + +def _parse_sse_event_types(text: str) -> tuple[list[str], str | None]: + """Parse SSE text into event types and any error text.""" + lines = [line for line in text.split("\n") if line.startswith("data: ")] + event_types = [] + error_text = None + for line in lines: + payload = line.removeprefix("data: ").strip() + if payload == "[DONE]": + event_types.append("[DONE]") + continue + try: + parsed = json.loads(payload) + etype = parsed.get("type", "unknown") + event_types.append(etype) + if etype == "error": + error_text = parsed.get("errorText", "") + except json.JSONDecodeError: + pass + return event_types, error_text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tool_mode", ["citation_lookup", "review_outline", "gap_analysis"]) +async def test_stream_tool_mode(client, tool_mode): + """Each tool_mode should produce a valid SSE event sequence without errors.""" + resp = await client.post( + "/api/v1/chat/stream", + json={ + "message": "Analyze deep learning applications in biology", + "knowledge_base_ids": [], + "tool_mode": tool_mode, + }, + ) + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("text/event-stream") + + event_types, error_text = _parse_sse_event_types(resp.text) + assert error_text is None, f"Stream returned error for mode {tool_mode}: {error_text}" + assert "start" in event_types, f"Missing 'start' event for {tool_mode}" + assert "text-delta" in event_types, f"Missing 'text-delta' event for {tool_mode}" + assert "finish" in event_types, f"Missing 'finish' event for {tool_mode}" + assert "[DONE]" in event_types, f"Missing '[DONE]' event for {tool_mode}" + + +@pytest.mark.asyncio +async def test_stream_qa_mode_explicit(client): + """Explicit tool_mode='qa' should work the same as default.""" + resp = await client.post( + "/api/v1/chat/stream", + json={ + "message": "What is machine learning?", + "knowledge_base_ids": [], + "tool_mode": "qa", + }, + ) + assert resp.status_code == 200 + event_types, error_text = _parse_sse_event_types(resp.text) + assert error_text is None + assert "start" in event_types + assert "text-delta" in event_types diff --git a/backend/tests/test_citation_graph.py b/backend/tests/test_citation_graph.py index da55414..dd2870e 100644 --- a/backend/tests/test_citation_graph.py +++ b/backend/tests/test_citation_graph.py @@ -121,22 +121,26 @@ async def test_graph_returns_nodes_and_edges(self, project_with_paper): assert "center_id" in graph assert len(graph["nodes"]) >= 1 - async def test_graph_empty_when_no_s2_id(self, project_with_paper): + async def test_graph_raises_502_when_no_s2_id(self, project_with_paper): + from fastapi import HTTPException + from app.services.citation_graph_service import CitationGraphService info = project_with_paper async with async_session_factory() as session: svc = CitationGraphService(session) - with patch.object( - svc, - "_resolve_s2_id", - AsyncMock(return_value=None), + with ( + patch.object( + svc, + "_resolve_s2_id", + AsyncMock(return_value=None), + ), + pytest.raises(HTTPException) as exc_info, ): - graph = await svc.get_citation_graph(info["paper_id"], info["project_id"]) + await svc.get_citation_graph(info["paper_id"], info["project_id"]) - assert graph["nodes"] == [] - assert graph["edges"] == [] + assert exc_info.value.status_code == 502 class TestCitationGraphAPI: diff --git a/backend/tests/test_crawler.py b/backend/tests/test_crawler.py index 7da7696..1aaa3bf 100644 --- a/backend/tests/test_crawler.py +++ b/backend/tests/test_crawler.py @@ -151,7 +151,10 @@ async def mock_get(*args, **kwargs): resp.raise_for_status = MagicMock() return resp - with patch("app.services.crawler_service.httpx.AsyncClient") as mock_client_cls: + with ( + patch("app.services.crawler_service.httpx.AsyncClient") as mock_client_cls, + patch("app.services.url_validator.validate_url_safe", return_value="https://example.com/paper.pdf"), + ): mock_client = MagicMock() mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) @@ -195,7 +198,10 @@ async def mock_get(url, **kwargs): resp.status_code = 200 return resp - with patch("app.services.crawler_service.httpx.AsyncClient") as mock_client_cls: + with ( + patch("app.services.crawler_service.httpx.AsyncClient") as mock_client_cls, + patch("app.services.url_validator.validate_url_safe", side_effect=lambda url: url), + ): mock_client = MagicMock() mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) @@ -226,7 +232,10 @@ async def mock_get(*args, **kwargs): resp.raise_for_status = MagicMock() return resp - with patch("app.services.crawler_service.httpx.AsyncClient") as mock_client_cls: + with ( + patch("app.services.crawler_service.httpx.AsyncClient") as mock_client_cls, + patch("app.services.url_validator.validate_url_safe", side_effect=lambda url: url), + ): mock_client = MagicMock() mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) diff --git a/backend/tests/test_dedup.py b/backend/tests/test_dedup.py index 5458281..f70cd0c 100644 --- a/backend/tests/test_dedup.py +++ b/backend/tests/test_dedup.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, patch import pytest +from conftest import remove_paper_doi_unique_constraint from httpx import ASGITransport, AsyncClient from app.database import Base, engine @@ -14,6 +15,7 @@ @pytest.fixture(autouse=True) async def setup_db(): + remove_paper_doi_unique_constraint() async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) yield @@ -190,7 +192,8 @@ async def test_find_llm_dedup_candidates(client: AsyncClient, project_id: int): resp = await client.get(f"/api/v1/projects/{project_id}/dedup/candidates") assert resp.status_code == 200 - candidates = resp.json()["data"] + data = resp.json()["data"] + candidates = data["items"] assert len(candidates) >= 1 assert "paper_a_id" in candidates[0] assert "paper_b_id" in candidates[0] @@ -212,7 +215,7 @@ async def test_find_llm_candidates_empty_when_no_similar(client: AsyncClient, pr resp = await client.get(f"/api/v1/projects/{project_id}/dedup/candidates") assert resp.status_code == 200 - assert resp.json()["data"] == [] + assert resp.json()["data"]["items"] == [] # --- LLM verify (mock) --- @@ -260,7 +263,7 @@ async def test_llm_verify_duplicate_with_patched_response(client: AsyncClient, p mock_result = {"is_duplicate": True, "confidence": 0.95, "reason": "Same DOI and title"} - with patch("app.services.llm_client.LLMClient.chat_json", new_callable=AsyncMock) as mock_chat: + with patch("app.services.llm.client.LLMClient.chat_json", new_callable=AsyncMock) as mock_chat: mock_chat.return_value = mock_result resp = await client.post( @@ -323,7 +326,7 @@ async def test_run_dedup_nonexistent_project(client: AsyncClient): async def test_list_candidates_empty(client: AsyncClient, project_id: int): resp = await client.get(f"/api/v1/projects/{project_id}/dedup/candidates") assert resp.status_code == 200 - assert resp.json()["data"] == [] + assert resp.json()["data"]["items"] == [] @pytest.mark.asyncio diff --git a/backend/tests/test_e2e_live_server.py b/backend/tests/test_e2e_live_server.py new file mode 100644 index 0000000..131009c --- /dev/null +++ b/backend/tests/test_e2e_live_server.py @@ -0,0 +1,426 @@ +"""End-to-end tests against a live backend server with real LLM (Volcengine). + +Prerequisites: + - Backend running on E2E_BASE_URL (default http://localhost:8099) + - LLM_PROVIDER=volcengine with valid API key in .env + - Test PDFs in /data0/djx/omelette_pdf_test/ + +Run: + pytest tests/test_e2e_live_server.py -v -s --timeout=300 + +Skip when server is not available: + Tests auto-skip if the server is unreachable. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import time +from pathlib import Path + +import httpx +import pytest + +logger = logging.getLogger(__name__) + +E2E_BASE_URL = os.getenv("E2E_BASE_URL", "http://localhost:8099") +E2E_PDF_DIR = Path(os.getenv("E2E_PDF_DIR", "/data0/djx/omelette_pdf_test")) +E2E_TIMEOUT = 120 + + +def _server_available() -> bool: + try: + r = httpx.get(f"{E2E_BASE_URL}/", timeout=5) + return r.status_code == 200 + except Exception: + return False + + +pytestmark = [ + pytest.mark.skipif(not _server_available(), reason=f"Live server not reachable at {E2E_BASE_URL}"), + pytest.mark.e2e, +] + + +@pytest.fixture(scope="module") +def base_url(): + return E2E_BASE_URL + + +@pytest.fixture(scope="module") +def client(base_url): + with httpx.Client(base_url=base_url, timeout=E2E_TIMEOUT) as c: + yield c + + +@pytest.fixture(scope="module") +def async_client(base_url): + return httpx.AsyncClient(base_url=base_url, timeout=E2E_TIMEOUT) + + +@pytest.fixture(scope="module") +def pdf_files(): + if not E2E_PDF_DIR.exists(): + pytest.skip(f"PDF test directory not found: {E2E_PDF_DIR}") + files = sorted(E2E_PDF_DIR.glob("*.pdf")) + if not files: + pytest.skip(f"No PDFs found in {E2E_PDF_DIR}") + return files + + +@pytest.fixture(scope="module") +def e2e_project(client): + """Create a test project for the entire E2E module.""" + r = client.post( + "/api/v1/projects", + json={"name": "E2E Test Project", "description": "Automated E2E testing"}, + ) + assert r.status_code in (200, 201), f"Failed to create project: {r.text}" + data = r.json()["data"] + project_id = data["id"] + yield project_id + with contextlib.suppress(Exception): + client.delete(f"/api/v1/projects/{project_id}/rag/index") + client.delete(f"/api/v1/projects/{project_id}") + + +class TestHealthAndRoot: + def test_root(self, client): + r = client.get("/") + assert r.status_code == 200 + data = r.json() + assert data["data"]["name"] == "Omelette" + assert data["data"]["version"] == "0.1.0" + + def test_health(self, client): + r = client.get("/api/v1/settings/health") + assert r.status_code == 200 + + def test_docs(self, client): + r = client.get("/docs") + assert r.status_code == 200 + + +class TestProjectCRUD: + def test_create_and_list(self, client, e2e_project): + r = client.get("/api/v1/projects") + assert r.status_code == 200 + projects = r.json()["data"]["items"] + assert any(p["id"] == e2e_project for p in projects) + + def test_get_project(self, client, e2e_project): + r = client.get(f"/api/v1/projects/{e2e_project}") + assert r.status_code == 200 + assert r.json()["data"]["name"] == "E2E Test Project" + + +class TestPDFUploadAndProcessing: + def test_upload_single_pdf(self, client, e2e_project, pdf_files): + pdf = pdf_files[0] + with open(pdf, "rb") as f: + r = client.post( + f"/api/v1/projects/{e2e_project}/papers/upload", + files=[("files", (pdf.name, f, "application/pdf"))], + ) + assert r.status_code == 200, f"Upload failed: {r.text}" + data = r.json()["data"] + assert data["total_uploaded"] >= 1 + + def test_upload_multiple_pdfs(self, client, e2e_project, pdf_files): + upload_files = pdf_files[1:4] + with contextlib.ExitStack() as stack: + file_tuples = [] + for pdf in upload_files: + fh = stack.enter_context(open(pdf, "rb")) + file_tuples.append(("files", (pdf.name, fh, "application/pdf"))) + r = client.post( + f"/api/v1/projects/{e2e_project}/papers/upload", + files=file_tuples, + ) + assert r.status_code == 200 + data = r.json()["data"] + assert data["total_uploaded"] >= 1 + + def test_list_papers_after_upload(self, client, e2e_project): + r = client.get(f"/api/v1/projects/{e2e_project}/papers") + assert r.status_code == 200 + papers = r.json()["data"]["items"] + assert len(papers) >= 1, "Expected at least 1 paper after upload" + + def test_wait_for_processing(self, client, e2e_project): + """Poll until papers reach OCR_COMPLETE or INDEXED status (max 180s).""" + deadline = time.time() + 180 + while time.time() < deadline: + r = client.get(f"/api/v1/projects/{e2e_project}/papers?page_size=50") + papers = r.json()["data"]["items"] + if not papers: + time.sleep(2) + continue + statuses = {p["status"] for p in papers} + if statuses <= {"ocr_complete", "indexed"}: + return + time.sleep(5) + r = client.get(f"/api/v1/projects/{e2e_project}/papers?page_size=50") + papers = r.json()["data"]["items"] + statuses = {p["status"] for p in papers} + assert statuses <= {"ocr_complete", "indexed", "pdf_downloaded"}, ( + f"Papers not processed in time. Statuses: {statuses}" + ) + + +class TestMinerUParsing: + """Verify that papers are processed via MinerU when configured.""" + + def test_paper_chunks_have_sections(self, client, e2e_project): + """MinerU chunks should have section headings (unlike pdfplumber).""" + r = client.get(f"/api/v1/projects/{e2e_project}/papers?page_size=1") + papers = r.json()["data"]["items"] + if not papers: + pytest.skip("No papers available") + paper_id = papers[0]["id"] + r = client.get(f"/api/v1/projects/{e2e_project}/papers/{paper_id}/chunks") + if r.status_code != 200: + pytest.skip(f"Chunks endpoint returned {r.status_code}") + chunks = r.json().get("data", {}).get("items", r.json().get("data", [])) + if not chunks: + pytest.skip("No chunks for paper") + has_section = any(c.get("section", "") for c in chunks if isinstance(c, dict)) + has_formula = any(c.get("has_formula", False) for c in chunks if isinstance(c, dict)) + assert has_section or has_formula, "Expected MinerU-style chunks with sections or formulas" + + def test_upload_all_pdfs(self, client, e2e_project, pdf_files): + """Upload all 8 test PDFs to exercise concurrent processing.""" + with contextlib.ExitStack() as stack: + file_tuples = [] + for pdf in pdf_files: + fh = stack.enter_context(open(pdf, "rb")) + file_tuples.append(("files", (pdf.name, fh, "application/pdf"))) + r = client.post( + f"/api/v1/projects/{e2e_project}/papers/upload", + files=file_tuples, + ) + assert r.status_code == 200, f"Bulk upload failed: {r.text}" + data = r.json()["data"] + assert data["total_uploaded"] >= len(pdf_files) - 4 # some may already exist + + def test_wait_all_processed(self, client, e2e_project, pdf_files): + """Wait for all papers to reach indexed/ocr_complete (max 600s for MinerU).""" + deadline = time.time() + 600 + total_expected = len(pdf_files) + while time.time() < deadline: + r = client.get(f"/api/v1/projects/{e2e_project}/papers?page_size=50") + papers = r.json()["data"]["items"] + done_count = sum(1 for p in papers if p["status"] in ("ocr_complete", "indexed")) + if done_count >= total_expected: + return + in_progress = sum(1 for p in papers if p["status"] == "pdf_downloaded") + logger.info("Processing: %d done, %d in progress, %d total", done_count, in_progress, len(papers)) + time.sleep(10) + r = client.get(f"/api/v1/projects/{e2e_project}/papers?page_size=50") + papers = r.json()["data"]["items"] + statuses = {p["status"] for p in papers} + done = sum(1 for p in papers if p["status"] in ("ocr_complete", "indexed")) + assert done >= total_expected * 0.5, f"Only {done}/{total_expected} processed. Statuses: {statuses}" + + +class TestRAGIndexAndQuery: + def test_build_index(self, client, e2e_project): + r = client.post(f"/api/v1/projects/{e2e_project}/rag/index") + if r.status_code == 500 and "CUDA out of memory" in r.text: + logger.warning("CUDA OOM on first attempt, retrying after 30s...") + time.sleep(30) + r = client.post(f"/api/v1/projects/{e2e_project}/rag/index") + assert r.status_code == 200, f"RAG index build failed ({r.status_code}): {r.text[:300]}" + data = r.json()["data"] + assert data.get("indexed", 0) >= 0 + + def test_index_stats(self, client, e2e_project): + r = client.get(f"/api/v1/projects/{e2e_project}/rag/stats") + assert r.status_code == 200 + + def test_rag_query_with_real_llm(self, client, e2e_project): + for attempt in range(3): + r = client.post( + f"/api/v1/projects/{e2e_project}/rag/query", + json={ + "question": "What are the main applications of virtual reality in biological research?", + "top_k": 5, + "use_reranker": False, + "include_sources": True, + }, + ) + assert r.status_code == 200, f"RAG query failed: {r.text}" + data = r.json()["data"] + assert "answer" in data + if len(data["answer"]) > 10: + break + logger.warning("RAG query attempt %d returned empty answer, retrying...", attempt + 1) + time.sleep(5) + assert len(data["answer"]) > 10, ( + f"Answer too short after 3 attempts (sources={len(data.get('sources', []))}): '{data['answer']}'" + ) + + +class TestChatStream: + def test_chat_stream_basic(self, client, e2e_project): + """SSE streaming chat without knowledge base.""" + with client.stream( + "POST", + "/api/v1/chat/stream", + json={ + "message": "Hello, briefly describe what Omelette is.", + "tool_mode": "qa", + }, + ) as response: + assert response.status_code == 200 + content_type = response.headers.get("content-type", "") + assert "text/event-stream" in content_type + + events = [] + full_text = "" + for line in response.iter_lines(): + if line.startswith("data: "): + payload = line[6:] + try: + event = json.loads(payload) + events.append(event) + if event.get("type") == "text-delta": + full_text += event.get("textDelta", "") + except json.JSONDecodeError: + if payload.strip() == "[DONE]": + events.append({"type": "[DONE]"}) + + event_types = {e.get("type") for e in events} + assert len(events) > 0, "Expected at least one SSE event" + has_content = "text-delta" in event_types or "step-start" in event_types or "start" in event_types + assert has_content, f"Expected streaming events, got: {event_types}" + + def test_chat_stream_with_rag(self, client, e2e_project): + """SSE streaming chat with knowledge base.""" + with client.stream( + "POST", + "/api/v1/chat/stream", + json={ + "message": "Summarize the key findings about VR in molecular visualization.", + "knowledge_base_ids": [e2e_project], + "tool_mode": "qa", + "rag_top_k": 5, + "use_reranker": False, + }, + ) as response: + assert response.status_code == 200 + + events = [] + for line in response.iter_lines(): + if line.startswith("data: "): + payload = line[6:] + try: + event = json.loads(payload) + events.append(event) + except json.JSONDecodeError: + pass + + assert len(events) > 0, "Expected at least one SSE event" + + +class TestWritingAssistant: + def _get_paper_ids(self, client, project_id: int, limit: int = 3) -> list[int]: + r = client.get(f"/api/v1/projects/{project_id}/papers?page_size={limit}") + papers = r.json()["data"]["items"] + return [p["id"] for p in papers[:limit]] + + def test_summarize(self, client, e2e_project): + paper_ids = self._get_paper_ids(client, e2e_project) + if not paper_ids: + pytest.skip("No papers available for summarization") + r = client.post( + f"/api/v1/projects/{e2e_project}/writing/summarize", + json={"paper_ids": paper_ids, "language": "en"}, + ) + assert r.status_code == 200, f"Summarize failed: {r.text}" + data = r.json()["data"] + assert "summaries" in data or "summary" in data or "content" in data + + def test_citations(self, client, e2e_project): + paper_ids = self._get_paper_ids(client, e2e_project) + if not paper_ids: + pytest.skip("No papers available for citations") + r = client.post( + f"/api/v1/projects/{e2e_project}/writing/citations", + json={"paper_ids": paper_ids, "style": "gb_t_7714"}, + ) + assert r.status_code == 200, f"Citations failed: {r.text}" + + def test_review_outline(self, client, e2e_project): + r = client.post( + f"/api/v1/projects/{e2e_project}/writing/review-outline", + json={"topic": "Virtual reality applications in biological research", "language": "en"}, + ) + assert r.status_code == 200, f"Review outline failed: {r.text}" + + def test_gap_analysis(self, client, e2e_project): + r = client.post( + f"/api/v1/projects/{e2e_project}/writing/gap-analysis", + json={"research_topic": "VR-based tools for single-molecule visualization"}, + ) + assert r.status_code == 200, f"Gap analysis failed: {r.text}" + + +class TestConversationPersistence: + def test_create_conversation(self, client): + r = client.post( + "/api/v1/conversations", + json={"title": "E2E Test Conversation", "tool_mode": "qa"}, + ) + assert r.status_code in (200, 201) + assert r.json()["data"]["id"] > 0 + + def test_list_conversations(self, client): + r = client.get("/api/v1/conversations") + assert r.status_code == 200 + + def test_chat_creates_conversation(self, client, e2e_project): + """Verify that chatting without conversation_id creates one.""" + events = [] + with client.stream( + "POST", + "/api/v1/chat/stream", + json={"message": "Say hello", "tool_mode": "qa"}, + ) as response: + for line in response.iter_lines(): + if line.startswith("data: "): + with contextlib.suppress(json.JSONDecodeError): + events.append(json.loads(line[6:])) + + convo_events = [e for e in events if e.get("type") == "metadata"] + if convo_events: + assert "conversationId" in convo_events[0] or "conversation_id" in convo_events[0] + + +class TestSettingsAndTasks: + def test_get_settings(self, client): + r = client.get("/api/v1/settings") + assert r.status_code == 200 + + def test_list_tasks(self, client): + r = client.get("/api/v1/tasks") + assert r.status_code == 200 + + def test_list_llm_models(self, client): + r = client.get("/api/v1/settings/models") + assert r.status_code == 200 + + +class TestCleanup: + """Cleanup test — runs last to verify delete works.""" + + def test_delete_index(self, client, e2e_project): + r = client.delete(f"/api/v1/projects/{e2e_project}/rag/index") + assert r.status_code == 200 + + def test_project_still_accessible(self, client, e2e_project): + r = client.get(f"/api/v1/projects/{e2e_project}") + assert r.status_code == 200 diff --git a/backend/tests/test_e2e_quality.py b/backend/tests/test_e2e_quality.py new file mode 100644 index 0000000..d573aff --- /dev/null +++ b/backend/tests/test_e2e_quality.py @@ -0,0 +1,186 @@ +"""Quality comparison tests: MinerU vs pdfplumber extraction. + +Compares the same PDF processed by both methods and logs quality metrics. +No hard assertions on which is better — just records the comparison. + +Requires: MinerU service running + omelette conda env with pdfplumber. +Run: pytest tests/test_e2e_quality.py -v -s +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +import httpx +import pdfplumber +import pytest + +E2E_PDF_DIR = Path(os.getenv("E2E_PDF_DIR", "/data0/djx/omelette_pdf_test")) +MINERU_URL = os.getenv("MINERU_API_URL", "http://localhost:8010") + +logger = logging.getLogger(__name__) + + +def _mineru_reachable() -> bool: + try: + return httpx.get(f"{MINERU_URL}/docs", timeout=5).status_code == 200 + except Exception: + return False + + +pytestmark = [ + pytest.mark.skipif(not _mineru_reachable(), reason=f"MinerU not reachable at {MINERU_URL}"), + pytest.mark.e2e, +] + + +@pytest.fixture(scope="module") +def test_pdf() -> Path: + if not E2E_PDF_DIR.exists(): + pytest.skip(f"PDF test dir not found: {E2E_PDF_DIR}") + pdfs = sorted(E2E_PDF_DIR.glob("*.pdf")) + if not pdfs: + pytest.skip("No PDFs found") + return pdfs[0] + + +def _extract_pdfplumber(pdf_path: Path) -> dict: + """Extract text using pdfplumber (baseline).""" + pages = [] + total_chars = 0 + table_count = 0 + try: + with pdfplumber.open(pdf_path) as pdf: + for page in pdf.pages: + text = page.extract_text(x_tolerance=1) or "" + tables = page.extract_tables() or [] + pages.append(text) + total_chars += len(text) + table_count += len(tables) + except Exception as e: + return {"error": str(e)} + + full_text = "\n".join(pages) + return { + "method": "pdfplumber", + "total_chars": total_chars, + "page_count": len(pages), + "has_formulas": "$" in full_text, + "table_count": table_count, + "chunk_estimate": max(1, total_chars // 1024), + "sample": full_text[:500], + } + + +def _extract_mineru(pdf_path: Path) -> dict: + """Extract text using MinerU API.""" + try: + with open(pdf_path, "rb") as f: + r = httpx.post( + f"{MINERU_URL}/file_parse", + data={ + "backend": "pipeline", + "return_md": "true", + "return_content_list": "false", + "return_images": "false", + "formula_enable": "true", + "table_enable": "true", + }, + files={"files": (pdf_path.name, f, "application/pdf")}, + timeout=600, + ) + except Exception as e: + return {"error": str(e)} + + if r.status_code != 200: + return {"error": f"HTTP {r.status_code}"} + + body = r.json() + results = body.get("results", {}) + if not results: + return {"error": "empty results"} + + file_result = next(iter(results.values())) + md_content = file_result.get("md_content", "") + + return { + "method": "mineru", + "total_chars": len(md_content), + "has_formulas": "$" in md_content, + "table_count": md_content.count("|---|"), + "chunk_estimate": max(1, len(md_content) // 1024), + "version": body.get("version", "unknown"), + "sample": md_content[:500], + } + + +class TestMinerUVsPdfplumber: + def test_extraction_comparison(self, test_pdf): + """Compare MinerU and pdfplumber extraction quality for the same PDF.""" + logger.info("Testing PDF: %s", test_pdf.name) + + plumber = _extract_pdfplumber(test_pdf) + mineru = _extract_mineru(test_pdf) + + logger.info("=" * 60) + logger.info("QUALITY COMPARISON: %s", test_pdf.name) + logger.info("=" * 60) + + if plumber.get("error"): + logger.warning("pdfplumber failed: %s", plumber["error"]) + else: + logger.info( + "pdfplumber: %d chars, %d tables, formulas=%s, ~%d chunks", + plumber["total_chars"], + plumber["table_count"], + plumber["has_formulas"], + plumber["chunk_estimate"], + ) + + if mineru.get("error"): + logger.warning("MinerU failed: %s", mineru["error"]) + else: + logger.info( + "MinerU: %d chars, %d tables, formulas=%s, ~%d chunks (v%s)", + mineru["total_chars"], + mineru["table_count"], + mineru["has_formulas"], + mineru["chunk_estimate"], + mineru.get("version", "?"), + ) + + if not plumber.get("error") and not mineru.get("error"): + ratio = mineru["total_chars"] / max(plumber["total_chars"], 1) + logger.info("MinerU/pdfplumber char ratio: %.2f", ratio) + logger.info( + "MinerU formula detection: %s (pdfplumber: %s)", mineru["has_formulas"], plumber["has_formulas"] + ) + + assert not plumber.get("error") or not mineru.get("error"), "Both methods failed" + + def test_all_pdfs_comparison(self): + """Compare all PDFs and produce a summary table.""" + if not E2E_PDF_DIR.exists(): + pytest.skip(f"PDF test dir not found: {E2E_PDF_DIR}") + pdfs = sorted(E2E_PDF_DIR.glob("*.pdf")) + if not pdfs: + pytest.skip("No PDFs found") + + logger.info("\n" + "=" * 80) + logger.info("FULL COMPARISON TABLE") + logger.info("%-50s %10s %10s %8s", "PDF", "pdfplumber", "MinerU", "Ratio") + logger.info("-" * 80) + + for pdf in pdfs: + plumber = _extract_pdfplumber(pdf) + mineru = _extract_mineru(pdf) + + p_chars = plumber.get("total_chars", 0) if not plumber.get("error") else -1 + m_chars = mineru.get("total_chars", 0) if not mineru.get("error") else -1 + ratio = m_chars / max(p_chars, 1) if p_chars > 0 and m_chars > 0 else 0 + + logger.info("%-50s %10d %10d %8.2f", pdf.name[:50], p_chars, m_chars, ratio) + + logger.info("=" * 80) diff --git a/backend/tests/test_e2e_stress.py b/backend/tests/test_e2e_stress.py new file mode 100644 index 0000000..bb8dd0e --- /dev/null +++ b/backend/tests/test_e2e_stress.py @@ -0,0 +1,311 @@ +"""Stress tests for concurrent PDF processing, RAG queries, and chat streams. + +Requires a live backend server and MinerU service. +Run: pytest tests/test_e2e_stress.py -v -s +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import subprocess +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +import httpx +import pytest + +logger = logging.getLogger(__name__) + +E2E_BASE_URL = os.getenv("E2E_BASE_URL", "http://localhost:8099") +E2E_PDF_DIR = Path(os.getenv("E2E_PDF_DIR", "/data0/djx/omelette_pdf_test")) +E2E_TIMEOUT = 600 + + +def _server_available() -> bool: + try: + return httpx.get(f"{E2E_BASE_URL}/", timeout=5).status_code == 200 + except Exception: + return False + + +pytestmark = [ + pytest.mark.skipif(not _server_available(), reason=f"Live server not reachable at {E2E_BASE_URL}"), + pytest.mark.e2e, +] + + +@pytest.fixture(scope="module") +def client(): + with httpx.Client(base_url=E2E_BASE_URL, timeout=E2E_TIMEOUT) as c: + yield c + + +@pytest.fixture(scope="module") +def pdf_files(): + if not E2E_PDF_DIR.exists(): + pytest.skip(f"PDF test directory not found: {E2E_PDF_DIR}") + files = sorted(E2E_PDF_DIR.glob("*.pdf")) + if not files: + pytest.skip(f"No PDFs found in {E2E_PDF_DIR}") + return files + + +@pytest.fixture(scope="module") +def stress_project(client, pdf_files): + """Create project, upload all PDFs, wait for processing.""" + r = client.post( + "/api/v1/projects", + json={"name": "Stress Test Project", "description": "Parallel processing stress test"}, + ) + assert r.status_code in (200, 201) + project_id = r.json()["data"]["id"] + + with contextlib.ExitStack() as stack: + file_tuples = [] + for pdf in pdf_files: + fh = stack.enter_context(open(pdf, "rb")) + file_tuples.append(("files", (pdf.name, fh, "application/pdf"))) + r = client.post( + f"/api/v1/projects/{project_id}/papers/upload", + files=file_tuples, + ) + assert r.status_code == 200, f"Upload failed: {r.text}" + uploaded = r.json()["data"]["total_uploaded"] + logger.info("Uploaded %d PDFs to project %d", uploaded, project_id) + + t0 = time.monotonic() + deadline = time.time() + 900 + while time.time() < deadline: + r = client.get(f"/api/v1/projects/{project_id}/papers?page_size=50") + papers = r.json()["data"]["items"] + done = sum(1 for p in papers if p["status"] in ("ocr_complete", "indexed")) + if done >= len(pdf_files): + break + time.sleep(15) + elapsed = time.monotonic() - t0 + logger.info("All papers processed in %.1fs", elapsed) + + yield project_id, elapsed + + with contextlib.suppress(Exception): + client.delete(f"/api/v1/projects/{project_id}/rag/index") + client.delete(f"/api/v1/projects/{project_id}") + + +class TestConcurrentUploadAndProcess: + def test_all_papers_processed(self, client, stress_project, pdf_files): + project_id, elapsed = stress_project + r = client.get(f"/api/v1/projects/{project_id}/papers?page_size=50") + papers = r.json()["data"]["items"] + done = sum(1 for p in papers if p["status"] in ("ocr_complete", "indexed")) + assert done >= len(pdf_files), f"Only {done}/{len(pdf_files)} papers completed" + logger.info( + "Processing time for %d PDFs: %.1fs (%.1fs per PDF)", len(pdf_files), elapsed, elapsed / len(pdf_files) + ) + + def test_processing_speed_reasonable(self, stress_project, pdf_files): + """Parallel processing should be faster than 120s per PDF on average.""" + _, elapsed = stress_project + avg_per_pdf = elapsed / len(pdf_files) + logger.info("Average processing time: %.1fs per PDF", avg_per_pdf) + assert avg_per_pdf < 120, f"Average {avg_per_pdf:.1f}s/PDF is too slow" + + +class TestConcurrentRAGQueries: + def test_build_index(self, client, stress_project): + project_id, _ = stress_project + r = client.post(f"/api/v1/projects/{project_id}/rag/index") + if r.status_code == 500 and "CUDA out of memory" in r.text: + logger.warning("CUDA OOM on first attempt, retrying after 30s...") + time.sleep(30) + r = client.post(f"/api/v1/projects/{project_id}/rag/index") + assert r.status_code == 200, f"RAG index build failed ({r.status_code}): {r.text[:300]}" + + def test_concurrent_rag_queries(self, client, stress_project): + project_id, _ = stress_project + questions = [ + "What are the applications of VR in molecular visualization?", + "How does vLUME handle 3D single-molecule data?", + "What deep learning methods are used with VR for brain cell analysis?", + "How is cloud computing leveraged for image annotation?", + "What VR headset designs exist for mouse neuroscience?", + ] + + results = [] + t0 = time.monotonic() + + def _query(q: str) -> dict: + with httpx.Client(base_url=E2E_BASE_URL, timeout=E2E_TIMEOUT) as c: + r = c.post( + f"/api/v1/projects/{project_id}/rag/query", + json={"question": q, "top_k": 5, "use_reranker": False}, + ) + return { + "status": r.status_code, + "question": q, + "answer_len": len(r.json().get("data", {}).get("answer", "")) if r.status_code == 200 else 0, + } + + with ThreadPoolExecutor(max_workers=5) as pool: + futures = {pool.submit(_query, q): q for q in questions} + for future in as_completed(futures): + results.append(future.result()) + + elapsed = time.monotonic() - t0 + success = sum(1 for r in results if r["status"] == 200) + logger.info("Concurrent RAG queries: %d/%d succeeded in %.1fs", success, len(questions), elapsed) + + assert success >= 3, f"Only {success}/{len(questions)} queries succeeded" + for r in results: + if r["status"] == 200: + assert r["answer_len"] > 10, f"Empty answer for: {r['question']}" + + +class TestConcurrentChatStreams: + def test_concurrent_chat_streams(self, client, stress_project): + project_id, _ = stress_project + prompts = [ + "Briefly describe VR applications in biology.", + "What is single-molecule microscopy?", + "Explain deep learning for brain cell analysis.", + ] + + def _stream_chat(msg: str) -> dict: + events = [] + with ( + httpx.Client(base_url=E2E_BASE_URL, timeout=E2E_TIMEOUT) as c, + c.stream( + "POST", + "/api/v1/chat/stream", + json={"message": msg, "tool_mode": "qa"}, + ) as response, + ): + for line in response.iter_lines(): + if line.startswith("data: "): + with contextlib.suppress(json.JSONDecodeError): + events.append(json.loads(line[6:])) + return {"prompt": msg, "event_count": len(events), "status": "ok" if events else "empty"} + + results = [] + t0 = time.monotonic() + + with ThreadPoolExecutor(max_workers=3) as pool: + futures = {pool.submit(_stream_chat, p): p for p in prompts} + for future in as_completed(futures): + results.append(future.result()) + + elapsed = time.monotonic() - t0 + success = sum(1 for r in results if r["status"] == "ok") + logger.info("Concurrent chat streams: %d/%d succeeded in %.1fs", success, len(prompts), elapsed) + + assert success >= 2, f"Only {success}/{len(prompts)} streams produced events" + + +def _nvidia_smi_snapshot() -> list[dict]: + """Take a snapshot of GPU utilization via nvidia-smi.""" + try: + result = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=index,utilization.gpu,memory.used,memory.total", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode != 0: + return [] + gpus = [] + for line in result.stdout.strip().splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) >= 4: + gpus.append( + { + "index": int(parts[0]), + "gpu_util": int(parts[1]), + "mem_used_mb": int(parts[2]), + "mem_total_mb": int(parts[3]), + } + ) + return gpus + except (FileNotFoundError, subprocess.TimeoutExpired): + return [] + + +class TestGPUUtilization: + """Verify that multiple GPUs are utilised during parallel processing.""" + + def test_gpu_utilization_during_processing(self, client, pdf_files): + """Upload 8 PDFs and sample GPU utilization during processing.""" + gpus_before = _nvidia_smi_snapshot() + if not gpus_before: + pytest.skip("nvidia-smi not available") + + logger.info("GPU baseline before upload:") + for g in gpus_before: + logger.info( + " GPU %d: util=%d%%, mem=%d/%dMB", g["index"], g["gpu_util"], g["mem_used_mb"], g["mem_total_mb"] + ) + + r = client.post( + "/api/v1/projects", + json={"name": "GPU Stress Project", "description": "GPU utilization test"}, + ) + assert r.status_code in (200, 201) + project_id = r.json()["data"]["id"] + + try: + with contextlib.ExitStack() as stack: + file_tuples = [] + for pdf in pdf_files: + fh = stack.enter_context(open(pdf, "rb")) + file_tuples.append(("files", (pdf.name, fh, "application/pdf"))) + r = client.post( + f"/api/v1/projects/{project_id}/papers/upload", + files=file_tuples, + ) + assert r.status_code == 200 + + peak_samples = [] + deadline = time.time() + 600 + while time.time() < deadline: + snapshot = _nvidia_smi_snapshot() + if snapshot: + peak_samples.append(snapshot) + active = [g for g in snapshot if g["gpu_util"] > 5 or g["mem_used_mb"] > 500] + if active: + for g in active: + logger.info( + " Active GPU %d: util=%d%%, mem=%d/%dMB", + g["index"], + g["gpu_util"], + g["mem_used_mb"], + g["mem_total_mb"], + ) + + r = client.get(f"/api/v1/projects/{project_id}/papers?page_size=50") + papers = r.json()["data"]["items"] + done = sum(1 for p in papers if p["status"] in ("ocr_complete", "indexed")) + if done >= len(pdf_files): + break + time.sleep(5) + + if peak_samples: + all_gpu_indices = set() + for sample in peak_samples: + for g in sample: + if g["gpu_util"] > 5 or g["mem_used_mb"] > 500: + all_gpu_indices.add(g["index"]) + + logger.info("GPUs that showed activity: %s", sorted(all_gpu_indices)) + logger.info("Total GPU snapshots: %d", len(peak_samples)) + + finally: + with contextlib.suppress(Exception): + client.delete(f"/api/v1/projects/{project_id}/rag/index") + client.delete(f"/api/v1/projects/{project_id}") diff --git a/backend/tests/test_embedding.py b/backend/tests/test_embedding.py index 290b7c6..7b77119 100644 --- a/backend/tests/test_embedding.py +++ b/backend/tests/test_embedding.py @@ -10,9 +10,11 @@ @pytest.fixture(autouse=True) def reset_embedding_cache(): """Clear cached embedding model between tests.""" - embedding_service._cached_embed_model = None + from app.services.gpu_model_manager import gpu_model_manager + + gpu_model_manager.unload("embedding") yield - embedding_service._cached_embed_model = None + gpu_model_manager.unload("embedding") class TestGetEmbeddingModel: @@ -67,9 +69,9 @@ def test_detect_gpu_returns_tuple(self): assert isinstance(has_gpu, bool) assert isinstance(count, int) assert isinstance(device, str) - assert device in ("cuda", "cpu") + assert device.startswith("cuda") or device == "cpu" def test_detect_gpu_no_raise(self): """detect_gpu never raises (handles missing torch).""" has_gpu, count, device = embedding_service.detect_gpu() - assert device in ("cuda", "cpu") + assert device.startswith("cuda") or device == "cpu" diff --git a/backend/tests/test_gpu_api.py b/backend/tests/test_gpu_api.py new file mode 100644 index 0000000..1be559f --- /dev/null +++ b/backend/tests/test_gpu_api.py @@ -0,0 +1,106 @@ +"""Tests for GPU monitoring API endpoints.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.main import app + + +@pytest.fixture() +def mock_managers(): + with ( + patch("app.services.gpu_model_manager.gpu_model_manager") as mock_gpu, + patch("app.services.mineru_process_manager.mineru_process_manager") as mock_mineru, + ): + mock_gpu.get_status.return_value = [] + mock_gpu.loaded_model_names = [] + mock_gpu.unload_all.return_value = None + mock_mineru.get_status.return_value = { + "status": "stopped", + "pid": None, + "port": 8010, + } + yield mock_gpu, mock_mineru + + +@pytest.mark.asyncio +async def test_gpu_status_no_models(mock_managers): + mock_gpu, mock_mineru = mock_managers + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + resp = await ac.get("/api/v1/gpu/status") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["models"] == [] + assert data["mineru"]["status"] == "stopped" + + +@pytest.mark.asyncio +async def test_gpu_status_with_models(mock_managers): + mock_gpu, _ = mock_managers + mock_gpu.get_status.return_value = [ + { + "name": "embedding", + "model_name": "Qwen/Qwen3-Embedding-0.6B", + "loaded": True, + "device": "cuda:0", + "idle_seconds": 30.5, + "ttl_remaining_seconds": 269.5, + } + ] + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + resp = await ac.get("/api/v1/gpu/status") + assert resp.status_code == 200 + models = resp.json()["data"]["models"] + assert len(models) == 1 + assert models[0]["name"] == "embedding" + assert models[0]["loaded"] is True + + +@pytest.mark.asyncio +async def test_gpu_status_mineru_running(mock_managers): + _, mock_mineru = mock_managers + mock_mineru.get_status.return_value = { + "status": "running", + "pid": 12345, + "port": 8010, + "idle_seconds": 10.0, + "ttl_remaining_seconds": 590.0, + } + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + resp = await ac.get("/api/v1/gpu/status") + assert resp.status_code == 200 + mineru = resp.json()["data"]["mineru"] + assert mineru["status"] == "running" + assert mineru["pid"] == 12345 + + +@pytest.mark.asyncio +async def test_gpu_unload(mock_managers): + mock_gpu, _ = mock_managers + mock_gpu.loaded_model_names = ["embedding", "reranker"] + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + resp = await ac.post("/api/v1/gpu/unload") + assert resp.status_code == 200 + data = resp.json()["data"] + assert "unloaded" in data + mock_gpu.unload_all.assert_called_once() + + +@pytest.mark.asyncio +async def test_gpu_unload_empty(mock_managers): + mock_gpu, _ = mock_managers + mock_gpu.loaded_model_names = [] + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + resp = await ac.post("/api/v1/gpu/unload") + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["unloaded"] == [] diff --git a/backend/tests/test_gpu_mode_config.py b/backend/tests/test_gpu_mode_config.py new file mode 100644 index 0000000..dd16a8c --- /dev/null +++ b/backend/tests/test_gpu_mode_config.py @@ -0,0 +1,147 @@ +"""Tests for GPU_MODE preset system and per-service config resolution.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from app.config import GPU_MODE_PRESETS, GpuMode, Settings +from app.services.paper_processor import _parse_ocr_gpu_ids, _resolve_parallel_limit + + +class TestGpuModePresets: + """Verify the three preset modes fill correct defaults.""" + + def test_conservative_defaults(self): + with patch.dict("os.environ", {"GPU_MODE": "conservative"}, clear=False): + s = Settings(_env_file=None) + assert s.gpu_mode == GpuMode.CONSERVATIVE + assert s.embed_batch_size == 1 + assert s.rerank_batch_size == 1 + assert s.ocr_parallel_limit == 1 + assert s.reranker_concurrency_limit == 1 + + def test_balanced_defaults(self): + with patch.dict("os.environ", {"GPU_MODE": "balanced"}, clear=False): + s = Settings(_env_file=None) + assert s.gpu_mode == GpuMode.BALANCED + assert s.embed_batch_size == 8 + assert s.rerank_batch_size == 16 + + def test_aggressive_defaults(self): + with patch.dict("os.environ", {"GPU_MODE": "aggressive"}, clear=False): + s = Settings(_env_file=None) + assert s.gpu_mode == GpuMode.AGGRESSIVE + assert s.embed_batch_size == 32 + assert s.rerank_batch_size == 50 + assert s.reranker_concurrency_limit == 2 + + +class TestUserOverride: + """User-set values take priority over GPU_MODE presets.""" + + def test_override_embed_batch(self): + with patch.dict("os.environ", {"GPU_MODE": "conservative", "EMBED_BATCH_SIZE": "16"}, clear=False): + s = Settings(_env_file=None) + assert s.embed_batch_size == 16 + assert s.rerank_batch_size == 1 + + def test_override_rerank_batch(self): + with patch.dict("os.environ", {"GPU_MODE": "aggressive", "RERANK_BATCH_SIZE": "8"}, clear=False): + s = Settings(_env_file=None) + assert s.rerank_batch_size == 8 + assert s.embed_batch_size == 32 + + def test_override_ocr_parallel(self): + with patch.dict("os.environ", {"GPU_MODE": "conservative", "OCR_PARALLEL_LIMIT": "4"}, clear=False): + s = Settings(_env_file=None) + assert s.ocr_parallel_limit == 4 + + +class TestGpuPinDefaults: + """GPU pin fields default to -1 (auto).""" + + def test_auto_defaults(self): + s = Settings(_env_file=None) + assert s.embed_gpu_id == -1 + assert s.rerank_gpu_id == -1 + assert s.ocr_gpu_ids == "" + + def test_explicit_pin(self): + with patch.dict("os.environ", {"EMBED_GPU_ID": "0", "RERANK_GPU_ID": "1"}, clear=False): + s = Settings(_env_file=None) + assert s.embed_gpu_id == 0 + assert s.rerank_gpu_id == 1 + + +class TestPresetCompleteness: + """Every preset defines all required keys.""" + + @pytest.mark.parametrize("mode", list(GpuMode)) + def test_all_keys_present(self, mode: GpuMode): + preset = GPU_MODE_PRESETS[mode] + expected_keys = {"ocr_parallel_limit", "embed_batch_size", "rerank_batch_size", "reranker_concurrency_limit"} + assert set(preset.keys()) == expected_keys + + +class TestParseOcrGpuIds: + """Verify OCR_GPU_IDS parsing.""" + + def test_empty_string_uses_all(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_gpu_ids = "" + result = _parse_ocr_gpu_ids(3) + assert result == [0, 1, 2] + + def test_single_id(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_gpu_ids = "1" + result = _parse_ocr_gpu_ids(3) + assert result == [1] + + def test_multiple_ids(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_gpu_ids = "0,2" + result = _parse_ocr_gpu_ids(3) + assert result == [0, 2] + + def test_out_of_range_filtered(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_gpu_ids = "0,5,1" + result = _parse_ocr_gpu_ids(3) + assert result == [0, 1] + + def test_all_invalid_falls_back(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_gpu_ids = "10,20" + result = _parse_ocr_gpu_ids(3) + assert result == [0, 1, 2] + + def test_cpu_only(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_gpu_ids = "" + result = _parse_ocr_gpu_ids(0) + assert result == [0] + + +class TestResolveParallelLimitAggressiveMode: + """Aggressive mode doubles the auto parallel limit.""" + + def test_aggressive_doubles(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_parallel_limit = 0 + mock_settings.gpu_mode = GpuMode.AGGRESSIVE + assert _resolve_parallel_limit(2) == 4 + + def test_balanced_no_double(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_parallel_limit = 0 + mock_settings.gpu_mode = GpuMode.BALANCED + assert _resolve_parallel_limit(2) == 2 + + def test_explicit_override_ignores_mode(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_parallel_limit = 3 + mock_settings.gpu_mode = GpuMode.AGGRESSIVE + assert _resolve_parallel_limit(2) == 3 diff --git a/backend/tests/test_gpu_model_manager.py b/backend/tests/test_gpu_model_manager.py new file mode 100644 index 0000000..84e0702 --- /dev/null +++ b/backend/tests/test_gpu_model_manager.py @@ -0,0 +1,173 @@ +"""Tests for GPUModelManager TTL-based model lifecycle.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import patch + +import pytest + + +@pytest.fixture() +def manager(): + from app.services.gpu_model_manager import GPUModelManager + + return GPUModelManager(ttl_seconds=2, check_interval=1) + + +def test_acquire_loads_model(manager): + calls = [] + + def loader(): + calls.append(1) + return "fake_model" + + model = manager.acquire("test", loader, model_name="m1", device="cpu") + assert model == "fake_model" + assert len(calls) == 1 + + +def test_acquire_returns_cached(manager): + calls = [] + + def loader(): + calls.append(1) + return "fake_model" + + m1 = manager.acquire("test", loader) + m2 = manager.acquire("test", loader) + assert m1 is m2 + assert len(calls) == 1 + + +def test_acquire_force_reload(manager): + calls = [] + + def loader(): + calls.append(1) + return f"model_{len(calls)}" + + m1 = manager.acquire("test", loader) + assert m1 == "model_1" + + with patch("app.services.gpu_utils.gc"): + m2 = manager.acquire("test", loader, force_reload=True) + assert m2 == "model_2" + assert len(calls) == 2 + + +def test_unload_removes_model(manager): + manager.acquire("test", lambda: "m") + assert manager.is_loaded("test") + + with patch("app.services.gpu_utils.gc"): + manager.unload("test") + assert not manager.is_loaded("test") + + +def test_unload_all(manager): + manager.acquire("a", lambda: "m1") + manager.acquire("b", lambda: "m2") + assert len(manager.loaded_model_names) == 2 + + with patch("app.services.gpu_utils.gc"): + manager.unload_all() + assert len(manager.loaded_model_names) == 0 + + +def test_touch_updates_timestamp(manager): + manager.acquire("test", lambda: "m") + entry = manager._models["test"] + old_ts = entry.last_used_at + time.sleep(0.05) + manager.touch("test") + assert entry.last_used_at > old_ts + + +def test_touch_nonexistent_is_noop(manager): + manager.touch("nonexistent") + + +def test_get_status_empty(manager): + assert manager.get_status() == [] + + +def test_get_status_with_models(manager): + manager.acquire("test", lambda: "m", model_name="TestModel", device="cuda:0") + status = manager.get_status() + assert len(status) == 1 + assert status[0]["name"] == "test" + assert status[0]["model_name"] == "TestModel" + assert status[0]["loaded"] is True + assert status[0]["device"] == "cuda:0" + assert "idle_seconds" in status[0] + assert "ttl_remaining_seconds" in status[0] + + +@pytest.mark.asyncio +async def test_ttl_expires_unloads(manager): + manager.acquire("test", lambda: "m") + assert manager.is_loaded("test") + + with patch("app.services.gpu_utils.gc"): + await manager.start() + await asyncio.sleep(3.5) + await manager.stop() + + assert not manager.is_loaded("test") + + +@pytest.mark.asyncio +async def test_acquire_resets_ttl(manager): + manager.acquire("test", lambda: "m") + + with patch("app.services.gpu_utils.gc"): + await manager.start() + await asyncio.sleep(1.5) + manager.acquire("test", lambda: "m2") + await asyncio.sleep(1.5) + assert manager.is_loaded("test") + await manager.stop() + + +def test_ttl_zero_disables_cleanup(): + from app.services.gpu_model_manager import GPUModelManager + + mgr = GPUModelManager(ttl_seconds=0, check_interval=1) + status = mgr.get_status() + assert status == [] + + +@pytest.mark.asyncio +async def test_ttl_zero_no_cleanup_task(): + from app.services.gpu_model_manager import GPUModelManager + + mgr = GPUModelManager(ttl_seconds=0, check_interval=1) + await mgr.start() + assert mgr._cleanup_task is None + await mgr.stop() + + +def test_concurrent_acquire_single_load(manager): + import threading + + calls = [] + barrier = threading.Barrier(3) + + def loader(): + calls.append(1) + time.sleep(0.1) + return "model" + + def worker(): + barrier.wait() + manager.acquire("shared", loader) + + threads = [threading.Thread(target=worker) for _ in range(3)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(calls) == 1 diff --git a/backend/tests/test_integration.py b/backend/tests/test_integration.py index 94036bc..9d12d30 100644 --- a/backend/tests/test_integration.py +++ b/backend/tests/test_integration.py @@ -1,6 +1,7 @@ """End-to-end integration test simulating the full Omelette workflow.""" import pytest +from conftest import remove_paper_doi_unique_constraint from httpx import ASGITransport, AsyncClient from app.database import Base, engine @@ -9,6 +10,7 @@ @pytest.fixture(autouse=True) async def setup_db(): + remove_paper_doi_unique_constraint() async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) yield diff --git a/backend/tests/test_keywords.py b/backend/tests/test_keywords.py index 27e9e0e..31ecf6d 100644 --- a/backend/tests/test_keywords.py +++ b/backend/tests/test_keywords.py @@ -58,7 +58,8 @@ async def test_list_keywords_empty(client: AsyncClient, project_id: int): assert resp.status_code == 200 body = resp.json() assert body["code"] == 200 - assert body["data"] == [] + assert body["data"]["items"] == [] + assert body["data"]["total"] == 0 @pytest.mark.asyncio @@ -80,8 +81,8 @@ async def test_list_keywords_by_level(client: AsyncClient, project_id: int): assert resp.status_code == 200 body = resp.json() assert body["code"] == 200 - assert len(body["data"]) == 2 - assert all(k["level"] == 1 for k in body["data"]) + assert len(body["data"]["items"]) == 2 + assert all(k["level"] == 1 for k in body["data"]["items"]) @pytest.mark.asyncio @@ -114,7 +115,7 @@ async def test_delete_keyword(client: AsyncClient, project_id: int): assert resp.status_code == 200 list_resp = await client.get(f"/api/v1/projects/{project_id}/keywords") - assert len(list_resp.json()["data"]) == 0 + assert len(list_resp.json()["data"]["items"]) == 0 @pytest.mark.asyncio @@ -130,7 +131,7 @@ async def test_bulk_create_keywords(client: AsyncClient, project_id: int): assert body["data"]["created"] == 3 list_resp = await client.get(f"/api/v1/projects/{project_id}/keywords") - assert len(list_resp.json()["data"]) == 3 + assert len(list_resp.json()["data"]["items"]) == 3 @pytest.mark.asyncio diff --git a/backend/tests/test_llm_config_resolver.py b/backend/tests/test_llm_config_resolver.py new file mode 100644 index 0000000..82dd513 --- /dev/null +++ b/backend/tests/test_llm_config_resolver.py @@ -0,0 +1,119 @@ +"""Tests for LLMConfigResolver — ensures consistent LLM config resolution.""" + +import pytest + +from app.config import settings +from app.services.llm_config_resolver import LLMConfigResolver + + +class TestFromEnv: + def test_default_uses_mock(self): + config = LLMConfigResolver.from_env() + assert config.provider == settings.llm_provider + + def test_override_provider(self): + config = LLMConfigResolver.from_env(provider="volcengine") + assert config.provider == "volcengine" + assert config.model == settings.volcengine_model + assert config.base_url == settings.volcengine_base_url + + def test_override_model(self): + config = LLMConfigResolver.from_env(provider="openai", model="gpt-4o") + assert config.model == "gpt-4o" + assert config.provider == "openai" + + def test_override_temperature_and_max_tokens(self): + config = LLMConfigResolver.from_env(temperature=0.1, max_tokens=100) + assert config.temperature == 0.1 + assert config.max_tokens == 100 + + def test_defaults_from_settings(self): + config = LLMConfigResolver.from_env() + assert config.temperature == settings.llm_temperature + assert config.max_tokens == settings.llm_max_tokens + + @pytest.mark.parametrize( + "provider,expected_base_url_attr", + [ + ("openai", ""), + ("anthropic", ""), + ("aliyun", "aliyun_base_url"), + ("volcengine", "volcengine_base_url"), + ("ollama", "ollama_base_url"), + ], + ) + def test_provider_base_urls(self, provider, expected_base_url_attr): + config = LLMConfigResolver.from_env(provider=provider) + expected = getattr(settings, expected_base_url_attr, "") if expected_base_url_attr else "" + assert config.base_url == expected + + def test_unknown_provider_returns_empty_fields(self): + config = LLMConfigResolver.from_env(provider="nonexistent") + assert config.provider == "nonexistent" + assert config.api_key == "" + assert config.base_url == "" + assert config.model == "mock-model" + + def test_mock_provider(self): + config = LLMConfigResolver.from_env(provider="mock") + assert config.provider == "mock" + assert config.api_key == "" + assert config.model == "mock-model" + + +class TestFromMerged: + def _make_merged(self, **overrides): + defaults = { + "llm_provider": "mock", + "llm_model": "", + "openai_api_key": "", + "openai_model": "", + "anthropic_api_key": "", + "anthropic_model": "", + "aliyun_api_key": "", + "aliyun_base_url": "", + "aliyun_model": "", + "volcengine_api_key": "test-key", + "volcengine_base_url": "https://ark.test", + "volcengine_model": "doubao-test", + "ollama_base_url": "", + "ollama_model": "", + } + defaults.update(overrides) + + class FakeMerged: + pass + + obj = FakeMerged() + for k, v in defaults.items(): + setattr(obj, k, v) + return obj + + def test_basic_mock_provider(self): + merged = self._make_merged() + config = LLMConfigResolver.from_merged(merged) + assert config.provider == "mock" + assert config.model == "mock-model" + + def test_volcengine_from_merged(self): + merged = self._make_merged(llm_provider="volcengine") + config = LLMConfigResolver.from_merged(merged) + assert config.provider == "volcengine" + assert config.api_key == "test-key" + assert config.base_url == "https://ark.test" + assert config.model == "doubao-test" + + def test_llm_model_override(self): + merged = self._make_merged(llm_provider="volcengine", llm_model="custom-model") + config = LLMConfigResolver.from_merged(merged) + assert config.model == "custom-model" + + def test_temperature_override(self): + merged = self._make_merged() + config = LLMConfigResolver.from_merged(merged, temperature=0.2) + assert config.temperature == 0.2 + + def test_max_tokens_override(self): + merged = self._make_merged() + config = LLMConfigResolver.from_merged(merged, max_tokens=256) + assert config.max_tokens == 256 diff --git a/backend/tests/test_llm_settings.py b/backend/tests/test_llm_settings.py index d190311..c48e4de 100644 --- a/backend/tests/test_llm_settings.py +++ b/backend/tests/test_llm_settings.py @@ -103,16 +103,16 @@ async def test_client_from_config(): assert isinstance(result, str) -# --- Backward-compatibility via old import path --- +# --- Verify direct import path works --- @pytest.mark.asyncio -async def test_backward_compat_import(): - from app.services.llm_client import LLMClient as OldLLMClient - from app.services.llm_client import get_llm_client as old_get +async def test_direct_import_path(): + from app.services.llm.client import LLMClient as DirectLLMClient + from app.services.llm.client import get_llm_client as direct_get - assert OldLLMClient is LLMClient - assert old_get is get_llm_client + assert DirectLLMClient is LLMClient + assert direct_get is get_llm_client # --- Settings API tests --- diff --git a/backend/tests/test_mcp.py b/backend/tests/test_mcp.py index a18eade..9ae2811 100644 --- a/backend/tests/test_mcp.py +++ b/backend/tests/test_mcp.py @@ -1,9 +1,15 @@ """Tests for MCP server tools and resources.""" +from unittest.mock import AsyncMock, patch + import pytest from app.database import Base, async_session_factory, engine from app.mcp_server import ( + add_paper_by_doi, + analyze_gaps, + find_citations, + generate_review_outline, get_kb_detail, get_paper_chunks, get_paper_resource, @@ -11,6 +17,10 @@ list_kb_resource, list_knowledge_bases, lookup_paper, + manage_keywords, + search_knowledge_base, + search_papers_by_keyword, + summarize_papers, ) from app.models import Paper, Project @@ -128,3 +138,210 @@ async def test_get_paper_resource(sample_kb): async def test_get_paper_chunks_empty(sample_kb): result = await get_paper_chunks(paper_id=sample_kb["paper_id"]) assert "No chunks found" in result or '"chunks": []' in result + + +# --- New Tool Tests --- + + +@pytest.mark.asyncio +@patch("app.services.rag_service.RAGService") +async def test_search_knowledge_base_success(mock_rag_cls, sample_kb): + mock_rag = AsyncMock() + mock_rag.query.return_value = { + "answer": "Deep learning excels at NLP tasks.", + "sources": [ + {"paper_title": "Deep Learning for NLP", "page_number": 1, "relevance_score": 0.95, "excerpt": "A survey."} + ], + } + mock_rag_cls.return_value = mock_rag + + result = await search_knowledge_base(query="deep learning NLP", kb_id=sample_kb["project_id"]) + assert "Deep learning excels" in result + assert "Deep Learning for NLP" in result + assert "0.95" in result + + +@pytest.mark.asyncio +async def test_search_knowledge_base_invalid_top_k(sample_kb): + result = await search_knowledge_base(query="test", kb_id=sample_kb["project_id"], top_k=0) + assert "Error" in result + + result = await search_knowledge_base(query="test", kb_id=sample_kb["project_id"], top_k=51) + assert "Error" in result + + +@pytest.mark.asyncio +@patch("app.services.rag_service.RAGService") +async def test_find_citations_success(mock_rag_cls, sample_kb): + mock_rag = AsyncMock() + mock_rag.query.return_value = { + "answer": "", + "sources": [ + {"paper_title": "Deep Learning for NLP", "relevance_score": 0.88, "page_number": 3, "excerpt": "Methods."} + ], + } + mock_rag_cls.return_value = mock_rag + + result = await find_citations(text="Deep learning methods for text analysis", kb_id=sample_kb["project_id"]) + assert "Potential Citations" in result + assert "Deep Learning for NLP" in result + + +@pytest.mark.asyncio +@patch("app.services.rag_service.RAGService") +async def test_find_citations_empty(mock_rag_cls, sample_kb): + mock_rag = AsyncMock() + mock_rag.query.return_value = {"answer": "", "sources": []} + mock_rag_cls.return_value = mock_rag + + result = await find_citations(text="unrelated topic", kb_id=sample_kb["project_id"]) + assert "No potential citation" in result + + +@pytest.mark.asyncio +@patch("app.mcp_server._fetch_crossref_metadata") +async def test_add_paper_by_doi_success(mock_crossref, sample_kb): + mock_crossref.return_value = { + "title": "New Paper", + "authors": [{"name": "Jane Doe"}], + "year": 2025, + "journal": "Science", + "abstract": "Abstract text.", + } + result = await add_paper_by_doi(doi="10.5678/newpaper", kb_id=sample_kb["project_id"]) + assert "Paper Added" in result + assert "New Paper" in result + + +@pytest.mark.asyncio +@patch("app.mcp_server._fetch_crossref_metadata") +async def test_add_paper_by_doi_duplicate(mock_crossref, sample_kb): + result = await add_paper_by_doi(doi="10.1234/test", kb_id=sample_kb["project_id"]) + assert "already exists" in result + mock_crossref.assert_not_called() + + +@pytest.mark.asyncio +async def test_add_paper_by_doi_invalid_doi(sample_kb): + result = await add_paper_by_doi(doi="invalid-doi", kb_id=sample_kb["project_id"]) + assert "Error" in result + + +@pytest.mark.asyncio +async def test_add_paper_by_doi_kb_not_found(): + result = await add_paper_by_doi(doi="10.1234/valid", kb_id=99999) + assert "not found" in result + + +@pytest.mark.asyncio +@patch("app.services.search_service.SearchService") +async def test_search_papers_by_keyword_success(mock_search_cls): + mock_svc = AsyncMock() + mock_svc.search.return_value = { + "papers": [ + {"title": "Paper A", "authors": [{"name": "Auth1"}], "year": 2024, "doi": "10.1/a", "source": "arxiv"} + ], + "total": 1, + } + mock_search_cls.return_value = mock_svc + + result = await search_papers_by_keyword(query="machine learning") + assert "Paper A" in result + assert "Auth1" in result + + +@pytest.mark.asyncio +async def test_search_papers_by_keyword_invalid_max_results(): + result = await search_papers_by_keyword(query="test", max_results=0) + assert "Error" in result + + result = await search_papers_by_keyword(query="test", max_results=101) + assert "Error" in result + + +@pytest.mark.asyncio +@patch("app.services.writing_service.WritingService") +async def test_summarize_papers(mock_writing_cls, sample_kb): + mock_svc = AsyncMock() + mock_svc.summarize.return_value = {"content": "This is a summary of the papers."} + mock_writing_cls.return_value = mock_svc + + result = await summarize_papers(kb_id=sample_kb["project_id"]) + assert "Summary" in result + assert "summary of the papers" in result + + +@pytest.mark.asyncio +@patch("app.services.writing_service.WritingService") +async def test_generate_review_outline(mock_writing_cls, sample_kb): + mock_svc = AsyncMock() + mock_svc.generate_review_outline.return_value = {"outline": "1. Introduction\n2. Methods\n3. Results"} + mock_writing_cls.return_value = mock_svc + + result = await generate_review_outline(kb_id=sample_kb["project_id"], topic="deep learning NLP") + assert "Review Outline" in result + assert "Introduction" in result + + +@pytest.mark.asyncio +@patch("app.services.writing_service.WritingService") +async def test_analyze_gaps(mock_writing_cls, sample_kb): + mock_svc = AsyncMock() + mock_svc.analyze_gaps.return_value = {"analysis": "Gap 1: Limited multimodal studies."} + mock_writing_cls.return_value = mock_svc + + result = await analyze_gaps(kb_id=sample_kb["project_id"], research_topic="VR in biology") + assert "Gap Analysis" in result + assert "multimodal" in result + + +@pytest.mark.asyncio +async def test_manage_keywords_list_empty(sample_kb): + result = await manage_keywords(kb_id=sample_kb["project_id"], action="list") + assert "No keywords" in result + + +@pytest.mark.asyncio +async def test_manage_keywords_add_and_list(sample_kb): + add_result = await manage_keywords(kb_id=sample_kb["project_id"], action="add", term="deep learning") + assert "Added" in add_result + + list_result = await manage_keywords(kb_id=sample_kb["project_id"], action="list") + assert "deep learning" in list_result + + +@pytest.mark.asyncio +async def test_manage_keywords_delete(sample_kb): + await manage_keywords(kb_id=sample_kb["project_id"], action="add", term="to_delete") + delete_result = await manage_keywords(kb_id=sample_kb["project_id"], action="delete", term="to_delete") + assert "Deleted" in delete_result + + +@pytest.mark.asyncio +async def test_manage_keywords_delete_not_found(sample_kb): + result = await manage_keywords(kb_id=sample_kb["project_id"], action="delete", term="nonexistent") + assert "not found" in result + + +@pytest.mark.asyncio +async def test_manage_keywords_invalid_action(sample_kb): + result = await manage_keywords(kb_id=sample_kb["project_id"], action="invalid") + assert "Error" in result + + +@pytest.mark.asyncio +async def test_manage_keywords_add_requires_term(sample_kb): + result = await manage_keywords(kb_id=sample_kb["project_id"], action="add") + assert "Error" in result + + +@pytest.mark.asyncio +@patch("app.services.keyword_service.KeywordService") +async def test_manage_keywords_expand(mock_kw_cls, sample_kb): + mock_svc = AsyncMock() + mock_svc.expand_keywords.return_value = {"expanded_terms": [{"term": "neural networks", "relation": "synonym"}]} + mock_kw_cls.return_value = mock_svc + + result = await manage_keywords(kb_id=sample_kb["project_id"], action="expand", term="deep learning") + assert "Expanded" in result + assert "neural networks" in result diff --git a/backend/tests/test_middleware.py b/backend/tests/test_middleware.py new file mode 100644 index 0000000..cbde093 --- /dev/null +++ b/backend/tests/test_middleware.py @@ -0,0 +1,110 @@ +"""Tests for auth middleware and health check.""" + +from unittest.mock import patch + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.config import settings +from app.database import Base, engine +from app.main import app + + +@pytest.fixture(autouse=True) +async def setup_db(): + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.mark.asyncio +async def test_no_api_key_config_passes_all(client): + with patch.object(settings, "api_secret_key", ""): + resp = await client.get("/api/v1/projects") + assert resp.status_code == 200 + + +@pytest.mark.asyncio +async def test_missing_api_key_returns_401(client): + with patch.object(settings, "api_secret_key", "test-key-123"): + resp = await client.get("/api/v1/projects") + assert resp.status_code == 401 + assert resp.json()["code"] == 401 + assert "API key" in resp.json()["message"] + + +@pytest.mark.asyncio +async def test_valid_header_api_key_passes(client): + with patch.object(settings, "api_secret_key", "test-key-123"): + resp = await client.get("/api/v1/projects", headers={"X-API-Key": "test-key-123"}) + assert resp.status_code == 200 + + +@pytest.mark.asyncio +async def test_invalid_api_key_returns_401(client): + with patch.object(settings, "api_secret_key", "test-key-123"): + resp = await client.get("/api/v1/projects", headers={"X-API-Key": "wrong-key"}) + assert resp.status_code == 401 + + +@pytest.mark.asyncio +async def test_exempt_paths_no_auth(client): + with patch.object(settings, "api_secret_key", "test-key-123"): + for path in ["/health", "/docs", "/"]: + resp = await client.get(path) + assert resp.status_code == 200, f"Failed for path {path}" + + +@pytest.mark.asyncio +async def test_options_no_auth(client): + with patch.object(settings, "api_secret_key", "test-key-123"): + resp = await client.options( + "/api/v1/projects", + headers={"Origin": "http://localhost:3000", "Access-Control-Request-Method": "GET"}, + ) + assert resp.status_code == 200 + + +@pytest.mark.asyncio +async def test_query_param_api_key_rejected(client): + with patch.object(settings, "api_secret_key", "test-key-123"): + resp = await client.get("/api/v1/projects?api_key=test-key-123") + assert resp.status_code == 401 + + +@pytest.mark.asyncio +async def test_health_endpoint_returns_ok(client): + resp = await client.get("/health") + assert resp.status_code == 200 + data = resp.json() + assert data["code"] == 200 + assert data["data"]["status"] == "ok" + + +@pytest.mark.asyncio +async def test_error_format_consistent(client): + resp = await client.get("/api/v1/projects/99999") + assert resp.status_code == 404 + data = resp.json() + assert data["code"] == 404 + assert "message" in data + assert data["data"] is None + + +@pytest.mark.asyncio +async def test_validation_error_format(client): + resp = await client.post("/api/v1/projects", json={}) + assert resp.status_code == 422 + data = resp.json() + assert data["code"] == 422 + assert data["message"] == "Validation error" + assert isinstance(data["data"], list) diff --git a/backend/tests/test_mineru_client.py b/backend/tests/test_mineru_client.py new file mode 100644 index 0000000..0b1445b --- /dev/null +++ b/backend/tests/test_mineru_client.py @@ -0,0 +1,199 @@ +"""Tests for MinerU API client. + +Unit tests run with mocked HTTP; E2E tests require a running MinerU service. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from app.services.mineru_client import MinerUClient + +E2E_PDF_DIR = Path(os.getenv("E2E_PDF_DIR", "/data0/djx/omelette_pdf_test")) +MINERU_URL = os.getenv("MINERU_API_URL", "http://localhost:8010") + + +def _mineru_reachable() -> bool: + try: + return httpx.get(f"{MINERU_URL}/docs", timeout=5).status_code == 200 + except Exception: + return False + + +# --------------------------------------------------------------------------- +# Unit tests (mocked HTTP) +# --------------------------------------------------------------------------- + + +class TestMinerUClientUnit: + """Unit tests — no real MinerU service needed.""" + + @pytest.mark.asyncio + async def test_health_check_success(self): + mock_resp = MagicMock() + mock_resp.status_code = 200 + + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_resp) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("httpx.AsyncClient", return_value=mock_client): + client = MinerUClient(base_url="http://fake:8010") + assert await client.health_check() is True + + @pytest.mark.asyncio + async def test_health_check_failure(self): + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=httpx.ConnectError("Connection refused")) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("httpx.AsyncClient", return_value=mock_client): + client = MinerUClient(base_url="http://fake:8010") + assert await client.health_check() is False + + @pytest.mark.asyncio + async def test_parse_pdf_file_not_found(self): + client = MinerUClient(base_url="http://fake:8010") + result = await client.parse_pdf("/nonexistent/path.pdf") + assert "error" in result + assert "not found" in result["error"].lower() + + @pytest.mark.asyncio + async def test_parse_pdf_timeout(self, tmp_path): + pdf = tmp_path / "test.pdf" + pdf.write_bytes(b"%PDF-1.4 fake content") + + mock_client = AsyncMock() + mock_client.post = AsyncMock(side_effect=httpx.TimeoutException("timeout")) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("httpx.AsyncClient", return_value=mock_client): + client = MinerUClient(base_url="http://fake:8010", timeout=10) + result = await client.parse_pdf(pdf) + assert "error" in result + assert "timeout" in result["error"].lower() + + @pytest.mark.asyncio + async def test_parse_pdf_connect_error(self, tmp_path): + pdf = tmp_path / "test.pdf" + pdf.write_bytes(b"%PDF-1.4 fake content") + + mock_client = AsyncMock() + mock_client.post = AsyncMock(side_effect=httpx.ConnectError("refused")) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("httpx.AsyncClient", return_value=mock_client): + client = MinerUClient(base_url="http://fake:8010") + result = await client.parse_pdf(pdf) + assert "error" in result + assert "connect" in result["error"].lower() + + @pytest.mark.asyncio + async def test_parse_pdf_api_error_status(self, tmp_path): + pdf = tmp_path / "test.pdf" + pdf.write_bytes(b"%PDF-1.4 fake content") + + mock_resp = MagicMock() + mock_resp.status_code = 500 + mock_resp.text = "Internal Server Error" + + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_resp) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("httpx.AsyncClient", return_value=mock_client): + client = MinerUClient(base_url="http://fake:8010") + result = await client.parse_pdf(pdf) + assert "error" in result + assert "500" in result["error"] + + @pytest.mark.asyncio + async def test_parse_pdf_success(self, tmp_path): + pdf = tmp_path / "test.pdf" + pdf.write_bytes(b"%PDF-1.4 fake content") + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = { + "backend": "pipeline", + "version": "2.7.6", + "results": { + "test.pdf": { + "md_content": "# Title\n\nSome extracted text from the PDF.", + "content_list": [], + } + }, + } + + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_resp) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("httpx.AsyncClient", return_value=mock_client): + client = MinerUClient(base_url="http://fake:8010") + result = await client.parse_pdf(pdf) + assert "error" not in result + assert "md_content" in result + assert len(result["md_content"]) > 10 + assert result["backend"] == "pipeline" + assert result["version"] == "2.7.6" + + +# --------------------------------------------------------------------------- +# E2E tests (require running MinerU service) +# --------------------------------------------------------------------------- + + +@pytest.mark.e2e +@pytest.mark.skipif(not _mineru_reachable(), reason=f"MinerU not reachable at {MINERU_URL}") +class TestMinerUClientE2E: + """Integration tests against a live MinerU service.""" + + @pytest.mark.asyncio + async def test_health_check_live(self): + client = MinerUClient(base_url=MINERU_URL) + assert await client.health_check() is True + + @pytest.mark.asyncio + async def test_parse_real_pdf(self): + if not E2E_PDF_DIR.exists(): + pytest.skip(f"PDF test dir not found: {E2E_PDF_DIR}") + pdfs = sorted(E2E_PDF_DIR.glob("*.pdf")) + if not pdfs: + pytest.skip("No PDFs found") + + pdf = pdfs[0] + client = MinerUClient(base_url=MINERU_URL, timeout=600) + result = await client.parse_pdf(pdf, start_page=0, end_page=2) + + assert "error" not in result, f"MinerU parse failed: {result.get('error')}" + assert "md_content" in result + assert len(result["md_content"]) > 50, "Extracted content too short" + assert "backend" in result + assert "version" in result + + @pytest.mark.asyncio + async def test_parse_returns_backend_and_version(self): + if not E2E_PDF_DIR.exists(): + pytest.skip(f"PDF test dir not found: {E2E_PDF_DIR}") + pdfs = sorted(E2E_PDF_DIR.glob("*.pdf")) + if not pdfs: + pytest.skip("No PDFs found") + + pdf = pdfs[0] + client = MinerUClient(base_url=MINERU_URL, timeout=600) + result = await client.parse_pdf(pdf, start_page=0, end_page=0) + + assert result.get("backend") in ("pipeline", "hybrid-auto-engine", "vlm-auto-engine") + assert result.get("version", "") != "" diff --git a/backend/tests/test_mineru_process_manager.py b/backend/tests/test_mineru_process_manager.py new file mode 100644 index 0000000..9819f61 --- /dev/null +++ b/backend/tests/test_mineru_process_manager.py @@ -0,0 +1,145 @@ +"""Tests for MinerUProcessManager subprocess lifecycle.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +@pytest.fixture() +def _disable_auto_manage(): + with patch("app.services.mineru_process_manager.settings") as mock_settings: + mock_settings.mineru_auto_manage = False + mock_settings.mineru_ttl_seconds = 600 + mock_settings.mineru_api_url = "http://localhost:8010" + mock_settings.mineru_conda_env = "mineru" + mock_settings.mineru_startup_timeout = 10 + mock_settings.mineru_gpu_ids = "" + mock_settings.cuda_visible_devices = "6,7" + yield mock_settings + + +@pytest.fixture() +def _enable_auto_manage(): + with patch("app.services.mineru_process_manager.settings") as mock_settings: + mock_settings.mineru_auto_manage = True + mock_settings.mineru_ttl_seconds = 600 + mock_settings.mineru_api_url = "http://localhost:8010" + mock_settings.mineru_conda_env = "mineru" + mock_settings.mineru_startup_timeout = 5 + mock_settings.mineru_gpu_ids = "" + mock_settings.cuda_visible_devices = "6,7" + yield mock_settings + + +@pytest.fixture() +def manager(): + from app.services.mineru_process_manager import MinerUProcessManager + + return MinerUProcessManager() + + +@pytest.mark.asyncio +async def test_auto_manage_false_skips(manager, _disable_auto_manage): + result = await manager.ensure_running() + assert result is False + + +@pytest.mark.asyncio +async def test_ensure_running_detects_external(manager, _enable_auto_manage): + with patch.object(manager, "_health_check", new_callable=AsyncMock, return_value=True): + result = await manager.ensure_running() + assert result is True + assert manager._is_external is True + + +@pytest.mark.asyncio +async def test_ensure_running_starts_subprocess(manager, _enable_auto_manage): + health_results = [False, False, True] + + async def mock_health(*args, **kwargs): + return health_results.pop(0) if health_results else True + + with ( + patch.object(manager, "_health_check", side_effect=mock_health), + patch.object(manager, "_start_subprocess", return_value=True) as mock_start, + ): + mock_process = MagicMock() + mock_process.poll.return_value = None + manager._process = mock_process + + result = await manager.ensure_running() + + assert result is True + mock_start.assert_called_once() + + +@pytest.mark.asyncio +async def test_start_failure_returns_false(manager, _enable_auto_manage): + with ( + patch.object(manager, "_health_check", new_callable=AsyncMock, return_value=False), + patch.object(manager, "_start_subprocess", return_value=False), + ): + result = await manager.ensure_running() + assert result is False + + +def test_get_status_stopped(manager): + status = manager.get_status() + assert status["status"] == "stopped" + assert status["pid"] is None + + +def test_get_status_running(manager): + mock_process = MagicMock() + mock_process.poll.return_value = None + mock_process.pid = 12345 + manager._process = mock_process + manager._last_used_at = 1000.0 + + with patch("time.monotonic", return_value=1050.0): + status = manager.get_status() + + assert status["status"] == "running" + assert status["pid"] == 12345 + + +def test_get_status_external(manager): + manager._is_external = True + status = manager.get_status() + assert status["status"] == "external" + + +@pytest.mark.asyncio +async def test_stop_kills_subprocess(manager): + mock_process = MagicMock() + mock_process.poll.return_value = None + mock_process.pid = 99999 + mock_process.wait.return_value = 0 + manager._process = mock_process + + await manager._kill_process() + + mock_process.send_signal.assert_called_once() + assert manager._process is None + + +@pytest.mark.asyncio +async def test_start_stop_lifecycle(manager, _enable_auto_manage): + await manager.start() + assert manager._cleanup_task is not None + await manager.stop() + assert manager._cleanup_task is None + + +def test_touch_updates_timestamp(manager): + manager._last_used_at = 0.0 + manager.touch() + assert manager._last_used_at > 0.0 + + +def test_start_subprocess_no_conda(manager): + with patch("shutil.which", return_value=None): + result = manager._start_subprocess() + assert result is False diff --git a/backend/tests/test_new_features.py b/backend/tests/test_new_features.py new file mode 100644 index 0000000..7b5729a --- /dev/null +++ b/backend/tests/test_new_features.py @@ -0,0 +1,212 @@ +"""Tests for batch delete, pipelines list, export/import, WebSocket manager, schema validation.""" + +from unittest.mock import AsyncMock + +import pytest +from httpx import ASGITransport, AsyncClient + +import app.models # noqa: F401 +from app.database import Base, engine +from app.main import app +from app.websocket.manager import PipelineConnectionManager + + +@pytest.fixture(autouse=True) +async def setup_db(): + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.fixture +async def project_id(client: AsyncClient) -> int: + resp = await client.post("/api/v1/projects", json={"name": "Test Project", "domain": "optics"}) + assert resp.status_code == 201 + return resp.json()["data"]["id"] + + +@pytest.mark.asyncio +async def test_batch_delete_papers(client: AsyncClient, project_id: int): + for i in range(5): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": f"Paper {i}", "abstract": f"Abstract {i}"}, + ) + list_resp = await client.get(f"/api/v1/projects/{project_id}/papers") + papers = list_resp.json()["data"]["items"] + ids_to_delete = [p["id"] for p in papers[:3]] + + resp = await client.post( + f"/api/v1/projects/{project_id}/papers/batch-delete", + json={"paper_ids": ids_to_delete}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["deleted"] == 3 + assert data["requested"] == 3 + + list_resp2 = await client.get(f"/api/v1/projects/{project_id}/papers") + assert list_resp2.json()["data"]["total"] == 2 + + +@pytest.mark.asyncio +async def test_batch_delete_nonexistent_ids(client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/papers/batch-delete", + json={"paper_ids": [99999, 99998]}, + ) + assert resp.status_code == 200 + data = resp.json()["data"] + assert data["deleted"] == 0 + assert data["requested"] == 2 + + +@pytest.mark.asyncio +async def test_list_pipelines_empty(client: AsyncClient): + from app.api.v1 import pipelines + + pipelines._running_tasks.clear() + resp = await client.get("/api/v1/pipelines") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert body["data"] == [] + + +@pytest.mark.asyncio +async def test_list_pipelines_returns_data(client: AsyncClient, project_id: int): + from app.api.v1 import pipelines + + pipelines._running_tasks["mock_thread_123"] = { + "status": "running", + "task_id": 1, + } + try: + resp = await client.get("/api/v1/pipelines") + assert resp.status_code == 200 + body = resp.json() + assert body["code"] == 200 + assert len(body["data"]) == 1 + assert body["data"][0]["thread_id"] == "mock_thread_123" + assert body["data"][0]["status"] == "running" + finally: + pipelines._running_tasks.pop("mock_thread_123", None) + + +@pytest.mark.asyncio +async def test_export_project(client: AsyncClient, project_id: int): + await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Exported Paper", "abstract": "Abstract", "year": 2024, "journal": "Nature"}, + ) + await client.post( + f"/api/v1/projects/{project_id}/keywords", + json={"term": "quantum", "term_en": "quantum", "level": 1, "category": "topic"}, + ) + + resp = await client.get(f"/api/v1/projects/{project_id}/export") + assert resp.status_code == 200 + data = resp.json()["data"] + assert "name" in data + assert "papers" in data + assert "keywords" in data + assert "subscriptions" in data + assert len(data["papers"]) == 1 + assert data["papers"][0]["title"] == "Exported Paper" + assert len(data["keywords"]) == 1 + assert data["keywords"][0]["term"] == "quantum" + + +@pytest.mark.asyncio +async def test_import_project(client: AsyncClient): + payload = { + "name": "Imported Project", + "description": "Test import", + "domain": "physics", + "papers": [ + {"title": "Paper 1", "abstract": "Abstract 1", "year": 2024, "journal": "Nature"}, + {"title": "Paper 2", "abstract": "Abstract 2", "year": 2023, "journal": "Science"}, + ], + "keywords": [{"term": "quantum", "term_en": "quantum", "level": 1, "category": "topic", "synonyms": ""}], + "subscriptions": [], + } + resp = await client.post("/api/v1/projects/import", json=payload) + assert resp.status_code == 201 + data = resp.json()["data"] + assert data["name"] == "Imported Project" + assert data["paper_count"] == 2 + assert data["keyword_count"] == 1 + + list_resp = await client.get(f"/api/v1/projects/{data['id']}/papers") + papers = list_resp.json()["data"]["items"] + assert len(papers) == 2 + titles = {p["title"] for p in papers} + assert "Paper 1" in titles + assert "Paper 2" in titles + + +@pytest.mark.asyncio +async def test_pipeline_connection_manager_connect_disconnect(): + manager = PipelineConnectionManager() + ws = AsyncMock() + ws.accept = AsyncMock() + + await manager.connect(ws, "room_1") + ws.accept.assert_called_once() + assert "room_1" in manager.rooms + assert ws in manager.rooms["room_1"] + + manager.disconnect(ws, "room_1") + assert "room_1" not in manager.rooms or ws not in manager.rooms.get("room_1", set()) + + +@pytest.mark.asyncio +async def test_pipeline_connection_manager_broadcast(): + manager = PipelineConnectionManager() + ws1 = AsyncMock() + ws1.accept = AsyncMock() + ws2 = AsyncMock() + ws2.accept = AsyncMock() + + await manager.connect(ws1, "room_broadcast") + await manager.connect(ws2, "room_broadcast") + + await manager.broadcast_to_room("room_broadcast", {"type": "status", "value": 42}) + ws1.send_json.assert_called_once_with({"type": "status", "value": 42}) + ws2.send_json.assert_called_once_with({"type": "status", "value": 42}) + + +@pytest.mark.asyncio +async def test_paper_year_out_of_range(client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "Test Paper", "year": 1000}, + ) + assert resp.status_code == 422 + + +@pytest.mark.asyncio +async def test_paper_title_too_long(client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/papers", + json={"title": "x" * 3000}, + ) + assert resp.status_code == 422 + + +@pytest.mark.asyncio +async def test_subscription_invalid_frequency(client: AsyncClient, project_id: int): + resp = await client.post( + f"/api/v1/projects/{project_id}/subscriptions", + json={"name": "Test", "frequency": "hourly"}, + ) + assert resp.status_code == 422 diff --git a/backend/tests/test_paper_processor_parallel.py b/backend/tests/test_paper_processor_parallel.py new file mode 100644 index 0000000..2b4a499 --- /dev/null +++ b/backend/tests/test_paper_processor_parallel.py @@ -0,0 +1,99 @@ +"""Tests for paper_processor GPU detection and parallel OCR logic.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from app.services.paper_processor import _detect_gpu_count, _resolve_parallel_limit + + +class TestDetectGpuCount: + """Verify _detect_gpu_count under CPU-only, single-GPU, and multi-GPU scenarios.""" + + def test_cpu_only_no_torch(self): + with ( + patch.dict("sys.modules", {"torch": None}), + patch("builtins.__import__", side_effect=ImportError("no torch")), + ): + assert _detect_gpu_count() == 0 + + def test_cpu_only_cuda_not_available(self): + mock_torch = type("torch", (), {"cuda": type("cuda", (), {"is_available": staticmethod(lambda: False)})()})() + with ( + patch("app.services.paper_processor.importlib", create=True), + patch.dict("sys.modules", {"torch": mock_torch}), + ): + result = _detect_gpu_count() + assert result == 0 + + def test_single_gpu(self): + mock_cuda = type( + "cuda", (), {"is_available": staticmethod(lambda: True), "device_count": staticmethod(lambda: 1)} + )() + mock_torch = type("torch", (), {"cuda": mock_cuda})() + with patch.dict("sys.modules", {"torch": mock_torch}): + assert _detect_gpu_count() == 1 + + def test_multi_gpu(self): + mock_cuda = type( + "cuda", (), {"is_available": staticmethod(lambda: True), "device_count": staticmethod(lambda: 3)} + )() + mock_torch = type("torch", (), {"cuda": mock_cuda})() + with patch.dict("sys.modules", {"torch": mock_torch}): + assert _detect_gpu_count() == 3 + + +class TestResolveParallelLimit: + """Verify _resolve_parallel_limit respects config and GPU count.""" + + def test_auto_cpu_only(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_parallel_limit = 0 + assert _resolve_parallel_limit(0) == 1 + + def test_auto_single_gpu(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_parallel_limit = 0 + assert _resolve_parallel_limit(1) == 1 + + def test_auto_multi_gpu(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_parallel_limit = 0 + assert _resolve_parallel_limit(3) == 3 + + def test_explicit_override(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_parallel_limit = 5 + assert _resolve_parallel_limit(3) == 5 + + def test_explicit_one(self): + with patch("app.services.paper_processor.settings") as mock_settings: + mock_settings.ocr_parallel_limit = 1 + assert _resolve_parallel_limit(3) == 1 + + +class TestGpuIdRoundRobin: + """Verify that worker GPU IDs rotate correctly.""" + + @pytest.mark.parametrize( + ("gpu_count", "worker_id", "expected_gpu_id"), + [ + (3, 0, 0), + (3, 1, 1), + (3, 2, 2), + (3, 3, 0), + (3, 7, 1), + (1, 0, 0), + (1, 5, 0), + ], + ) + def test_round_robin(self, gpu_count: int, worker_id: int, expected_gpu_id: int): + gpu_id = worker_id % gpu_count if gpu_count > 0 else 0 + assert gpu_id == expected_gpu_id + + def test_cpu_only_always_zero(self): + for worker_id in range(10): + gpu_id = worker_id % 1 if 0 > 0 else 0 # noqa: SIM300 + assert gpu_id == 0 diff --git a/backend/tests/test_pdf_metadata.py b/backend/tests/test_pdf_metadata.py new file mode 100644 index 0000000..d3a589c --- /dev/null +++ b/backend/tests/test_pdf_metadata.py @@ -0,0 +1,210 @@ +"""Tests for pdf_metadata service — mock fitz and httpx, verify extraction and Crossref fallback.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from app.schemas.knowledge_base import NewPaperData +from app.services import pdf_metadata + +# --------------------------------------------------------------------------- +# test_extract_local_normal_pdf +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_extract_local_normal_pdf(tmp_path): + """Mock fitz.open returning doc with title/author/doi metadata.""" + pdf_path = tmp_path / "normal.pdf" + pdf_path.write_bytes(b"%PDF-1.4 minimal") + + mock_doc = MagicMock() + mock_doc.metadata = { + "title": "Deep Learning for Microscopy", + "author": "Alice Smith; Bob Jones", + "subject": "Nature Methods, doi: 10.1234/test-paper", + "creationDate": "2024-01-01", + } + mock_doc.page_count = 1 + mock_doc.__iter__ = lambda self: iter([]) # no pages to scan for DOI/abstract + + with patch("app.services.pdf_metadata.fitz.open", return_value=mock_doc): + result = await pdf_metadata.extract_metadata(pdf_path, fallback_title="Untitled") + + assert isinstance(result, NewPaperData) + assert result.title == "Deep Learning for Microscopy" + assert result.authors == [{"name": "Alice Smith"}, {"name": "Bob Jones"}] + assert result.doi == "10.1234/test-paper" + assert result.year == 2024 + assert result.pdf_path == str(pdf_path) + assert result.source == "pdf_upload" + + +# --------------------------------------------------------------------------- +# test_extract_local_corrupted_pdf +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_extract_local_corrupted_pdf(tmp_path): + """Mock fitz.open raising; should return fallback title.""" + pdf_path = tmp_path / "corrupted.pdf" + pdf_path.write_bytes(b"not a valid pdf") + + with patch("app.services.pdf_metadata.fitz.open", side_effect=Exception("Invalid PDF")): + result = await pdf_metadata.extract_metadata(pdf_path, fallback_title="Fallback Title") + + assert isinstance(result, NewPaperData) + assert result.title == "Fallback Title" + assert result.pdf_path == str(pdf_path) + assert result.source == "pdf_upload" + + +# --------------------------------------------------------------------------- +# test_extract_local_no_doi +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_extract_local_no_doi(tmp_path): + """PDF with title/author but no DOI in metadata or text.""" + pdf_path = tmp_path / "no_doi.pdf" + pdf_path.write_bytes(b"%PDF-1.4 minimal") + + mock_doc = MagicMock() + mock_doc.metadata = { + "title": "Paper Without DOI", + "author": "Jane Doe", + "subject": "Some Journal", + "creationDate": "", + } + mock_doc.page_count = 1 + mock_doc.__iter__ = lambda self: iter([]) + + with patch("app.services.pdf_metadata.fitz.open", return_value=mock_doc): + result = await pdf_metadata.extract_metadata(pdf_path, fallback_title="Untitled") + + assert result.title == "Paper Without DOI" + assert result.authors == [{"name": "Jane Doe"}] + assert result.doi is None + + +# --------------------------------------------------------------------------- +# test_extract_doi_cleaning +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_extract_doi_cleaning(tmp_path): + """DOI with trailing punctuation/URL prefix should be cleaned.""" + pdf_path = tmp_path / "doi_clean.pdf" + pdf_path.write_bytes(b"%PDF-1.4 minimal") + + mock_doc = MagicMock() + mock_doc.metadata = { + "title": "Test", + "author": "", + "subject": "Journal, 10.5678/cleaned-doi).", + "creationDate": "", + } + mock_doc.page_count = 1 + mock_doc.__iter__ = lambda self: iter([]) + + with patch("app.services.pdf_metadata.fitz.open", return_value=mock_doc): + result = await pdf_metadata.extract_metadata(pdf_path, fallback_title="Untitled") + + assert result.doi == "10.5678/cleaned-doi" + + +# --------------------------------------------------------------------------- +# test_lookup_crossref_success +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_lookup_crossref_success(tmp_path): + """Mock httpx returning Crossref metadata; should merge with local.""" + pdf_path = tmp_path / "with_doi.pdf" + pdf_path.write_bytes(b"%PDF-1.4 minimal") + + mock_doc = MagicMock() + mock_doc.metadata = { + "title": "Local Title", + "author": "Local Author", + "subject": "10.1234/crossref-test", + "creationDate": "", + } + mock_doc.page_count = 1 + mock_doc.__iter__ = lambda self: iter([]) + + crossref_response = { + "message": { + "title": ["Crossref Title"], + "author": [{"given": "Crossref", "family": "Author"}], + "container-title": ["Crossref Journal"], + "published": {"date-parts": [[2023]]}, + "abstract": "

Crossref abstract

", + } + } + + async def mock_get(*args, **kwargs): + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = crossref_response + return resp + + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(side_effect=mock_get) + + with ( + patch("app.services.pdf_metadata.fitz.open", return_value=mock_doc), + patch("app.services.pdf_metadata.httpx.AsyncClient", return_value=mock_client), + ): + result = await pdf_metadata.extract_metadata(pdf_path, fallback_title="Untitled") + + assert result.title == "Crossref Title" + assert result.authors == [{"name": "Crossref Author"}] + assert result.journal == "Crossref Journal" + assert result.year == 2023 + assert result.abstract == "Crossref abstract" + assert result.pdf_path == str(pdf_path) + + +# --------------------------------------------------------------------------- +# test_lookup_crossref_failure +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_lookup_crossref_failure(tmp_path): + """Mock httpx raising; should fallback to local metadata.""" + pdf_path = tmp_path / "crossref_fail.pdf" + pdf_path.write_bytes(b"%PDF-1.4 minimal") + + mock_doc = MagicMock() + mock_doc.metadata = { + "title": "Local Only Title", + "author": "Local Author", + "subject": "10.9999/crossref-fail", + "creationDate": "", + } + mock_doc.page_count = 1 + mock_doc.__iter__ = lambda self: iter([]) + + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(side_effect=Exception("Network error")) + + with ( + patch("app.services.pdf_metadata.fitz.open", return_value=mock_doc), + patch("app.services.pdf_metadata.httpx.AsyncClient", return_value=mock_client), + ): + result = await pdf_metadata.extract_metadata(pdf_path, fallback_title="Untitled") + + assert result.title == "Local Only Title" + assert result.authors == [{"name": "Local Author"}] + assert result.doi == "10.9999/crossref-fail" + assert result.pdf_path == str(pdf_path) diff --git a/backend/tests/test_pipeline_real_pdf.py b/backend/tests/test_pipeline_real_pdf.py new file mode 100644 index 0000000..bd4387d --- /dev/null +++ b/backend/tests/test_pipeline_real_pdf.py @@ -0,0 +1,218 @@ +"""Pipeline integration tests with real PDF files. + +Requires test PDFs at /data0/djx/omelette_pdf_test/ (skipped otherwise). +These tests exercise the upload pipeline with real metadata extraction. +""" + +import os +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient +from langgraph.checkpoint.memory import MemorySaver +from langgraph.types import Command + +from app.database import Base, async_session_factory, engine +from app.models import Paper, Project +from app.pipelines.graphs import create_upload_pipeline +from app.pipelines.state import PipelineState + +PDF_TEST_DIR = os.environ.get("E2E_PDF_DIR", "/data0/djx/omelette_pdf_test") +PDF_DIR_EXISTS = os.path.isdir(PDF_TEST_DIR) + +pytestmark = pytest.mark.skipif(not PDF_DIR_EXISTS, reason=f"Test PDF directory not available: {PDF_TEST_DIR}") + + +def _smallest_pdf() -> str: + """Find the smallest PDF in the test directory.""" + pdfs = sorted(Path(PDF_TEST_DIR).glob("*.pdf"), key=lambda p: p.stat().st_size) + if not pdfs: + pytest.skip("No PDFs found in test directory") + return str(pdfs[0]) + + +@pytest.fixture(autouse=True) +async def setup_db(): + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def project(): + async with async_session_factory() as db: + p = Project(name="pdf-test-kb", description="for real PDF testing") + db.add(p) + await db.commit() + await db.refresh(p) + return p + + +@pytest.fixture +def test_client(): + from app.main import app + + return AsyncClient(transport=ASGITransport(app=app), base_url="http://test") + + +# ── Upload pipeline with real PDF ── + + +async def test_upload_pipeline_real_pdf(project): + """Upload pipeline with a real PDF should extract metadata and import the paper.""" + pdf_path = _smallest_pdf() + + saver = MemorySaver() + graph = create_upload_pipeline(checkpointer=saver) + + initial: PipelineState = { + "project_id": project.id, + "thread_id": "test_upload_real", + "pipeline_type": "upload", + "params": {"pdf_paths": [pdf_path]}, + "papers": [], + "conflicts": [], + "resolved_conflicts": [], + "progress": 0, + "total": 100, + "stage": "starting", + "error": None, + "cancelled": False, + "result": {}, + } + + config = {"configurable": {"thread_id": "test_upload_real"}} + result = await graph.ainvoke(initial, config=config) + + assert result["progress"] == 100 + assert result.get("error") is None + + papers = result.get("papers", []) + assert len(papers) >= 1 + first = papers[0] + assert first.get("title"), "Extracted title should not be empty" + + +# ── HITL interrupt → resume flow ── + + +async def test_upload_hitl_interrupt_resume(project): + """When an uploaded PDF has the same title as an existing paper, + the dedup node should trigger HITL. Resuming with 'skip' should complete.""" + pdf_path = _smallest_pdf() + + from app.services.pdf_metadata import extract_metadata + + meta = await extract_metadata(Path(pdf_path), fallback_title="fallback") + + async with async_session_factory() as db: + existing = Paper( + project_id=project.id, + title=meta.title, + doi=meta.doi or "", + source="manual", + ) + db.add(existing) + await db.commit() + + saver = MemorySaver() + graph = create_upload_pipeline(checkpointer=saver) + + initial: PipelineState = { + "project_id": project.id, + "thread_id": "test_hitl_resume", + "pipeline_type": "upload", + "params": {"pdf_paths": [pdf_path]}, + "papers": [], + "conflicts": [], + "resolved_conflicts": [], + "progress": 0, + "total": 100, + "stage": "starting", + "error": None, + "cancelled": False, + "result": {}, + } + + config = {"configurable": {"thread_id": "test_hitl_resume"}} + await graph.ainvoke(initial, config=config) + + snapshot = graph.get_state(config) + assert "hitl_dedup" in snapshot.next, f"Expected HITL interrupt, got {snapshot.next}" + conflicts = snapshot.values.get("conflicts", []) + assert len(conflicts) >= 1 + + result = await graph.ainvoke( + Command(resume=[{"action": "skip", "new_paper": {}}]), + config=config, + ) + assert result["progress"] == 100 + assert result["stage"] in ("index", "import") + + +# ── Pipeline API path safety ── + + +async def test_pipeline_path_traversal_rejected(test_client, project): + """Paths outside pdf_dir should be rejected with 400.""" + resp = await test_client.post( + "/api/v1/pipelines/upload", + json={ + "project_id": project.id, + "pdf_paths": ["/etc/passwd"], + }, + ) + assert resp.status_code == 400 + assert "not within allowed directory" in resp.json().get("message", "") + + +async def test_pipeline_path_dot_dot_rejected(test_client, project): + """Paths with '..' that resolve outside pdf_dir should be rejected.""" + resp = await test_client.post( + "/api/v1/pipelines/upload", + json={ + "project_id": project.id, + "pdf_paths": [f"{PDF_TEST_DIR}/../../etc/passwd"], + }, + ) + assert resp.status_code == 400 + + +# ── Pipeline list endpoint ── + + +async def test_pipeline_list_includes_started(test_client, project, monkeypatch): + """After starting a pipeline, GET /pipelines should list it.""" + from app.api.v1 import pipelines + + pipelines._running_tasks.clear() + + async def mock_search(self, query="", sources=None, max_results=100): + return {"papers": [], "total": 0} + + from app.services import search_service + + monkeypatch.setattr(search_service.SearchService, "search", mock_search) + + resp = await test_client.post( + "/api/v1/pipelines/search", + json={ + "project_id": project.id, + "query": "test", + "max_results": 5, + }, + ) + assert resp.status_code == 200 + + import asyncio + + await asyncio.sleep(0.5) + + list_resp = await test_client.get("/api/v1/pipelines") + assert list_resp.status_code == 200 + data = list_resp.json()["data"] + assert len(data) >= 1 + + pipelines._running_tasks.clear() diff --git a/backend/tests/test_pipelines.py b/backend/tests/test_pipelines.py index 19fcfad..4c29ff2 100644 --- a/backend/tests/test_pipelines.py +++ b/backend/tests/test_pipelines.py @@ -416,10 +416,7 @@ async def mock_search(self, query="", sources=None, max_results=100): thread_id = data["thread_id"] - import asyncio - - await asyncio.sleep(1) - + # Get status immediately before pipeline completes and removes itself resp2 = await client.get(f"/api/v1/pipelines/{thread_id}/status") assert resp2.status_code == 200 diff --git a/backend/tests/test_reranker_service.py b/backend/tests/test_reranker_service.py new file mode 100644 index 0000000..41d89f2 --- /dev/null +++ b/backend/tests/test_reranker_service.py @@ -0,0 +1,93 @@ +"""Tests for reranker_service — model loading, caching, and async inference.""" + +from unittest.mock import MagicMock, patch + +import pytest + + +@pytest.fixture(autouse=True) +def _reset_reranker_cache(): + from app.services.gpu_model_manager import gpu_model_manager + + gpu_model_manager.unload("reranker") + yield + gpu_model_manager.unload("reranker") + + +class TestGetReranker: + @patch("app.services.reranker_service._build_reranker") + def test_returns_cached_instance(self, mock_build): + from app.services.reranker_service import get_reranker + + sentinel = MagicMock() + mock_build.return_value = sentinel + result = get_reranker() + assert result is sentinel + mock_build.assert_called_once() + + @patch("app.services.reranker_service._build_reranker") + def test_caching_returns_same_instance(self, mock_build): + from app.services.reranker_service import get_reranker + + sentinel = MagicMock() + mock_build.return_value = sentinel + r1 = get_reranker() + r2 = get_reranker() + assert r1 is r2 + mock_build.assert_called_once() + + @patch("app.services.reranker_service._build_reranker") + def test_custom_model_name(self, mock_build): + from app.services.reranker_service import get_reranker + + mock_build.return_value = MagicMock() + get_reranker(model_name="custom/reranker") + mock_build.assert_called_with("custom/reranker") + + +class TestRerankNodes: + @pytest.mark.asyncio + async def test_empty_nodes_returns_empty(self): + from app.services.reranker_service import rerank_nodes + + result = await rerank_nodes([], "test query", top_n=5) + assert result == [] + + @pytest.mark.asyncio + @patch("app.services.reranker_service.get_reranker") + async def test_rerank_returns_top_n(self, mock_get_reranker): + from app.services.reranker_service import rerank_nodes + + mock_node_1 = MagicMock() + mock_node_1.score = 0.9 + mock_node_2 = MagicMock() + mock_node_2.score = 0.5 + mock_node_3 = MagicMock() + mock_node_3.score = 0.7 + + mock_reranker = MagicMock() + mock_reranker.postprocess_nodes.return_value = [mock_node_1, mock_node_3, mock_node_2] + mock_get_reranker.return_value = mock_reranker + + result = await rerank_nodes([mock_node_1, mock_node_2, mock_node_3], "query", top_n=2) + assert len(result) == 2 + assert result[0] is mock_node_1 + + @pytest.mark.asyncio + @patch("app.services.reranker_service.get_reranker", side_effect=ImportError("no model")) + async def test_fallback_on_import_error(self, _mock): + from app.services.reranker_service import rerank_nodes + + nodes = [MagicMock() for _ in range(5)] + result = await rerank_nodes(nodes, "query", top_n=3) + assert len(result) == 3 + assert result == nodes[:3] + + @pytest.mark.asyncio + @patch("app.services.reranker_service.get_reranker", side_effect=RuntimeError("GPU error")) + async def test_fallback_on_runtime_error(self, _mock): + from app.services.reranker_service import rerank_nodes + + nodes = [MagicMock() for _ in range(4)] + result = await rerank_nodes(nodes, "query", top_n=2) + assert len(result) == 2 diff --git a/backend/tests/test_search.py b/backend/tests/test_search.py index 011a519..b6a2bce 100644 --- a/backend/tests/test_search.py +++ b/backend/tests/test_search.py @@ -414,7 +414,7 @@ async def mock_search(*args, **kwargs): resp = await client.post( f"/api/v1/projects/{project_id}/search/execute", - params={"query": "machine learning"}, + json={"query": "machine learning"}, ) assert resp.status_code == 200 body = resp.json() @@ -431,16 +431,16 @@ async def test_execute_search_no_query_no_keywords(client: AsyncClient): resp = await client.post( f"/api/v1/projects/{project_id}/search/execute", - params={"query": ""}, + json={"query": ""}, ) assert resp.status_code == 400 - assert "no keywords" in resp.json()["detail"].lower() + assert "no keywords" in resp.json()["message"].lower() @pytest.mark.asyncio async def test_execute_search_nonexistent_project(client: AsyncClient): resp = await client.post( "/api/v1/projects/99999/search/execute", - params={"query": "test"}, + json={"query": "test"}, ) assert resp.status_code == 404 diff --git a/backend/tests/test_subscription.py b/backend/tests/test_subscription.py index d5396ca..ef94531 100644 --- a/backend/tests/test_subscription.py +++ b/backend/tests/test_subscription.py @@ -6,8 +6,9 @@ import pytest from httpx import ASGITransport, AsyncClient -from app.database import Base, engine +from app.database import Base, async_session_factory, engine from app.main import app +from app.models import Project from app.services.subscription_service import SubscriptionService @@ -20,6 +21,16 @@ async def setup_db(): await conn.run_sync(Base.metadata.drop_all) +@pytest.fixture +async def project(setup_db): + async with async_session_factory() as db: + p = Project(name="Test Project", description="For subscription tests") + db.add(p) + await db.commit() + await db.refresh(p) + return p + + @pytest.fixture async def client(): transport = ASGITransport(app=app) @@ -88,7 +99,10 @@ async def test_check_rss_feed(self, mock_rss_xml): mock_resp.text = mock_rss_xml mock_resp.raise_for_status = MagicMock() - with patch("httpx.AsyncClient") as mock_client_cls: + with ( + patch("app.services.url_validator.validate_url_safe", return_value="https://example.com/feed.xml"), + patch("httpx.AsyncClient") as mock_client_cls, + ): mock_client = AsyncMock() mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) @@ -108,7 +122,10 @@ async def test_check_rss_feed_with_since_filter(self, mock_rss_xml): mock_resp.text = mock_rss_xml mock_resp.raise_for_status = MagicMock() - with patch("httpx.AsyncClient") as mock_client_cls: + with ( + patch("app.services.url_validator.validate_url_safe", return_value="https://example.com/feed.xml"), + patch("httpx.AsyncClient") as mock_client_cls, + ): mock_client = AsyncMock() mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) @@ -135,12 +152,15 @@ async def test_list_common_feeds(self, client): assert len(body["data"]) >= 4 @pytest.mark.asyncio - async def test_check_rss_mock(self, client, mock_rss_xml): + async def test_check_rss_mock(self, client, project, mock_rss_xml): mock_resp = MagicMock() mock_resp.text = mock_rss_xml mock_resp.raise_for_status = MagicMock() - with patch("httpx.AsyncClient") as mock_client_cls: + with ( + patch("app.services.url_validator.validate_url_safe", return_value="https://example.com/feed.xml"), + patch("httpx.AsyncClient") as mock_client_cls, + ): mock_client = AsyncMock() mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) @@ -148,7 +168,7 @@ async def test_check_rss_mock(self, client, mock_rss_xml): mock_client_cls.return_value = mock_client resp = await client.post( - "/api/v1/projects/1/subscriptions/check-rss", + f"/api/v1/projects/{project.id}/subscriptions/check-rss", params={"feed_url": "https://example.com/feed.xml", "since_days": 7}, ) assert resp.status_code == 200 diff --git a/backend/tests/test_url_validator.py b/backend/tests/test_url_validator.py new file mode 100644 index 0000000..de4e559 --- /dev/null +++ b/backend/tests/test_url_validator.py @@ -0,0 +1,65 @@ +"""Tests for app.services.url_validator.""" + +import pytest + +from app.services.url_validator import validate_doi, validate_url_safe + + +def test_valid_https_url(): + result = validate_url_safe("https://8.8.8.8/") + assert result == "https://8.8.8.8/" + + +def test_valid_http_url(): + result = validate_url_safe("http://1.1.1.1/") + assert result == "http://1.1.1.1/" + + +def test_ftp_scheme_rejected(): + with pytest.raises(ValueError, match="Unsupported scheme: ftp"): + validate_url_safe("ftp://example.com/file") + + +def test_no_scheme_rejected(): + with pytest.raises(ValueError, match="Unsupported scheme"): + validate_url_safe("example.com/path") + + +def test_private_ip_rejected(): + with pytest.raises(ValueError, match="Blocked.*private"): + validate_url_safe("http://192.168.1.1/") + + +def test_loopback_rejected(): + with pytest.raises(ValueError, match="Blocked.*private"): + validate_url_safe("http://127.0.0.1/") + + +def test_metadata_google_rejected(): + with pytest.raises(ValueError, match="Blocked hostname"): + validate_url_safe("http://metadata.google.internal/") + + +def test_metadata_aws_rejected(): + with pytest.raises(ValueError, match="Blocked hostname"): + validate_url_safe("http://metadata.amazonaws.com/") + + +def test_valid_doi(): + result = validate_doi("10.1038/nature12373") + assert result == "10.1038/nature12373" + + +def test_valid_doi_with_special_chars(): + result = validate_doi("10.1000/xyz123") + assert result == "10.1000/xyz123" + + +def test_invalid_doi_no_prefix(): + with pytest.raises(ValueError, match="Invalid DOI format"): + validate_doi("not-a-doi") + + +def test_invalid_doi_wrong_format(): + with pytest.raises(ValueError, match="Invalid DOI format"): + validate_doi("11.1234/abc") diff --git a/backend/tests/test_writing.py b/backend/tests/test_writing.py index 9a7561a..26796af 100644 --- a/backend/tests/test_writing.py +++ b/backend/tests/test_writing.py @@ -68,7 +68,7 @@ async def project_with_papers(): @pytest.mark.asyncio async def test_summarize_papers(project_with_papers): - from app.services.llm_client import LLMClient + from app.services.llm.client import LLMClient from app.services.writing_service import WritingService project_id, paper_ids = project_with_papers @@ -84,7 +84,7 @@ async def test_summarize_papers(project_with_papers): @pytest.mark.asyncio async def test_generate_citations_gb_t_7714(project_with_papers): - from app.services.llm_client import LLMClient + from app.services.llm.client import LLMClient from app.services.writing_service import WritingService project_id, paper_ids = project_with_papers @@ -103,7 +103,7 @@ async def test_generate_citations_gb_t_7714(project_with_papers): @pytest.mark.asyncio async def test_generate_citations_apa(project_with_papers): - from app.services.llm_client import LLMClient + from app.services.llm.client import LLMClient from app.services.writing_service import WritingService project_id, paper_ids = project_with_papers @@ -119,7 +119,7 @@ async def test_generate_citations_apa(project_with_papers): @pytest.mark.asyncio async def test_generate_citations_mla(project_with_papers): - from app.services.llm_client import LLMClient + from app.services.llm.client import LLMClient from app.services.writing_service import WritingService project_id, paper_ids = project_with_papers @@ -135,7 +135,7 @@ async def test_generate_citations_mla(project_with_papers): @pytest.mark.asyncio async def test_generate_review_outline(project_with_papers): - from app.services.llm_client import LLMClient + from app.services.llm.client import LLMClient from app.services.writing_service import WritingService project_id, paper_ids = project_with_papers @@ -156,7 +156,7 @@ async def test_generate_review_outline(project_with_papers): @pytest.mark.asyncio async def test_analyze_gaps(project_with_papers): - from app.services.llm_client import LLMClient + from app.services.llm.client import LLMClient from app.services.writing_service import WritingService project_id, paper_ids = project_with_papers @@ -300,7 +300,7 @@ async def test_assist_unknown_task(client: AsyncClient, project_with_papers): f"/api/v1/projects/{project_id}/writing/assist", json={"task": "unknown_task"}, ) - assert resp.status_code == 200 # We return 400 in data + assert resp.status_code == 400 body = resp.json() assert body["code"] == 400 assert "Unknown task" in body["message"] diff --git a/docs/api-endpoints.md b/docs/api-endpoints.md new file mode 100644 index 0000000..999dce2 --- /dev/null +++ b/docs/api-endpoints.md @@ -0,0 +1,249 @@ +# Omelette API Endpoints Reference + +This document lists all API v1 endpoints exposed by the Omelette backend. Endpoints are grouped by module. Base URL: `/api/v1`. + +**Legend:** +- 🤖 Involves LLM calls +- 📄 Involves file I/O (upload, download, PDF processing, vector store) +- 🔄 SSE streaming response + +--- + +## Summary by Module + +| Module | Endpoints | 🤖 LLM | 📄 File I/O | 🔄 SSE | +|--------|-----------|--------|-------------|--------| +| Projects | 6 | 0 | 2 | 0 | +| Papers | 9 | 0 | 2 | 0 | +| Upload | 2 | 0 | 2 | 0 | +| Keywords | 7 | 2 | 0 | 0 | +| Search | 2 | 0 | 0 | 0 | +| Dedup | 5 | 4 | 2 | 0 | +| Crawler | 2 | 0 | 1 | 0 | +| OCR | 2 | 0 | 1 | 0 | +| Subscriptions | 9 | 0 | 0 | 0 | +| RAG | 5 | 1 | 4 | 1 | +| Writing | 6 | 5 | 0 | 1 | +| Tasks | 3 | 0 | 0 | 0 | +| Settings | 5 | 1 | 0 | 0 | +| Conversations | 5 | 0 | 0 | 0 | +| Chat | 2 | 2 | 0 | 1 | +| Rewrite | 1 | 1 | 0 | 1 | +| Pipelines | 5 | 0 | 2 | 0 | +| **Total** | **76** | **16** | **14** | **4** | + +--- + +## Projects + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| GET | `/api/v1/projects` | List projects with pagination | `page`, `page_size` | | +| POST | `/api/v1/projects` | Create a new project | Body: `ProjectCreate` (name, description, domain, settings) | | +| GET | `/api/v1/projects/{project_id}` | Get project by ID | `project_id` | | +| PUT | `/api/v1/projects/{project_id}` | Update project | `project_id`, Body: `ProjectUpdate` | | +| DELETE | `/api/v1/projects/{project_id}` | Delete project | `project_id` | | +| POST | `/api/v1/projects/{project_id}/pipeline/run` | Trigger crawl → OCR → index pipeline for all pending papers | `project_id` | 📄 | +| POST | `/api/v1/projects/{project_id}/pipeline/paper/{paper_id}` | Trigger pipeline for a single paper | `project_id`, `paper_id` | 📄 | + +--- + +## Papers + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| GET | `/api/v1/projects/{project_id}/papers` | List papers with filters and pagination | `project_id`, `page`, `page_size`, `status`, `year`, `q`, `sort_by`, `order` | | +| POST | `/api/v1/projects/{project_id}/papers` | Create a paper | `project_id`, Body: `PaperCreate` | | +| POST | `/api/v1/projects/{project_id}/papers/bulk` | Bulk import papers | `project_id`, Body: `PaperBulkImport` (papers[]) | | +| GET | `/api/v1/projects/{project_id}/papers/{paper_id}` | Get paper by ID | `project_id`, `paper_id` | | +| PUT | `/api/v1/projects/{project_id}/papers/{paper_id}` | Update paper | `project_id`, `paper_id`, Body: `PaperUpdate` | | +| DELETE | `/api/v1/projects/{project_id}/papers/{paper_id}` | Delete paper | `project_id`, `paper_id` | | +| GET | `/api/v1/projects/{project_id}/papers/{paper_id}/pdf` | Serve PDF file | `project_id`, `paper_id` | 📄 | +| GET | `/api/v1/projects/{project_id}/papers/{paper_id}/citation-graph` | Get citation relationship graph via Semantic Scholar | `project_id`, `paper_id`, `depth`, `max_nodes` | | + +--- + +## Upload (Papers) + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/projects/{project_id}/papers/upload` | Upload PDFs, extract metadata, run dedup check | `project_id`, `files` (multipart) | 📄 | +| POST | `/api/v1/projects/{project_id}/papers/process` | Trigger OCR + RAG indexing for papers | `project_id`, `paper_ids` (optional) | 📄 | + +--- + +## Keywords + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| GET | `/api/v1/projects/{project_id}/keywords` | List keywords with pagination | `project_id`, `page`, `page_size`, `level` | | +| POST | `/api/v1/projects/{project_id}/keywords` | Create keyword | `project_id`, Body: `KeywordCreate` | | +| POST | `/api/v1/projects/{project_id}/keywords/bulk` | Bulk create keywords | `project_id`, Body: `KeywordCreate[]` | | +| GET | `/api/v1/projects/{project_id}/keywords/search-formula` | Generate boolean search formula from project keywords | `project_id`, `database` | 🤖 | +| PUT | `/api/v1/projects/{project_id}/keywords/{keyword_id}` | Update keyword | `project_id`, `keyword_id`, Body: `KeywordUpdate` | | +| DELETE | `/api/v1/projects/{project_id}/keywords/{keyword_id}` | Delete keyword | `project_id`, `keyword_id` | | +| POST | `/api/v1/projects/{project_id}/keywords/expand` | Expand seed keywords with synonyms via LLM | `project_id`, Body: `KeywordExpandRequest` | 🤖 | + +--- + +## Search + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/projects/{project_id}/search/execute` | Execute federated search (Semantic Scholar, OpenAlex, arXiv, Crossref) | `project_id`, `query`, `sources`, `max_results`, `auto_import` | | +| GET | `/api/v1/projects/{project_id}/search/sources` | List available search sources and status | `project_id` | | + +--- + +## Dedup + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/projects/{project_id}/dedup/run` | Run deduplication pipeline | `project_id`, `strategy` (full, doi_only, title_only) | 🤖 | +| GET | `/api/v1/projects/{project_id}/dedup/candidates` | List potential duplicate pairs for manual review | `project_id` | 🤖 | +| POST | `/api/v1/projects/{project_id}/dedup/verify` | Use LLM to verify if two papers are duplicates | `project_id`, `paper_a_id`, `paper_b_id` | 🤖 | +| POST | `/api/v1/projects/{project_id}/dedup/resolve` | Resolve upload conflict (keep_old, keep_new, merge, skip) | `project_id`, Body: `ResolveConflictRequest` | 📄 | +| POST | `/api/v1/projects/{project_id}/dedup/auto-resolve` | Use LLM to suggest resolution for conflict pairs | `project_id`, Body: `AutoResolveRequest` | 🤖 📄 | + +--- + +## Crawler + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/projects/{project_id}/crawl/start` | Start PDF download for papers needing PDFs | `project_id`, `priority`, `max_papers` | 📄 | +| GET | `/api/v1/projects/{project_id}/crawl/stats` | Return download statistics for project | `project_id` | | + +--- + +## OCR + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/projects/{project_id}/ocr/process` | Run OCR/text extraction on downloaded PDFs | `project_id`, `paper_ids`, `force_ocr`, `use_gpu` | 📄 | +| GET | `/api/v1/projects/{project_id}/ocr/stats` | Return OCR processing statistics | `project_id` | | + +--- + +## Subscriptions + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| GET | `/api/v1/projects/{project_id}/subscriptions/feeds` | List common academic RSS feed templates | `project_id` | | +| POST | `/api/v1/projects/{project_id}/subscriptions/check-rss` | Check RSS feed for new entries | `project_id`, `feed_url`, `since_days` | | +| POST | `/api/v1/projects/{project_id}/subscriptions/check-updates` | Check for new papers via API search | `project_id`, `query`, `sources`, `since_days`, `max_results` | | +| GET | `/api/v1/projects/{project_id}/subscriptions` | List subscriptions for project | `project_id` | | +| POST | `/api/v1/projects/{project_id}/subscriptions` | Create subscription | `project_id`, Body: `SubscriptionCreate` | | +| GET | `/api/v1/projects/{project_id}/subscriptions/{sub_id}` | Get subscription by ID | `project_id`, `sub_id` | | +| PUT | `/api/v1/projects/{project_id}/subscriptions/{sub_id}` | Update subscription | `project_id`, `sub_id`, Body: `SubscriptionUpdate` | | +| DELETE | `/api/v1/projects/{project_id}/subscriptions/{sub_id}` | Delete subscription | `project_id`, `sub_id` | | +| POST | `/api/v1/projects/{project_id}/subscriptions/{sub_id}/trigger` | Manually trigger subscription update | `project_id`, `sub_id`, `since_days` | | + +--- + +## RAG + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/projects/{project_id}/rag/query` | Answer question using RAG over indexed literature | `project_id`, Body: `RAGQueryRequest` (question, top_k, use_reranker, include_sources) | 🤖 | +| POST | `/api/v1/projects/{project_id}/rag/index` | Build or rebuild vector index for processed papers | `project_id` | 📄 | +| POST | `/api/v1/projects/{project_id}/rag/index/stream` | SSE streaming index rebuild with progress events | `project_id` | 📄 🔄 | +| GET | `/api/v1/projects/{project_id}/rag/stats` | Return indexing statistics | `project_id` | | +| DELETE | `/api/v1/projects/{project_id}/rag/index` | Delete vector index for project | `project_id` | 📄 | + +--- + +## Writing + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/projects/{project_id}/writing/assist` | AI writing assistance (summarize, cite, outline, gap analysis) | `project_id`, Body: `WritingAssistRequest` | 🤖 | +| POST | `/api/v1/projects/{project_id}/writing/summarize` | Generate summaries for selected papers | `project_id`, Body: `SummarizeRequest` | 🤖 | +| POST | `/api/v1/projects/{project_id}/writing/citations` | Generate formatted citations | `project_id`, Body: `CitationsRequest` | | +| POST | `/api/v1/projects/{project_id}/writing/review-outline` | Generate literature review outline | `project_id`, Body: `ReviewOutlineRequest` | 🤖 | +| POST | `/api/v1/projects/{project_id}/writing/gap-analysis` | Analyze research gaps | `project_id`, Body: `GapAnalysisRequest` | 🤖 | +| POST | `/api/v1/projects/{project_id}/writing/review-draft/stream` | Stream literature review draft via SSE | `project_id`, Body: `ReviewDraftRequest` | 🤖 🔄 | + +--- + +## Tasks + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| GET | `/api/v1/tasks/{task_id}` | Get task status and details | `task_id` | | +| GET | `/api/v1/tasks` | List tasks with pagination | `project_id`, `status`, `page`, `page_size` | | +| POST | `/api/v1/tasks/{task_id}/cancel` | Cancel a running task | `task_id` | | + +--- + +## Settings + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| GET | `/api/v1/settings` | Get merged settings (DB overrides .env); API keys masked | | | +| PUT | `/api/v1/settings` | Update user settings and persist to DB | Body: `SettingsUpdateSchema` | | +| GET | `/api/v1/settings/models` | List available LLM providers and models | | | +| POST | `/api/v1/settings/test-connection` | Test LLM configuration with simple prompt | | 🤖 | +| GET | `/api/v1/settings/health` | Simple health check | | | + +--- + +## Conversations + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| GET | `/api/v1/conversations` | List conversations, newest first | `page`, `page_size`, `knowledge_base_id` | | +| POST | `/api/v1/conversations` | Create new conversation | Body: `ConversationCreateSchema` | | +| GET | `/api/v1/conversations/{conversation_id}` | Get conversation with all messages | `conversation_id` | | +| PUT | `/api/v1/conversations/{conversation_id}` | Update conversation title or settings | `conversation_id`, Body: `ConversationUpdateSchema` | | +| DELETE | `/api/v1/conversations/{conversation_id}` | Delete conversation and messages | `conversation_id` | | + +--- + +## Chat + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/chat/stream` | Data Stream Protocol (Vercel AI SDK 5.0) chat endpoint | Body: `ChatStreamRequest` | 🤖 🔄 | +| POST | `/api/v1/chat/complete` | Short text completion for autocomplete | Body: `CompletionRequest` | 🤖 | + +--- + +## Rewrite + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/chat/rewrite` | SSE streaming excerpt rewrite (simplify, academic, translate, custom) | Body: `RewriteRequest` | 🤖 🔄 | + +--- + +## Pipelines + +| Method | Path | Description | Params | Flags | +|--------|------|-------------|--------|-------| +| POST | `/api/v1/pipelines/search` | Start keyword-search pipeline (search → dedup → crawl → OCR → index) | Body: `SearchPipelineRequest` | 📄 | +| POST | `/api/v1/pipelines/upload` | Start PDF-upload pipeline (extract → dedup → OCR → index) | Body: `UploadPipelineRequest` | 📄 | +| GET | `/api/v1/pipelines/{thread_id}/status` | Get pipeline execution status | `thread_id` | | +| POST | `/api/v1/pipelines/{thread_id}/resume` | Resume interrupted pipeline with resolved conflicts | `thread_id`, Body: `ResumeRequest` | | +| POST | `/api/v1/pipelines/{thread_id}/cancel` | Cancel running pipeline | `thread_id` | | + +--- + +## Authentication + +The API uses **optional API key authentication** via `API_SECRET_KEY` (configured in `.env`). + +- **When `API_SECRET_KEY` is set:** All requests must include the key via: + - Header: `X-API-Key: ` + - Or query param: `?api_key=` + +- **Exempt paths** (no auth required): + - `/` — Root + - `/health` — Health check + - `/api/v1/settings/health` — Settings health check + - `/docs` — Swagger UI + - `/openapi.json` — OpenAPI spec + - `/redoc` — ReDoc + - Any path under `/mcp` — MCP server + +- **When `API_SECRET_KEY` is unset:** All endpoints are accessible without authentication. diff --git a/docs/brainstorms/2026-03-17-backend-comprehensive-review-brainstorm.md b/docs/brainstorms/2026-03-17-backend-comprehensive-review-brainstorm.md new file mode 100644 index 0000000..5fd4df0 --- /dev/null +++ b/docs/brainstorms/2026-03-17-backend-comprehensive-review-brainstorm.md @@ -0,0 +1,384 @@ +# 后端全面复盘与提示词审计 + +**日期**: 2026-03-17 +**范围**: 后端架构、代码质量、提示词管理、改进计划 +**状态**: 进行中 + +--- + +## 一、我们要做什么 + +对 Omelette 后端进行全面复盘，涵盖： +1. 当前架构和模块职责的梳理 +2. 按优先级列出所有需要修复的问题 +3. 对每个 LLM 提示词的质量评估和改进建议 +4. 具体的优化方案和实施步骤 + +--- + +## 二、现状梳理 + +### 2.1 整体架构 + +``` +FastAPI App (main.py) +├── Middleware: Auth → CORS → RateLimit +├── API Layer (api/v1/) +│ ├── projects, papers, upload, keywords, search +│ ├── dedup, crawler, ocr, subscription +│ ├── rag, writing, tasks, settings +│ ├── conversations, chat, rewrite, pipelines +│ └── deps.py (共享依赖) +├── Service Layer (services/) +│ ├── llm/ (LLMClient + 工厂模式 + 适配器) +│ ├── rag_service (LlamaIndex + ChromaDB) +│ ├── writing_service, completion_service +│ ├── dedup_service, keyword_service +│ ├── search_service, crawler_service, ocr_service +│ ├── subscription_service, pipeline_service +│ ├── embedding_service, citation_graph_service +│ └── paper_processor, pdf_metadata, mineru_client +├── Pipeline Layer (pipelines/) +│ ├── Search Pipeline: search→dedup→[HITL]→import→crawl→ocr→index +│ ├── Upload Pipeline: extract→dedup→[HITL]→ocr→index +│ └── Chat Pipeline: understand→retrieve→rank→clean→generate→persist +├── Models (SQLAlchemy Async): Project, Paper, PaperChunk, Keyword, +│ Subscription, Task, Conversation, Message, UserSettings +├── Schemas (Pydantic v2): 每个模块的请求/响应 +├── MCP Server: tools + resources + prompts +└── Alembic Migrations (5 versions) +``` + +### 2.2 数据流 + +**Chat 流程**: 用户消息 → understand (加载历史 + 构建 system prompt) → retrieve (RAG 并行查询多知识库) → rank (匹配论文元数据 + 构建引用列表) → clean (并行 LLM 清洗 OCR 摘录) → generate (流式生成回答) → persist (保存到 DB) + +**搜索管道**: 关键词 → 多源搜索 (Semantic Scholar, OpenAlex, arXiv, Crossref) → 去重 (DOI → 标题相似度 → LLM 验证) → [HITL 冲突解决] → 导入 → PDF 爬取 → OCR → 知识库索引 + +**写作流程**: 选择文献 → 生成提纲 → 逐章节 RAG 检索 → 逐章节 LLM 撰写 → SSE 流式输出 + +### 2.3 LLM 调用链路 + +``` +调用方 → LLMClient.chat/chat_stream/chat_json + → _to_langchain_messages (dict → SystemMessage/HumanMessage/AIMessage) + → LangChain BaseChatModel.ainvoke/astream + → Provider (OpenAI/Anthropic/Aliyun/Volcengine/Ollama/Mock) +``` + +所有 LLM 调用都通过 `LLMClient`，没有直接调用 provider SDK 的情况。 + +--- + +## 三、问题清单（按优先级） + +### P0 — 高优先级（影响正确性或稳定性） + +#### 3.1 异步中的同步阻塞调用 + +| 位置 | 阻塞调用 | 影响 | +|------|---------|------| +| `subscription_service.py:29` | `feedparser.parse(resp.text)` | CPU 密集，阻塞事件循环 | +| `pdf_metadata.py:39` | `fitz.open()` + 页面遍历 | I/O + CPU 密集 | +| `rag_service.py:220,230,299,305` | `collection.count()` (sync ChromaDB) | 每次 query 调用两次 | +| `database.py:71,112` | `subprocess.run()` 跑 Alembic | 启动时阻塞（可接受但非理想） | + +**修复方案**: 全部包裹 `asyncio.to_thread()`。`collection.count()` 结果可缓存。 + +#### 3.2 会话双重提交 + +`conversations.py` 中 create/update/delete 手动调用 `await db.commit()`，而 `get_session()` 在 yield 后也会自动 commit。 + +```python +# database.py — get_session 自动 commit +yield session +await session.commit() # ← 第二次 + +# conversations.py — 手动 commit +await db.commit() # ← 第一次 +``` + +**修复方案**: 移除 `conversations.py` 中手动 commit，依赖 `get_session()` 的自动 commit。同样检查 `pipelines/chat/nodes.py:persist_node`（line 430 手动 commit）。 + +#### 3.3 异常吞没 + +| 位置 | 问题 | +|------|------| +| `rag_service.py:206-207` | `except Exception: pass` — 上下文扩展失败静默忽略 | +| `rag_service.py:393-395` | `get_stats` 异常返回空结果，无日志 | +| `completion_service.py:69` | 异常时返回空补全，丢失错误信息 | +| `main.py:75` | MCP server 挂载失败静默处理 | + +**修复方案**: 至少添加 `logger.warning`/`logger.debug`；关键路径应传播异常。 + +### P1 — 中优先级（影响可维护性和性能） + +#### 3.4 提示词散落与重复 + +提示词分布在 10+ 个文件中，没有集中管理。具体重复： +- "You are a scientific terminology expert. Return valid JSON only." — 出现在 `keyword_service.py` 和 `keywords.py` +- "You are a deduplication expert. Return valid JSON only." — 出现在 `dedup_service.py` 和 `dedup.py` +- "You are a scientific writing expert. Generate well-structured review outlines." — 在 `writing_service.py` 中出现两次 +- 关键词扩展的 user prompt 在 `keyword_service.py` 和 `keywords.py` 中各有一份，内容不同 + +**修复方案**: 见第四节「提示词审计」。 + +#### 3.5 API 层包含业务逻辑 + +`api/v1/dedup.py` 的 `auto_resolve_conflict` (line 198-222) 和 `api/v1/keywords.py` 的 `expand_keywords` (line 119-130) 直接在 API 层构建 LLM prompt 并调用 LLM，而不是委托给 service。 + +**修复方案**: 将 LLM 调用移到对应 service 中，API 层只做请求分发。 + +#### 3.6 缺少分页 + +| 端点 | 问题 | +|------|------| +| `keywords.py:list_keywords` | 返回所有关键词，无分页 | +| `subscription.py:list_subscriptions` | 返回所有订阅，无分页 | +| `tasks.py:list_tasks` | 只有 `limit=50`，无页码 | + +#### 3.7 写作服务串行 LLM 调用 + +`writing_service.py:summarize_papers` 逐篇调用 LLM 生成摘要，可用 `asyncio.gather` 并行化。同理 `generate_citations` 虽无 LLM 调用但可类似优化。 + +#### 3.8 去重算法 O(n²) 复杂度 + +`title_similarity_dedup` 和 `find_llm_dedup_candidates` 使用双重循环对比所有论文标题，O(n²) 复杂度。当论文数量大时性能差。 + +**修复方案**: 可考虑分桶（首字母/长度）减少比较次数，或使用 MinHash/SimHash 近似算法。 + +### P2 — 低优先级（改进体验和代码质量） + +#### 3.9 资源 404 检查重复 + +`subscription.py` 中相同的 404 检查模式重复 4 次，`papers.py`、`keywords.py` 中也有类似模式。 + +**修复方案**: 抽取 `get_resource_or_404(db, Model, id, project_id)` 通用依赖。 + +#### 3.10 健康检查需要认证 + +`/api/v1/settings/health` 不在 `EXEMPT_PATHS` 中，设置 `api_secret_key` 后负载均衡器无法直接访问。 + +#### 3.11 硬编码配置值 + +| 位置 | 值 | 建议 | +|------|---|------| +| `upload.py:28` | `MAX_FILE_SIZE_MB = 50` | 移入 config | +| `rate_limit.py:15` | `"120/minute"` | 移入 config | +| `settings_api.py:65` | 版本号 `"0.1.0"` | 移入 config 或 pyproject.toml | +| `chat/nodes.py:56` | `_clean_semaphore = Semaphore(3)` | 移入 config | +| `rewrite.py:23` | `_rewrite_semaphore = Semaphore(3)` | 移入 config | + +#### 3.12 pyproject.toml 依赖清理 + +`aiohttp` 在依赖列表中但未使用（项目使用 `httpx`），应移除。 + +#### 3.13 conversation 列表查询中的内存过滤 + +`list_conversations` 在 `knowledge_base_id` 过滤时，先从 DB 取出数据再在 Python 中过滤（line 70-73），导致分页计数不准确。应在 SQL 层完成过滤。 + +--- + +## 四、提示词审计 + +### 4.1 提示词分布总览 + +共发现 **16 个不同的提示词**，分布在 8 个文件中： + +| # | 文件 | 用途 | system prompt | 语言 | 质量评级 | +|---|------|------|---------------|------|---------| +| 1 | `chat/nodes.py` | QA 模式 | "You are a scientific research assistant..." | EN | ★★★☆ | +| 2 | `chat/nodes.py` | Citation lookup | "You are a citation finder..." | EN | ★★★☆ | +| 3 | `chat/nodes.py` | Review outline | "You are a literature review expert..." | EN | ★★★☆ | +| 4 | `chat/nodes.py` | Gap analysis | "You are a research gap analyst..." | EN | ★★★☆ | +| 5 | `chat/nodes.py` | 无 KB 回退 | "You are a helpful scientific research assistant..." | EN | ★★☆☆ | +| 6 | `chat/nodes.py` | OCR 清洗 | "Clean up the following text..." | EN | ★★★★ | +| 7 | `writing_service.py` | 章节撰写 | "你是一位学术综述写作专家..." | **ZH** | ★★★☆ | +| 8 | `writing_service.py` | 论文摘要 | "You are a scientific paper analyst..." | EN | ★★☆☆ | +| 9 | `writing_service.py` | 综述提纲 | "You are a scientific writing expert..." | EN | ★★☆☆ | +| 10 | `writing_service.py` | 差距分析 | "You are a research gap analyst..." | EN | ★★☆☆ | +| 11 | `rag_service.py` | 知识库问答 | "You are a scientific research assistant..." | EN | ★★★☆ | +| 12 | `dedup_service.py` | LLM 去重 | "You are a scientific literature deduplication expert..." | EN | ★★★☆ | +| 13 | `dedup.py` (API) | 自动解决冲突 | "You are a deduplication expert..." | EN | ★★☆☆ | +| 14 | `keyword_service.py` | 关键词扩展 | "You are a scientific terminology expert..." | EN | ★★★☆ | +| 15 | `completion_service.py` | 写作补全 | "你是一个科研写作助手..." | **ZH** | ★★★★ | +| 16 | `rewrite.py` | 文本改写 | (4 个子 prompt) | 混合 | ★★★☆ | + +### 4.2 逐项评估与改进建议 + +#### P1: Chat 管道 system prompts (`TOOL_MODE_PROMPTS`) + +**现状**: 4 个模式各有简短 system prompt，功能角色定义清晰，但缺少： +- 输出格式约束（回答长度、结构要求） +- 语言偏好指示 +- 对"不知道"的处理边界更精确的指导 + +**改进建议**: +``` +"qa": 增加 "Structure your answer with clear paragraphs. " + "Respond in the same language as the user's question." +"citation_lookup": 增加 "Include DOI when available." +"review_outline": 增加 "Use markdown headers for sections." +"gap_analysis": 增加 "Organize by theme, not by individual papers." +``` + +#### P2: Chat 无 KB 回退 prompt + +**现状**: "You are a helpful scientific research assistant." — 过于泛化，没有约束。 +**改进建议**: 增加领域约束和行为边界： +``` +"You are a scientific research assistant specializing in academic literature analysis. " +"Answer questions clearly and accurately based on your knowledge. " +"When the user's question is outside your expertise or you are uncertain, say so honestly. " +"Respond in the same language as the user's question." +``` + +#### P3: Writing Service — 语言不一致 + +**现状**: `SECTION_SYSTEM_PROMPT` 用中文，其他 3 个 system prompt 用英文。 +**改进建议**: 统一为英文。中文需求通过 user prompt 中的 `language` 参数控制。 + +#### P4: Writing Service — system prompt 过于简短 + +**现状**: `"You are a scientific paper analyst. Provide concise, accurate summaries."` — 太笼统。 +**改进建议**: +``` +"You are a scientific paper analyst. Provide structured, accurate summaries. " +"Focus on empirical findings and methodology. " +"Do not hallucinate information not present in the provided metadata." +``` + +#### P5: RAG Service 的 system prompt 与 Chat QA 重复 + +**现状**: `rag_service._generate_answer` 和 `chat/nodes.py` 的 QA 模式有高度相似但不完全一致的 system prompt。 +**改进建议**: 统一为同一个常量。 + +#### P6: Dedup — API 层和 Service 层提示词不一致 + +**现状**: +- Service 层: "You are a **scientific literature** deduplication expert." +- API 层: "You are a **deduplication** expert." — 少了 "scientific literature" + +**改进建议**: 移除 API 层的 LLM 调用，统一到 service 层。 + +#### P7: 关键词扩展 — 双重实现 + +**现状**: `keyword_service.py` 和 `keywords.py` 各自独立实现了关键词扩展，user prompt 内容不同。 +**改进建议**: 移除 API 层实现，统一使用 service。 + +#### P8: Completion prompt — 中文硬编码 + +**现状**: `COMPLETION_SYSTEM_PROMPT` 写死中文 "你是一个科研写作助手"。 +**改进建议**: 改为英文，保持一致性： +``` +"You are a scientific writing assistant. Predict and complete the user's text. " +"Return only the completion (do not repeat the user's input), max 50 characters. " +"If you cannot predict, return an empty string. " +"Return plain text only, no quotes, explanations, or formatting." +``` + +#### P9: Rewrite prompts — translate_zh 单独用中文 + +**现状**: 4 个改写 prompt 中有 3 个英文、1 个中文。 +**改进建议**: 统一为英文。 + +### 4.3 提示词集中管理方案 + +**推荐方案**: 创建 `app/prompts/` 模块，按功能域组织： + +``` +app/prompts/ +├── __init__.py # 统一导出 +├── chat.py # Chat 管道 prompts +├── writing.py # 写作助手 prompts +├── rag.py # RAG 知识库 prompts +├── dedup.py # 去重 prompts +├── keyword.py # 关键词 prompts +├── completion.py # 补全 prompts +└── rewrite.py # 改写 prompts +``` + +每个文件导出命名常量： +```python +# app/prompts/chat.py +CHAT_QA_SYSTEM = "You are a scientific research assistant..." +CHAT_CITATION_SYSTEM = "You are a citation finder..." +CHAT_OUTLINE_SYSTEM = "You are a literature review expert..." +CHAT_GAP_SYSTEM = "You are a research gap analyst..." +CHAT_FALLBACK_SYSTEM = "You are a scientific research assistant..." +EXCERPT_CLEAN_SYSTEM = "Clean up the following text..." +``` + +优点： +- 所有提示词集中管理，便于审查和迭代 +- 消除重复定义 +- 方便未来支持用户自定义提示词 +- 可轻松添加多语言支持 + +--- + +## 五、关键决策 + +1. **提示词统一为英文** — LLM 对英文提示词理解最好，用户语言偏好通过 prompt 参数动态传递 +2. **提示词集中到 `app/prompts/` 模块** — 不用外部文件（YAML/JSON），保持 Python 代码的类型安全和重构友好 +3. **API 层不直接调用 LLM** — 所有 LLM 相关逻辑归入 service 层 +4. **所有 LLM 调用都应有 system prompt** — 即使是简单任务（如 connection test）也应有明确角色定义 +5. **异步阻塞修复优先于新功能** — 保证运行时稳定性 + +--- + +## 六、改进计划 + +### Phase 1: 关键修复（1-2 天） + +- [ ] 修复异步中的同步阻塞调用（P0-3.1） + - `subscription_service.py`: 包裹 `feedparser.parse` + - `pdf_metadata.py`: 包裹 `fitz.open` + - `rag_service.py`: 包裹 `collection.count()`，考虑缓存 +- [ ] 修复会话双重提交（P0-3.2） + - 移除 `conversations.py` 中手动 commit + - 检查 `persist_node` 的 commit 行为 +- [ ] 修复异常吞没（P0-3.3） + - RAG service: 至少 `logger.debug` + - Completion service: 添加日志 + - Main.py: MCP 挂载失败应 `logger.error` + +### Phase 2: 提示词重构（1-2 天） + +- [ ] 创建 `app/prompts/` 模块，集中所有提示词 +- [ ] 统一提示词语言为英文 +- [ ] 消除重复的提示词定义 +- [ ] 将 API 层的 LLM 调用移到 service 层 + - `dedup.py:auto_resolve_conflict` → `DedupService.auto_resolve` + - `keywords.py:expand_keywords` → `KeywordService.expand_keywords_with_llm` +- [ ] 改进每个提示词的质量（增加格式约束、语言动态化等） + +### Phase 3: 架构优化（2-3 天） + +- [ ] 添加分页到 keywords, subscriptions, tasks 端点 +- [ ] 并行化 writing_service LLM 调用（`asyncio.gather`） +- [ ] 抽取 `get_resource_or_404` 通用依赖 +- [ ] 修复 conversation 列表的内存过滤问题 +- [ ] 健康检查端点免认证 +- [ ] 硬编码配置移入 `config.py` +- [ ] 清理 `pyproject.toml` 未使用依赖 + +### Phase 4: 进阶改进（可选） + +- [ ] 去重算法优化（MinHash/SimHash 替代 O(n²)） +- [ ] 添加 LLM 调用结果缓存 +- [ ] 提示词版本化和 A/B 测试支持 +- [ ] 用户自定义提示词功能 + +--- + +## 七、已解决问题 + +1. **提示词不需要支持用户自定义** — 开发者直接改代码即可，无需额外存储层 +2. **提示词版本化暂不需要** — 以后再考虑 +3. **去重 O(n²) 不是实际瓶颈** — 当前项目论文规模 < 500 篇，标记为低优先级 +4. **Chat Pipeline 的 persist_node 手动 commit 确实有问题** — 经代码确认，`persist_node` 使用的 session 来自 `Depends(get_db)` → `get_session()`，后者在 yield 后自动 commit。因此 `persist_node:430` 的手动 `await db.commit()` 会导致双重提交，需要移除 + +## 八、剩余开放问题 + +1. **ChromaDB count() 缓存策略** — 缓存 TTL 多长合适？数据频繁变更时如何失效？建议：使用 TTL=60s 的简单内存缓存，index 操作后主动失效 diff --git a/docs/brainstorms/2026-03-17-config-rag-api-testing-brainstorm.md b/docs/brainstorms/2026-03-17-config-rag-api-testing-brainstorm.md new file mode 100644 index 0000000..102f96f --- /dev/null +++ b/docs/brainstorms/2026-03-17-config-rag-api-testing-brainstorm.md @@ -0,0 +1,101 @@ +# Brainstorm: 配置修复 + RAG 召回优化 + 全接口测试 + +**Date**: 2026-03-17 +**Status**: Approved + +## What We're Building + +三阶段改进： +1. **配置一致性修复** — 同步 `config.py` 默认值、`.env.example`、`.env` 三者 +2. **向量召回优化** — 实现 reranking + MMR 多样性 + HNSW 调优（BM25 混合检索留作后续迭代） +3. **全接口文档 + 测试** — 77 个 API 端点的文档化 + pytest 单元测试 + E2E 真实 LLM 测试 + +## Why This Approach + +### 配置修复 +当前 `config.py` 默认值（`BAAI/bge-m3`）与实际使用的 Qwen3 模型脱节，`.env.example` 混入了环境特定配置（`CUDA=5,6,7`）。这会导致新开发者环境搭建困惑。 + +### RAG 优化 +当前向量召回仅有 dense search，reranking 虽有配置但完全未实现（死代码）。对于学术文献检索，缺少精确术语匹配（BM25）和结果多样性（MMR）会严重影响检索质量。 + +### 全接口测试 +上一轮重构修改了 20 个文件，现有 229 个测试只覆盖 mock 场景。需要真实 LLM（Volcengine doubao-seed-2-0-mini）验证端到端行为。 + +## Key Decisions + +1. **配置策略**: `.env.example` 更新为当前实际使用的配置（debug=true, mineru, Qwen3 模型等） +2. **RAG 优化范围**: reranking + MMR + HNSW 调优（BM25 混合检索复杂度高，留作后续迭代） +3. **Embedding 模型**: Qwen/Qwen3-Embedding-0.6B（`.env` 实际）；`.env.example` 推荐 8B 版本 +4. **Reranker 模型**: Qwen/Qwen3-Reranker-0.6B（实际）；config.py 默认更新 +5. **测试范围**: 全部 77 个端点，pytest + E2E 双轨 +6. **测试 LLM**: Volcengine doubao-seed-2-0-mini（真实 LLM）+ mock（现有测试） +7. **测试数据**: 8 篇 VR/生物化学 PDF 位于 `/data0/djx/omelette_pdf_test/` +8. **PaddleOCR**: 保留作为扫描 PDF 后备方案，MinerU 为主力 parser + +## Resolved Questions + +- **Embedding 模型默认值**: config.py 默认更新为 Qwen3-Embedding-0.6B（与 .env 一致） +- **Reranker 实现方式**: 使用 LlamaIndex 内置 reranker + Qwen3-Reranker +- **混合检索方案**: 暂不实现 BM25，reranker 已能弥补精确匹配不足；后续如需可用 LlamaIndex QueryFusionRetriever +- **测试并行化**: 按模块拆分，开多个 Agent 并行测试不同模块 +- **服务器状态**: 未运行，需要启动用于 E2E 测试 + +## Architecture Notes + +### 向量召回优化架构 + +``` +User Question + ↓ +Dense Retrieval: Qwen3-Embedding → ChromaDB HNSW (cosine, top_k * 3) + ↓ +Reranking: Qwen3-Reranker (shrink to top_k) + ↓ +MMR Diversity Filter (reduce redundant chunks from same paper/section) + ↓ +Adjacent Chunk Expansion (window=1) + ↓ +Sources → LLM Generation +``` + +**后续迭代（不在本次范围）**: BM25 稀疏检索 + RRF 融合、Query Expansion + +### 测试架构 + +``` +Test Suite +├── pytest (unit + integration) +│ ├── mock LLM → 现有 229 测试 +│ └── real LLM → 新增 Volcengine 测试 +└── E2E (live server) + ├── 启动 FastAPI server (port 8000) + ├── 多 Agent 并行测试各模块 + └── 测试数据: 8 篇 PDF 论文 +``` + +## Risks & Constraints + +1. **Reranker GPU 内存**: Qwen3-Reranker-0.6B 需要额外 GPU 内存；与 embedding 模型共用 GPU 5,6,7 可能有竞争 +2. **E2E 测试稳定性**: 真实 LLM 响应不确定，测试断言需要模糊匹配而非精确比对 +3. **MinerU 服务依赖**: E2E 测试 OCR/upload 流程需要 MinerU 服务在 localhost:8010 运行 +4. **Volcengine 速率限制**: 并行多 Agent 同时调用 LLM 可能触发 API 限流 +5. **Scope creep**: 77 个端点全测容易失控，需严格分批 + 时间框约束 + +## Scope Guard + +**本次不做**: +- BM25 混合检索 +- Query Expansion +- 更换向量数据库（ChromaDB → Milvus/Qdrant） +- 前端测试 +- 性能基准测试 + +### API 端点分类（按测试复杂度） + +| 类别 | 端点数 | 说明 | +|------|--------|------| +| **纯 CRUD** | ~30 | Projects, Papers, Keywords, Conversations, Subscriptions, Tasks | +| **LLM 依赖** | ~15 | Chat, RAG query, Writing, Dedup verify/resolve, Keyword expand, Rewrite, Completion | +| **管线/异步** | ~10 | Pipelines (search/upload), Crawler, OCR, RAG index | +| **配置/状态** | ~7 | Settings, Health, Search sources | +| **文件 I/O** | ~5 | PDF upload/serve, OCR process | diff --git a/docs/brainstorms/2026-03-17-mineru-gpu-parallel-e2e-brainstorm.md b/docs/brainstorms/2026-03-17-mineru-gpu-parallel-e2e-brainstorm.md new file mode 100644 index 0000000..223eccb --- /dev/null +++ b/docs/brainstorms/2026-03-17-mineru-gpu-parallel-e2e-brainstorm.md @@ -0,0 +1,100 @@ +--- +date: 2026-03-17 +topic: mineru-gpu-parallel-e2e-testing +--- + +# MinerU 启用 + GPU 多卡并行 + 全面 E2E 测试 + +## What We're Building + +三阶段优化： + +1. **启动 MinerU 服务并验证 PDF 解析质量** — MinerU v2.7.6 已安装在 conda `mineru` 环境中，但从未在测试中使用过。需要启动服务、确认 API 兼容性、并与 pdfplumber 进行解析质量对比。 + +2. **PDF 处理并行化 + GPU 多卡分配** — 当前论文处理是串行 `for` 循环，三块 GPU (5,6,7) 只有第一块在工作。需要实现并行 OCR/MinerU 处理，并让 Embedding、Reranker、OCR 自由分配到可用 GPU 上。同时保持对单 GPU 和纯 CPU 环境的兼容。 + +3. **全面 E2E 测试** — 冒烟测试（核心流程）+ 压力测试（8 篇 PDF 并发上传+查询）+ MinerU vs pdfplumber 解析质量对比。 + +## Why This Approach + +### 现状问题 + +| 问题 | 影响 | +|------|------| +| MinerU 零测试覆盖 | 无法验证主力 PDF 解析器是否正常工作 | +| 串行处理 8 篇 PDF | GPU 利用率 < 5%，用户等待时间线性增长 | +| 三块 GPU 只用一块 | 2/3 的 GPU 资源浪费 | +| 所有模型挤一张卡 | Embedding(0.6B) + Reranker(0.6B) + PaddleOCR 可能 OOM | +| E2E 测试未覆盖 MinerU 路径 | pdfplumber 回退掩盖了真实环境行为 | + +### 选择此方案的原因 + +- **MinerU 优先**：它是主力解析器（`pdf_parser=mineru`），不测它等于不测真实环境 +- **仅并行化 OCR**：OCR/MinerU 是处理瓶颈（每篇 PDF 数十秒），Embedding 索引本身是批量操作 +- **GPU 自由分配**：比手动指定更灵活，让 PyTorch 的 CUDA 内存管理自动处理 + +## Key Decisions + +- **MinerU 启动端口**: 8010（与 `config.py` 中 `mineru_api_url` 默认值一致） +- **MinerU 后端模式**: `pipeline`（通用性强，支持多语言，无幻觉） +- **并行度控制**: 通过 `asyncio.Semaphore` 限制并发 OCR 数量（默认=可用 GPU 数量或 3） +- **GPU 分配策略**: `CUDA_VISIBLE_DEVICES` 环境变量 + 轮转分配（round-robin），兼容单 GPU 和 CPU +- **MinerU 并发**: MinerU 服务自带 `_request_semaphore` 控制并发 +- **测试分层**: 冒烟（基本流程）→ 压力（并发）→ 质量（MinerU vs pdfplumber） +- **知识沉淀**: MinerU 启动方式写入项目 README/docs + +## Resolved Questions + +- **MinerU conda 环境名**: `mineru` +- **MinerU 版本**: 2.7.6 +- **MinerU 启动命令**: `conda run -n mineru python -m mineru.cli.fast_api --host 0.0.0.0 --port 8010` +- **MinerU API 端点**: `POST /file_parse`（与 `mineru_client.py` 匹配） +- **优先级**: MinerU → GPU 并行 → 全面测试（按顺序） +- **并行范围**: 仅 PDF 解析/OCR 并行（它是瓶颈） +- **GPU 分配**: 三张卡都可用，并行任务自由分配，同时兼容单 GPU 和纯 CPU +- **成功标准**: 所有 E2E 测试通过，包括 MinerU 路径 + +## Architectural Notes + +### MinerU 服务启动 + +```bash +# 在 conda mineru 环境中启动 MinerU FastAPI 服务 +conda run -n mineru python -m mineru.cli.fast_api --host 0.0.0.0 --port 8010 + +# MinerU 默认使用 pipeline 后端，可通过请求参数覆盖 +# 支持 GPU 加速（自动检测） +``` + +### GPU 轮转分配方案 + +``` +GPU 5: Embedding (常驻) + OCR Worker 1 +GPU 6: Reranker (按需加载) + OCR Worker 2 +GPU 7: OCR Worker 3 +``` + +但实际实现时用 `CUDA_VISIBLE_DEVICES=5,6,7` + 轮转： +- Worker 0 → device 0 (实际 GPU 5) +- Worker 1 → device 1 (实际 GPU 6) +- Worker 2 → device 2 (实际 GPU 7) + +### 并行 OCR 架构 + +``` +paper_processor.py: + before: for paper in papers: await ocr(paper) # 串行 + after: await asyncio.gather(*[ocr(paper, gpu_id=i%n) for i, paper in enumerate(papers)]) # 并行 +``` + +## Scope Guard + +**不做**： +- 不改 MinerU 服务本身的代码 +- 不做 Embedding 模型的多 GPU 分布式推理（tensor parallel） +- 不做 RAG 索引的并行化（ChromaDB 本身是单线程写入） +- 不做前端改动 + +## Next Steps + +→ `/ce-plan` 生成实施计划 diff --git a/docs/brainstorms/2026-03-18-backend-comprehensive-review-brainstorm.md b/docs/brainstorms/2026-03-18-backend-comprehensive-review-brainstorm.md new file mode 100644 index 0000000..9558c40 --- /dev/null +++ b/docs/brainstorms/2026-03-18-backend-comprehensive-review-brainstorm.md @@ -0,0 +1,175 @@ +--- +title: "后端全面代码审核 — 改进建议汇总" +date: 2026-03-18 +status: approved +tags: [backend, review, code-quality, security, testing] +--- + +# 后端全面代码审核 — 改进建议汇总 + +## 背景 + +在完成 GPU 资源自动管理（TTL + MinerU 子进程自治）功能并通过全量测试（572 通过，3 跳过，0 失败）后，对整个后端代码进行全面审核。审核覆盖四个维度：服务层代码质量、API 设计与安全、配置/模型/数据库/Pipeline、测试覆盖与质量。 + +## 发现汇总 + +共发现 **48 项** 改进点，按优先级分为： + +| 优先级 | 数量 | 主要关注 | +|--------|------|----------| +| P0（高） | 10 | 异步阻塞、资源泄漏、安全漏洞、数据一致性 | +| P1（中） | 18 | 配置一致性、输入验证、代码重复、测试覆盖 | +| P2（低） | 20 | 文档、国际化、风格、OpenAPI 完善 | + +--- + +## P0 — 高优先级（必须修复） + +### 1. 异步上下文中的阻塞调用 + +**影响**: 拖慢整个异步应用，降低并发吞吐量 + +| 文件 | 问题 | 修复方案 | +|------|------|----------| +| `crawler_service.py:84-86` | `validate_url_safe()` 内部 `socket.getaddrinfo()` 阻塞 | `await asyncio.to_thread(validate_url_safe, url)` | +| `mineru_process_manager.py:213-216` | `_kill_process()` 中 `process.wait(timeout=10)` 阻塞 | `await asyncio.to_thread(self._process.wait, timeout)` | +| `mineru_process_manager.py:192` | `process.stderr.read()` 阻塞 | `await asyncio.to_thread(...)` 或 asyncio 子进程 | + +### 2. 资源泄漏 + +| 文件 | 问题 | 修复方案 | +|------|------|----------| +| `ocr_service.py:177-188` | `fitz.open(pdf_path)` 异常时不关闭 | 改用 `with fitz.open(pdf_path) as pdf_doc:` | + +### 3. 安全漏洞 + +| 文件 | 问题 | 修复方案 | +|------|------|----------| +| `subscription_service.py:22-27` | RSS `feed_url` 未做 SSRF 校验 | 调用 `validate_url_safe(feed_url)` | +| `rag.py` 多个接口 | 未校验 `project_id` 对应的项目是否存在 | 添加 `project: Project = Depends(get_project)` | +| `subscription.py` 部分接口 | `list_common_feeds`/`check_rss`/`check_updates` 未校验 project | 同上 | +| `search.py:list_search_sources` | 路径含 `project_id` 但未校验 | 同上 | +| `gpu.py:gpu_unload` | `POST /gpu/unload` 可释放所有 GPU，无认证保护 | 添加 API key 校验 | + +### 4. 数据一致性 + +| 文件 | 问题 | 修复方案 | +|------|------|----------| +| `models.py` | Paper 缺少 `(project_id, doi)` 唯一约束 | 添加 UniqueConstraint（doi 非空时） | + +### 5. Pipeline 内存泄漏 + +| 文件 | 问题 | 修复方案 | +|------|------|----------| +| `pipelines.py` | `_cancelled` 和 `_running_tasks` 完成后不清理 | 任务结束时 `del _cancelled[thread_id]` 和 `_running_tasks.pop(thread_id)` | +| `pipelines.py` + `nodes.py` | **真实 Bug**: `ResolvedConflict` schema 只有 `merged_paper`，但 `apply_resolution_node` 用 `res.get("new_paper")` 读取，导致 `keep_new` action 拿到空 dict，论文丢失 | 在 `ResolvedConflict` 添加 `new_paper` 字段，或 node 中统一读 `merged_paper` | + +--- + +## P1 — 中优先级（建议修复） + +### 代码质量 + +| # | 文件 | 问题 | 修复方案 | +|---|------|------|----------| +| 1 | 多文件 | GPU 内存清理逻辑重复（`gc.collect()` + `torch.cuda.empty_cache()`） | 抽取到 `gpu_utils.py` 公共函数 | +| 2 | `pipeline_service.py:79-80` | 使用 `asyncio.to_thread(ocr.process_pdf)` 而非 `process_pdf_async`（缺少 MinerU 优先） | 统一使用 `process_pdf_async` | +| 3 | `llm_config_resolver.py:101-103` | temperature/max_tokens 回退到 `settings` 而非 `merged_settings` | 使用 `merged_settings` | +| 4 | `ocr_service.py:182` | 硬编码 `/tmp/omelette_ocr_page_*.png` | 使用 `tempfile.gettempdir()` | +| 5 | `embedding_service.py:124,127` | lambda + `noqa: E731` | 改为显式辅助函数 | +| 6 | `pipelines/nodes.py` | `_is_cancelled` 反向依赖 `app.api.v1.pipelines._cancelled` | 抽到共享模块 | + +### 输入验证 + +| # | 文件 | 问题 | 修复方案 | +|---|------|------|----------| +| 7 | 多文件 | 分页参数不统一（`page_size` 默认值 20/50 不一致，缺少 `ge`/`le`） | 统一 `PaginationParams` | +| 8 | `dedup.py` | `strategy` 未做白名单校验 | 改用 `Literal["full","doi_only","title_only"]` | +| 9 | `crawler.py` | `priority` 可为任意字符串 | 改用 `Literal["high","low"]` | +| 10 | `search.py` | `execute_search` 参数未用 Pydantic 模型 | 定义 `SearchExecuteRequest` | +| 11 | `projects.py:import_project` | 导入数据为 `list[dict]` 直接解包 | 定义 Pydantic 导入模型 | + +### 基础设施 + +| # | 文件 | 问题 | 修复方案 | +|---|------|------|----------| +| 12 | `main.py` | shutdown 时未 `engine.dispose()` | 添加 `await engine.dispose()` | +| 13 | `database.py` | 未配置连接池参数（`pool_size`、`pool_pre_ping`） | 显式设置 | +| 14 | `pipelines.py` | MemorySaver 重启后状态丢失 | 评估 AsyncSqliteSaver | + +### 测试覆盖 + +| # | 缺失测试 | 重要性 | +|---|----------|--------| +| 15 | `pdf_metadata` 服务 — 无任何测试 | 高（上传核心） | +| 16 | `papers bulk_import` API | 高 | +| 17 | `papers list_paper_chunks` API | 高 | +| 18 | Search → Dedup → Crawl → OCR → RAG 完整 E2E 链路 | 中 | + +--- + +## P2 — 低优先级（可选改进） + +| # | 问题 | 说明 | +|---|------|------| +| 1 | 中文硬编码错误信息（`citation_graph_service`） | i18n 不友好 | +| 2 | `user_settings_service` 模型列表硬编码 | 扩展性差 | +| 3 | `pdf_metadata` Crossref User-Agent fallback 邮箱硬编码 | 配置化 | +| 4 | `reranker_service` top_n 与 batch_size 混用 | 语义不清 | +| 5 | `crawler_service` batch results 并发更新无锁 | 理论竞态 | +| 6 | 多数 endpoint 缺少 summary/description | OpenAPI 不完整 | +| 7 | Tags 使用不统一（`rewrite` 在 `/chat` 路由） | 文档混乱 | +| 8 | SSE 错误响应格式各异 | 前端处理复杂 | +| 9 | `test_connection` 异常时 code=500 但 HTTP 200 | 状态码不一致 | +| 10 | Streaming 接口缺少 response_model | OpenAPI 描述不完整 | +| 11 | 部分列表接口缺分页（`list_subscriptions`） | 大数据量问题 | +| 12 | 限流覆盖不完整（upload/search/dedup/writing） | 资源保护 | +| 13 | `dedup candidates` 未分页 | 大数据量 | +| 14 | 测试 fixture 重复（`setup_db`/`client` 在 6+ 文件） | 统一到 conftest | +| 15 | 测试过度依赖 happy path | 错误路径覆盖不足 | +| 16 | 测试中 `sleep` 等待异步（脆弱） | 改用轮询/事件 | +| 17 | `test_new_features` 直接修改模块全局状态 | 测试隔离差 | +| 18 | `cuda_visible_devices` 默认 `"6,7"` | 非通用默认 | +| 19 | Conversation `knowledge_base_ids` 存 JSON | 查询不便 | +| 20 | Keyword `parent_id` 缺索引 | 查询性能 | + +--- + +## 建议实施优先级 + +### 立即修复（P0 — 影响运行时安全和正确性） + +1. 异步阻塞调用修复（3 处 `asyncio.to_thread`） +2. `fitz.open` 资源泄漏 +3. SSRF 校验（subscription feed_url） +4. API project_id 校验（rag、subscription、search） +5. Pipeline `_cancelled`/`_running_tasks` 清理 +6. ResolvedConflict 字段不一致 + +### 短期改进（P1 — 提升代码健壮性） + +7. GPU 清理逻辑复用 +8. 输入验证统一（分页、strategy、priority） +9. `engine.dispose()` + 连接池配置 +10. 补充 `pdf_metadata` 测试 + +### 长期优化（P2 — 可维护性和完善度） + +11. OpenAPI 文档完善 +12. 测试 fixture 统一 +13. SSE 错误格式统一 +14. 限流扩展 + +## 关键决策 + +| 决策 | 选择 | 理由 | +|------|------|------| +| 修复范围 | 全部 48 项（P0 + P1 + P2） | 一次性全面提升代码质量 | +| Pipeline checkpoint | 实施 MemorySaver → AsyncSqliteSaver | 重启后 pipeline 状态不应丢失 | +| Paper 唯一约束 | 需要，含数据迁移 | 防止重复导入是核心需求 | + +## 已解决问题 + +1. ~~Pipeline checkpoint 持久化~~ → 需要实施 +2. ~~Paper 唯一约束~~ → 需要，创建 Alembic 迁移 +3. ~~优先级顺序~~ → 全部修复，按 P0 → P1 → P2 顺序 diff --git a/docs/brainstorms/2026-03-18-backend-deep-audit-brainstorm.md b/docs/brainstorms/2026-03-18-backend-deep-audit-brainstorm.md new file mode 100644 index 0000000..2564ab5 --- /dev/null +++ b/docs/brainstorms/2026-03-18-backend-deep-audit-brainstorm.md @@ -0,0 +1,299 @@ +# 后端深度审计：接口补充与代码改进 + +**日期**: 2026-03-18 +**状态**: 已审核 +**分支**: `refactor/backend-comprehensive-optimization` + +--- + +## 我们要解决什么 + +对 Omelette 后端全部代码进行深度扫描，识别需要补充的接口、需要修复的 Bug、安全隐患、性能瓶颈和代码质量问题，形成可操作的改进清单。 + +## 审计方法 + +- 逐模块扫描全部 API 路由（17 个路由文件，76+ 端点） +- 检查全部 Service（18 个服务模块） +- 检查全部 Model（10 个数据模型） +- 检查全部 Schema（12 个 schema 文件） +- 检查 MCP Server（7 个 Tool + 4 个 Resource + 2 个 Prompt） +- 检查中间件、配置、Pipeline 系统 +- 交叉比对前后端接口需求 + +--- + +## 一、错误处理一致性 [高优先级] + +### 问题 +后端约定所有响应使用 `ApiResponse` 格式 `{"code": ..., "message": ..., "data": ...}`，但 `HTTPException` 和 Pydantic `RequestValidationError` 使用 FastAPI 默认格式 `{"detail": "..."}`，导致前端需要处理两种错误格式。 + +### 具体表现 + +| 场景 | 当前行为 | 期望行为 | +|------|---------|---------| +| `HTTPException(404)` | `{"detail": "Not found"}` | `{"code": 404, "message": "Not found", "data": null}` | +| Pydantic 验证失败(422) | `{"detail": [{...}]}` | `{"code": 422, "message": "Validation error", "data": [{...}]}` | +| 全局异常(500) | ApiResponse 格式 ✓ | 已正确 | +| Auth 中间件(401) | ApiResponse 格式 ✓ | 已正确 | + +### 建议 +在 `main.py` 添加 `HTTPException` 和 `RequestValidationError` 的自定义 handler： + +```python +from fastapi.exceptions import RequestValidationError + +@app.exception_handler(HTTPException) +async def http_exception_handler(request, exc): + return JSONResponse( + status_code=exc.status_code, + content={"code": exc.status_code, "message": exc.detail, "data": None}, + ) + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request, exc): + return JSONResponse( + status_code=422, + content={"code": 422, "message": "Validation error", "data": exc.errors()}, + ) +``` + +### 附加问题 +- `writing.py:87-91`：未知 task 返回 200 + `code=400`，应该改为 `HTTPException(400)` +- `delete_paper` 返回 200，REST 惯例为 204 No Content + +--- + +## 二、输入验证加强 [高优先级] + +### Schema 缺失验证 + +| Schema | 字段 | 问题 | 建议 | +|--------|------|------|------| +| `PaperCreate` | `title` | 无 `max_length` | 加 `max_length=2000` | +| `PaperCreate` | `abstract` | 无长度限制 | 加 `max_length=50000` | +| `PaperCreate` | `year` | 无范围限制 | 加 `ge=1800, le=2100` | +| `PaperCreate` | `citation_count` | 无上界 | 加 `ge=0` | +| `PaperCreate` | `authors` | `list[dict[str, str]]` 无结构约束 | 定义 `AuthorSchema(name: str, affiliation: str = "")` | +| `PaperCreate` | `pdf_url` | 无 URL 格式验证 | 加 `HttpUrl` 类型或正则验证 | +| `PaperBulkImport` | `papers` | 无列表长度限制 | 加 `max_length=500` | +| `PaperUpdate` | `status` | 纯 `str`，可任意值 | 用 `Literal[...]` 或 `PaperStatus` 枚举 | +| `SubscriptionCreate` | `frequency` | 纯 `str` | 用 `Literal["daily", "weekly", "monthly"]` | +| `ConversationCreateSchema` | `tool_mode` | 纯 `str` | 用 `Literal["qa", "citation_lookup", "review_outline", "gap_analysis"]` | +| `ResolveConflictRequest` | `action` | 纯 `str` | 用 `Literal["keep_old", "keep_new", "merge", "skip"]` | +| `KeywordExpandRequest` | `seed_terms` | 无 `max_length` | 加 `max_length=50` 防止过大 LLM 调用 | +| `SettingsUpdateSchema` | API key 字段 | 无长度限制 | 加 `max_length=500` | +| `NewPaperData` | `title` | 无 `max_length` | 加 `max_length=2000` | +| `ChatStreamRequest` | `tool_mode` | 纯 `str` | 用 `Literal[...]` 约束 | + +--- + +## 三、安全问题 [高优先级] + +### 3.1 SSRF 风险 +- **位置**: `crawler_service.py`、`mcp_server.py` +- **描述**: 用户可通过 `pdf_url` 字段传入内网地址（如 `http://169.254.169.254/`、`http://localhost:6379/`），Crawler 会直接请求 +- **建议**: 添加 URL 安全验证函数，禁止私有 IP 和 localhost + +### 3.2 DOI 注入 +- **位置**: `mcp_server.py:244` — `f"https://api.crossref.org/works/{doi}"` +- **描述**: 畸形 DOI 可能影响 URL 构造 +- **建议**: 用正则验证 DOI 格式 `^10\.\d{4,9}/[-._;()/:A-Z0-9]+$` + +### 3.3 API Key 通过 Query 参数传输 +- **位置**: `middleware/auth.py` +- **描述**: 支持 `?api_key=xxx` 查询参数传递 API key，会记录在访问日志和浏览器历史中 +- **建议**: 仅允许 `X-API-Key` Header 传递 + +### 3.4 CORS 配置不完整 +- **位置**: `main.py:45-51` +- **描述**: 未设置 `expose_headers`（自定义 Header 对前端不可见）和 `max_age`（每次预检请求都重新发） +- **建议**: 添加 `expose_headers=["X-Request-ID"]`, `max_age=600` + +--- + +## 四、性能问题 [中优先级] + +### 4.1 OCR 阻塞事件循环 +- **位置**: `ocr.py:54` +- **描述**: `service.process_pdf()` 是同步方法，在 async 端点中直接调用会阻塞事件循环 +- **建议**: 改用 `await asyncio.to_thread(service.process_pdf, ...)` 或 `service.process_pdf_async()` + +### 4.2 数据库缺少复合索引 +- **位置**: `models/paper.py`, `models/task.py` +- **描述**: `papers` 表常按 `(project_id, status)` 查询，`tasks` 表常按 `(project_id, status)` 查询，但缺少复合索引 +- **建议**: 添加复合索引： + - `Index("ix_paper_project_status", "project_id", "status")` + - `Index("ix_task_project_status", "project_id", "status")` + - `Index("ix_conversation_updated_at", "updated_at")` （列表排序用） + +### 4.3 OCR 临时文件清理 +- **位置**: `ocr_service.py:150-155` +- **描述**: 创建 `/tmp/omelette_ocr_page_*.png` 但若异常发生在 `unlink` 前文件不会清理 +- **建议**: 改用 `try/finally` 或 `tempfile.NamedTemporaryFile` + +### 4.4 Rate Limiting 不区分端点 +- **位置**: `middleware/rate_limit.py` +- **描述**: 全局统一限速（120/min），重操作（RAG 建索引、chat stream、upload）和轻操作共享同一限额 +- **建议**: 对重操作添加单独限速 `@limiter.limit("10/minute")` + +### 4.5 Rate Limiting 内存存储 +- **位置**: `rate_limit.py:18` +- **描述**: `storage_uri="memory://"` 不持久、不支持多 worker +- **建议**: 短期可接受（单进程），长期换 Redis + +--- + +## 五、Pipeline 系统问题 [中优先级] + +### 5.1 状态仅在内存中 +- **位置**: `pipelines.py:18` — `_running_tasks: dict[str, dict] = {}` +- **描述**: 所有 Pipeline 状态存在内存中，进程重启后全部丢失 +- **建议**: 将状态写入 Task 表，利用已有的 `tasks` 模型 + +### 5.2 取消操作实际无效 +- **位置**: `pipelines.py:240-248` +- **描述**: `cancel_pipeline` 只设置 `task["status"] = "cancelled"`，但 Pipeline 节点不检查此状态，Graph 仍在运行 +- **建议**: 在 state 中设置 `cancelled=True`，并在每个节点开头检查 `if state.get("cancelled"): return state` + +### 5.3 ResumeRequest 缺乏类型约束 +- **位置**: `pipelines.py:33-34` +- **描述**: `resolved_conflicts: list[dict] = []` 没有任何 schema 约束 +- **建议**: 定义 `ResolvedConflict(conflict_id: str, action: Literal[...], merged_paper: dict | None)` + +### 5.4 Pipeline 无列表端点 +- **描述**: 无法查看所有正在运行/已中断的 Pipeline +- **建议**: 添加 `GET /api/v1/pipelines` 列表端点 + +### 5.5 asyncio.create_task 未保存引用 +- **位置**: `pipelines.py:86, 153, 235` +- **描述**: `asyncio.create_task(_run())` 创建的 Task 无引用，如果 GC 回收会静默取消 +- **建议**: 保存 task 引用到 `_running_tasks[thread_id]["asyncio_task"]` + +--- + +## 六、MCP Server 工具补全 [中优先级] + +### 当前覆盖 +- ✅ 知识库列表/搜索 +- ✅ 论文查找/添加/摘要 +- ✅ 引用查找 +- ✅ 关键词搜索 + +### 缺失工具 + +| 工具名 | 功能 | 重要性 | +|--------|------|--------| +| `summarize_papers` | 对知识库中的论文进行摘要总结 | 高 | +| `generate_review_outline` | 生成综述大纲 | 高 | +| `analyze_gaps` | 研究空白分析 | 中 | +| `manage_keywords` | 关键词管理（创建/扩展/搜索公式） | 中 | +| `start_pipeline` | 启动搜索/上传 Pipeline | 中 | +| `manage_subscriptions` | 订阅管理 | 低 | +| `run_dedup` | 运行去重 | 低 | + +### 输入验证 +- `top_k` 和 `max_results` 无上下界 +- `summary_type` 不校验合法值 +- DOI 无格式验证 + +--- + +## 七、缺失的 API 端点 [中优先级] + +| 端点 | 用途 | 建议路径 | +|------|------|---------| +| 批量删除论文 | 前端批量操作 | `POST /projects/{id}/papers/batch-delete` | +| Pipeline 列表 | 查看所有运行中 Pipeline | `GET /pipelines` | +| 项目统计 | 仪表盘数据聚合 | `GET /projects/{id}/stats` | +| 项目导出 | 备份/迁移 | `GET /projects/{id}/export` | +| 项目导入 | 恢复 | `POST /projects/import` | +| 论文标签管理 | 批量打标签 | `POST /projects/{id}/papers/batch-tag` | +| Subscription trigger 导入论文 | 订阅触发后自动导入 | 增强 `trigger_subscription` | +| 健康检查根路由 | 与 auth exempt 一致 | `GET /health` | + +### Subscription trigger 不导入论文 +- **位置**: `subscription.py:139-173` +- **描述**: `trigger_subscription` 检查到新论文后只更新计数，不会将论文导入到项目中 +- **建议**: 增加自动导入选项 `auto_import: bool = False` + +--- + +## 八、代码质量与一致性 [低优先级] + +### 8.1 LLM 客户端双入口 +- `services/llm_client.py` 是 `services/llm/client.py` 的 shim +- `api/deps.py` 用 `llm_client`，`chat pipeline` 用 `llm.client` +- **建议**: 统一入口，删除 shim 文件 + +### 8.2 Conversation 与 Project 无 FK 关系 +- `Conversation.knowledge_base_ids` 存储项目 ID 列表（JSON），但无外键约束 +- 使用 `json_each` 原始 SQL 查询 +- **建议**: 短期可接受（多对多用 JSON 存储），但考虑添加关联表 + +### 8.3 Schema 导出不完整 +- `schemas/__init__.py` 未导出部分 schema（`SubscriptionRead`, `ConversationSchema`, `ChunkRead` 等） +- **建议**: 统一在 `__init__.py` 导出所有公开 schema + +### 8.4 Auth Exempt 路径不一致 +- `/health` 在 exempt 列表中但不存在该路由 +- 实际健康检查在 `/api/v1/settings/health` +- **建议**: 删除不存在的路径，或添加 `/health` 路由 + +--- + +## 九、测试覆盖空白 [低优先级] + +### 缺少专项测试的模块 + +| 模块 | 当前状态 | 建议 | +|------|---------|------| +| `middleware/auth.py` | 无专项测试 | 添加 auth 中间件测试 | +| `middleware/rate_limit.py` | 无专项测试 | 添加限速测试 | +| `reranker_service.py` | 仅通过 RAG 测试间接覆盖 | 添加独立测试 | +| Pipeline 取消/恢复 | 部分覆盖 | 增强取消和恢复的边界测试 | +| MCP 工具输入验证 | 部分覆盖 | 增加边界值和异常输入测试 | + +--- + +## 改进优先级排序 + +### P0 — 应立即修复 +1. OCR 阻塞事件循环（影响并发性能） +2. Pipeline 取消操作无效（用户体验 Bug） +3. SSRF 风险（安全问题） +4. asyncio.create_task 未保存引用（潜在任务丢失） + +### P1 — 本迭代完成 +5. HTTPException / ValidationError 统一为 ApiResponse 格式 +6. Schema 输入验证加强（Literal 枚举 + 长度限制 + 范围限制） +7. DOI 格式验证 +8. Pipeline 状态持久化到 Task 表 +9. 批量删除论文接口 +10. Pipeline 列表接口 + +### P2 — 下一迭代 +11. MCP 工具扩展（writing, keywords, pipelines） +12. 数据库复合索引 +13. Rate Limiting 分端点配置 +14. Subscription trigger 自动导入论文 +15. LLM 客户端入口统一 +16. Schema 导出完善 + +### P3 — 长期改进 +17. 项目导出/导入 +18. 健康检查路由统一 +19. CORS expose_headers / max_age +20. Conversation-Project 关联表 +21. 中间件专项测试 + +--- + +## 已解决问题 + +1. **WebSocket** — 增加 WebSocket 支持，与 SSE 共存。Pipeline 长时间运行和实时状态推送使用 WS。 +2. **API 版本管理** — 保持 v1，暂不规划 v2。 +3. **用户系统** — 保持单用户设计，不实施多用户。 + +## 实施范围 + +尽可能完成所有改进（P0 + P1 + P2 + P3），优先级从高到低依次实施。 diff --git a/docs/brainstorms/2026-03-18-backend-quality-testing-gaps-brainstorm.md b/docs/brainstorms/2026-03-18-backend-quality-testing-gaps-brainstorm.md new file mode 100644 index 0000000..1f6debc --- /dev/null +++ b/docs/brainstorms/2026-03-18-backend-quality-testing-gaps-brainstorm.md @@ -0,0 +1,75 @@ +--- +title: "后端质量与测试缺口修补" +date: 2026-03-18 +status: approved +tags: [backend, testing, code-quality, mcp, pipeline, chat] +--- + +# 后端质量与测试缺口修补 + +## 背景 + +在完成 21 项后端综合优化（安全、性能、一致性、新功能、架构）后，经深度审计发现仍存在以下缺口： +- 11 个 MCP 工具中 7 个无单元测试 +- Pipeline 集成测试缺少真实 PDF 全流程和 HITL 中断→恢复 +- 代码中存在硬编码常量、不一致的错误处理、路径注入风险 +- Chat 的 4 种 tool_mode 仅测试了 qa 模式 + +## 改进内容 + +### 1. 代码质量全面改进 + +**硬编码值提取到 config.py：** +- `upload.py`: `MAX_FILE_SIZE_MB = 50`, `TITLE_SIMILARITY_THRESHOLD = 0.85` +- `citation_graph_service.py`: `S2_API_BASE`, `S2_TIMEOUT = 15`, `S2_MAX_PER_REQUEST = 100` +- `rewrite.py`: `REWRITE_TIMEOUT = 30.0` +- `main.py`: 版本号 `"0.1.0"` + +**错误处理统一：** +- `citation_graph_service.py`: 返回 200 + `{"error": "..."}` → 改为 `HTTPException` +- `rewrite.py`: 过宽的 `Exception` 捕获 → 缩窄并区分可恢复/不可恢复异常 + +**输入验证与路径安全：** +- `UploadPipelineRequest.pdf_paths`: 添加路径遍历验证（禁止 `..`、绝对路径限制） +- `lookup_paper` ilike 查询安全性确认 + +**缺失测试补写：** +- `LLMConfigResolver` 单元测试 +- `RerankerService` 单元测试（mock SentenceTransformer） + +### 2. MCP 工具测试补全 + +为以下 7 个 MCP 工具添加单元测试： +- `search_knowledge_base` — RAG 搜索 +- `find_citations` — 引文图谱 +- `add_paper_by_doi` — DOI 导入 +- `search_papers_by_keyword` — 关键词搜索 +- `summarize_papers` — 论文摘要（WritingService） +- `generate_review_outline` — 综述大纲 +- `analyze_gaps` — 研究缺口分析 +- `manage_keywords` — 关键词管理（list/add/expand/delete） + +### 3. Pipeline 真实 PDF 集成测试 + +**测试数据：** 使用 `/data0/djx/omelette_pdf_test/` 中的论文 PDF（最小的 ~700KB） + +**覆盖流程：** +- Upload Pipeline API：PDF 上传 → 元数据提取 → 去重 → OCR → 索引 +- HITL 中断→恢复：模拟去重冲突 → 中断 → 解决冲突 → 恢复 +- Pipeline 列表与状态查询 + +### 4. Chat tool_mode 全模式测试 + +补充 3 种未测试的 tool_mode： +- `citation_lookup` — 引文检索模式 +- `review_outline` — 综述大纲模式 +- `gap_analysis` — 研究缺口分析模式 + +使用 mock LLM 验证各模式的 prompt 构造和输出格式。 + +## 关键决策 + +- **单用户系统**：不涉及多用户/权限改进 +- **真实 PDF 测试**：使用已有的测试 PDF，标记为需要外部数据的 marker +- **不涉及真实 LLM 调用**：本轮改进重点在结构性测试，真实 LLM 测试由现有 E2E 覆盖 +- **MCP 测试使用 mock**：避免依赖外部服务 diff --git a/docs/brainstorms/2026-03-18-gpu-cleanup-on-exit-brainstorm.md b/docs/brainstorms/2026-03-18-gpu-cleanup-on-exit-brainstorm.md new file mode 100644 index 0000000..8ab4098 --- /dev/null +++ b/docs/brainstorms/2026-03-18-gpu-cleanup-on-exit-brainstorm.md @@ -0,0 +1,85 @@ +--- +title: "程序退出时 GPU 资源自动释放" +date: 2026-03-18 +status: approved +tags: [backend, gpu, resource-management, reliability] +--- + +# 程序退出时 GPU 资源自动释放 + +## 背景 + +当前 Omelette 后端已实现 TTL 自动释放 GPU 模型（空闲 5 分钟后卸载），但在程序退出时存在资源未释放的场景。 + +### 当前 GPU 状态（实测） + +| 进程 | PID | GPU | 显存 | 说明 | +|------|-----|-----|------|------| +| Omelette | 2015162 | GPU 6 | 668 MiB | Embedding 模型（TTL 剩余 222s） | +| Omelette | 2015162 | GPU 7 | 2114 MiB | Reranker/CUDA context | +| MinerU | 2074601 | GPU 6 | 2872 MiB | 手动启动的外部进程 | + +### 现有清理机制 + +lifespan shutdown (`main.py`) 在正常退出时调用 `gpu_model_manager.stop()` 和 `mineru_process_manager.stop()`。 + +### 退出场景覆盖分析 + +| 场景 | 当前是否清理 | 原因 | +|------|-------------|------| +| Ctrl+C | ✅ | uvicorn 捕获 SIGINT → lifespan shutdown | +| kill PID (SIGTERM) | ✅ | uvicorn 捕获 SIGTERM → lifespan shutdown | +| kill -9 (SIGKILL) | ❌ | 内核强杀，无法捕获 | +| 程序崩溃/OOM | ❌ | event loop 已死 | +| 关闭终端 (SIGHUP) | ❌ | uvicorn 默认不处理 SIGHUP | +| 外部 MinerU | ❌ | `external` 状态不管 | + +## 我们要构建什么 + +**两层防护机制**，最大化覆盖所有退出场景： + +### 第一层：进程内安全网（方案 A） + +在 Omelette 进程内增加多层清理机制： + +1. **`atexit` 注册同步清理函数** — 程序正常退出或 Python 解释器退出时调用。 + 覆盖：正常退出、未捕获异常、部分崩溃场景。 + 不覆盖：SIGKILL、段错误。 + +2. **`signal.signal(SIGHUP)` 处理关闭终端** — 终端关闭时发送 SIGHUP。 + 覆盖：SSH 断开、关闭终端窗口。 + +3. **Lifespan shutdown 时杀外部 MinerU** — 通过查找端口对应 PID 并发送 SIGTERM。 + 当前只杀自己启动的子进程，改为也杀外部 MinerU。 + +### 第二层：外部看门狗（方案 B） + +独立脚本监控 Omelette 主进程，主进程退出后清理残留 GPU 资源： + +1. **PID 文件** — Omelette 启动时写 PID 到文件 +2. **看门狗脚本** — 持续监控主进程 PID： + - 主进程存活 → 无操作 + - 主进程消失 → 执行清理（kill MinerU、释放显存） +3. 覆盖 kill -9、OOM、段错误等所有场景 + +## 为什么选择这个方案 + +- 方案 A 简单直接，覆盖 90% 场景（Ctrl+C、SIGTERM、SIGHUP、崩溃） +- 方案 B 作为最后防线，覆盖方案 A 无法处理的 kill -9 和 OOM +- 两层组合实现接近 100% 的资源释放保障 + +## 关键决策 + +| 决策 | 选择 | 理由 | +|------|------|------| +| 清理范围 | 所有 GPU 资源 + 外部 MinerU | 关闭程序时应完全释放 | +| MinerU 外部进程处理 | 通过端口查 PID 后 kill | MinerU 无关闭 API | +| 看门狗实现 | 独立 Python 脚本 | 轻量、与主进程隔离 | +| PID 文件位置 | `{data_dir}/omelette.pid` | 与其他数据文件一致 | +| kill -9 处理 | 仅看门狗能覆盖 | 进程内无法捕获 SIGKILL | + +## 已解决问题 + +1. ~~TTL 是否正常工作~~ → 确认正常（Embedding 空闲 77s，TTL 剩余 222s） +2. ~~外部 MinerU 如何关闭~~ → 通过端口查找 PID + SIGTERM +3. ~~kill -9 如何处理~~ → 外部看门狗 diff --git a/docs/brainstorms/2026-03-18-gpu-resource-auto-management-brainstorm.md b/docs/brainstorms/2026-03-18-gpu-resource-auto-management-brainstorm.md new file mode 100644 index 0000000..fca962c --- /dev/null +++ b/docs/brainstorms/2026-03-18-gpu-resource-auto-management-brainstorm.md @@ -0,0 +1,120 @@ +--- +title: "GPU 资源自动管理 + MinerU 子进程自治" +date: 2026-03-18 +status: approved +tags: [backend, gpu, performance, mineru, resource-management] +--- + +# GPU 资源自动管理 + MinerU 子进程自治 + +## 背景 + +当前 Omelette 后端存在 GPU 资源浪费问题： + +1. **Embedding 模型**（Qwen3-Embedding-0.6B）：全局变量缓存，加载后永不释放，占用 ~500 MiB +2. **Reranker 模型**（Qwen3-Reranker-0.6B）：`@lru_cache(maxsize=1)` 缓存，永不清除，占用 ~500 MiB +3. **PaddleOCR**：实例级缓存，但 GPU 显存未被显式释放，占用 ~2-4 GB +4. **MinerU 服务器**：独立外部进程，需手动启动，持续占用 ~2.7 GB 显存 + +实际观察：GPU 6 占用 3.2 GB，GPU 7 占用 20.7 GB，即使没有 API 调用也不释放。 + +## 改进内容 + +### 1. GPU 模型 TTL 自动卸载 + +**策略**：模型在空闲 5 分钟（可配置 `model_ttl_seconds`）后自动卸载并释放 VRAM。 + +**覆盖模型**： +- **Embedding 模型**：`embedding_service.py` 全局 `_cached_embed_model` +- **Reranker 模型**：`reranker_service.py` `@lru_cache` 缓存 +- **PaddleOCR**：`ocr_service.py` 实例级 `_paddle_ocr` + +**机制**： +- 统一的 `GPUModelManager` 管理所有模型的生命周期 +- 每个模型维护 `last_used_at` 时间戳 +- 后台 `asyncio.Task` 定期检查（每 30 秒），卸载超时模型 +- 卸载时调用 `del model` + `gc.collect()` + `torch.cuda.empty_cache()` +- API 调用时自动按需加载，使用后更新时间戳 + +**配置**： +- `model_ttl_seconds: int = 300`（默认 5 分钟） +- `model_ttl_check_interval: int = 30`（检查间隔 30 秒） + +### 2. MinerU 子进程自动管理 + +**策略**：Omelette 在需要 MinerU 时自动启动子进程，空闲 10 分钟后自动停止。 + +**实现**： +- 新增 `MinerUProcessManager` 管理 MinerU 子进程生命周期 +- 启动命令：`conda run -n mineru python -m mineru.cli.fast_api --host 0.0.0.0 --port ` +- 环境变量继承：`CUDA_VISIBLE_DEVICES` 从 omelette 配置获取 +- 健康检查：启动后轮询 `/docs` 端点直到可用 +- 空闲检测：与 GPU 模型 TTL 类似，使用后更新 `last_used_at` +- 停止方式：`process.terminate()` + `process.wait(timeout=10)` + `process.kill()` + +**配置**： +- `mineru_auto_manage: bool = True`（默认启用自动管理） +- `mineru_conda_env: str = "mineru"`（conda 环境名） +- `mineru_ttl_seconds: int = 600`（默认 10 分钟空闲后停止） +- `mineru_startup_timeout: int = 120`（启动超时 120 秒） +- `mineru_gpu_ids: str = ""`（MinerU 使用的 GPU，空则继承 `cuda_visible_devices`） + +**容错**： +- 启动失败（conda 不存在 / mineru 环境缺失 / 端口冲突）→ 自动 fallback 到 pdfplumber +- 运行中崩溃 → 下次需要时重新启动 +- 日志中记录 warning 提示用户 + +**兼容性**： +- `mineru_auto_manage = False` 时保持现有行为（用户手动管理） +- 已运行的外部 MinerU 实例不受影响（先检查端口是否已在用） + +### 3. GPU 监控 API + +**端点**：`GET /api/v1/gpu/status` + +**返回数据**： +```json +{ + "models": [ + { + "name": "embedding", + "model_name": "Qwen/Qwen3-Embedding-0.6B", + "loaded": true, + "device": "cuda:0", + "last_used_at": "2026-03-18T12:00:00", + "ttl_remaining_seconds": 180 + } + ], + "mineru": { + "status": "running", + "pid": 12345, + "port": 8010, + "last_used_at": "2026-03-18T12:00:00", + "ttl_remaining_seconds": 420 + }, + "gpu_memory": [ + {"gpu_id": 6, "total_mb": 24576, "used_mb": 1024, "free_mb": 23552}, + {"gpu_id": 7, "total_mb": 24576, "used_mb": 500, "free_mb": 24076} + ] +} +``` + +**附加端点**： +- `POST /api/v1/gpu/unload` — 立即卸载所有模型并释放显存 + +## 关键决策 + +- **TTL 时长**：GPU 模型 5 分钟，MinerU 10 分钟（MinerU 启动较慢） +- **MinerU 管理**：子进程方式，使用独立的 `mineru` conda 环境 +- **单用户系统**：不考虑并发用户竞争 GPU 资源 +- **PaddleOCR 处理**：当前已是按实例创建；改进点是在实例销毁时显式调用 `torch.cuda.empty_cache()` 确保显存释放 +- **向后兼容**：所有新功能默认启用，但可通过配置禁用 + +## 已解决的问题 + +- Q: 模型卸载后重新加载要多久？ + A: Embedding ~3-5s，Reranker ~3-5s，PaddleOCR ~5-8s，MinerU ~30-60s。TTL 策略正是为了平衡这个延迟。 +- Q: 多个请求同时需要同一模型怎么办？ + A: 使用 `asyncio.Lock` 确保同一模型只加载一次，后续请求等待加载完成。 +- Q: MinerU 子进程的 GPU 内存如何隔离？ + A: 通过 `CUDA_VISIBLE_DEVICES` 环境变量控制，可配置 `mineru_gpu_ids` 指定专用 GPU。 diff --git a/docs/brainstorms/2026-03-18-gpu-scheduling-modes-brainstorm.md b/docs/brainstorms/2026-03-18-gpu-scheduling-modes-brainstorm.md new file mode 100644 index 0000000..9a8a6fd --- /dev/null +++ b/docs/brainstorms/2026-03-18-gpu-scheduling-modes-brainstorm.md @@ -0,0 +1,129 @@ +# GPU 资源调度与模式化配置 + +**日期**: 2026-03-18 +**状态**: completed + +## 背景 + +服务器有 8 张 GPU（0-7）。当前部署： + +| GPU | 用途 | 管理方式 | +|-----|------|----------| +| 5 | MinerU PDF 解析服务（独立 conda 环境） | 单独启动，常驻 | +| 6, 7 | Omelette 后端（OCR、Embedding、Reranker） | `CUDA_VISIBLE_DEVICES=6,7` | +| 0-4 | 不使用 / 其他用途 | — | + +**问题**：当前 batch_size 和并发数要么硬编码，要么默认值较高，在双卡环境下容易 OOM。需要一套灵活的配置体系，既有"一键设置"的预设模式，又允许用户按需覆盖。 + +## What We're Building + +一套 **GPU 资源调度配置系统**，包含： + +1. **GPU_MODE 全局预设** — `conservative / balanced / aggressive` 三档，一键控制所有 GPU 相关参数的默认行为 +2. **按服务细粒度覆盖** — 用户可通过 `.env` 单独设置任意服务的 batch_size、并发数、GPU 绑定 +3. **智能 GPU 选择 + 手动 Pin** — 默认选显存最空闲的 GPU，但允许用户将 Embedding/Reranker/OCR 固定到指定 GPU + +## Why This Approach + +- **不用重启即知道参数是否合理** — 三种预设覆盖了"调试/日常/压满"三个典型场景 +- **YAGNI** — 不做运行时 API 切换（改 .env 重启即可），不做自动 OOM 降档（复杂度高收益低） +- **向后兼容** — 不设 GPU_MODE 时行为等同 `balanced`，与现有默认值一致 + +## Key Decisions + +### 1. 三档预设模式 + +| 参数 | `conservative` | `balanced`（默认） | `aggressive` | +|------|---------------|-------------------|-------------| +| `OCR_PARALLEL_LIMIT` | 1 | auto (GPU 数) | GPU 数 × 2 | +| `EMBED_BATCH_SIZE` | 1 | 8 | 32 | +| `RERANK_BATCH_SIZE` | 1 | 16 | 50 | +| `RERANKER_CONCURRENCY_LIMIT` | 1 | 1 | 2 | + +- `conservative`：调试场景 / 小显存（8-16G），绝不 OOM +- `balanced`：日常使用，兼顾性能和稳定 +- `aggressive`：大显存（32G+）、大批量处理，追求最大吞吐 + +### 2. 新增配置项 + +| 配置项 | 类型 | 默认 | 说明 | +|--------|------|------|------| +| `GPU_MODE` | str | `balanced` | 全局预设：conservative / balanced / aggressive | +| `EMBED_BATCH_SIZE` | int | 0（跟随模式） | Embedding 推理 batch size，0=跟随 GPU_MODE | +| `RERANK_BATCH_SIZE` | int | 0（跟随模式） | Reranker 内部 top_n，0=跟随 GPU_MODE | +| `EMBED_GPU_ID` | int | -1（自动） | Embedding 模型固定到哪张 GPU，-1=自动选择 | +| `RERANK_GPU_ID` | int | -1（自动） | Reranker 模型固定到哪张 GPU，-1=自动选择 | +| `OCR_GPU_IDS` | str | ""（自动） | OCR 使用的 GPU 列表（逗号分隔），空=全部轮转 | + +### 3. 优先级规则 + +``` +用户显式设置 > GPU_MODE 预设 > 硬编码默认值 +``` + +示例：`GPU_MODE=conservative` + `EMBED_BATCH_SIZE=16` → Embedding 用 16，其余跟 conservative。 + +### 4. GPU 选择策略 + +- **Embedding / Reranker**：默认 `_pick_best_gpu()`（选显存最空闲的），可通过 `EMBED_GPU_ID` / `RERANK_GPU_ID` 固定 +- **OCR**：默认轮转所有可见 GPU，可通过 `OCR_GPU_IDS` 限制范围（如只用 GPU 7） +- 固定 GPU 时跳过 `_pick_best_gpu()` 直接用 `cuda:N` + +### 5. 不做的事 + +- **不做运行时 API 切换**：改 `.env` 重启即可，避免热切换带来的状态管理复杂度 +- **不做自动降档**：OOM 时已有 retry + 换 GPU 机制，不再叠加自动模式切换 +- **不在单 paper 内部加并发**：OCR 已经是 per-page 串行处理，PaddleOCR 内部有自己的并行优化 + +## Resolved Questions + +**Q: "多 paper 多 batchsize" vs "单 paper 多 batchsize"？** +A: "多 paper" 由 `OCR_PARALLEL_LIMIT` 控制（同时处理几篇论文），"多 batchsize" 由各服务的 `*_BATCH_SIZE` 控制（每次推理处理多少数据）。两者正交，不需要额外模式。 + +**Q: MinerU 需要纳入调度吗？** +A: 不需要。MinerU 是独立 conda 环境启动的外部服务，通过 HTTP API 调用。它的 GPU 分配由启动命令的 `CUDA_VISIBLE_DEVICES` 控制，与后端隔离。 + +**Q: 运行时切换模式的需求？** +A: 当前阶段通过 `.env` 重启即可。未来如有需要，可以通过 Settings API 暴露，但不是现在的优先级。 + +## Architectural Notes + +### 配置解析流程 + +``` +启动 → 读 .env → 解析 GPU_MODE → + 对每个参数: + if 用户显式设置了 → 用用户值 + elif GPU_MODE 有预设 → 用预设值 + else → 用硬编码默认值 +``` + +### 实现要点 + +1. `config.py` 中定义 `GpuMode` 枚举和 `_resolve_gpu_param()` 辅助函数 +2. 各 batch_size 字段默认 0，表示"跟随 GPU_MODE" +3. `embedding_service.py` 和 `reranker_service.py` 读取解析后的值，不再硬编码 +4. `.env.example` 中用注释说明预设对应的参数值 +5. `_pick_best_gpu()` 在 `*_GPU_ID >= 0` 时短路返回 `cuda:N` + +### 典型使用场景 + +```bash +# 场景 1: 调试，双卡但想保守 +GPU_MODE=conservative +CUDA_VISIBLE_DEVICES=6,7 + +# 场景 2: 日常（什么都不设，默认 balanced） +CUDA_VISIBLE_DEVICES=6,7 + +# 场景 3: 大批量处理，压满显卡 +GPU_MODE=aggressive +CUDA_VISIBLE_DEVICES=6,7 +OCR_PARALLEL_LIMIT=4 + +# 场景 4: 精细控制 +GPU_MODE=balanced +EMBED_GPU_ID=0 # 固定 embedding 到 cuda:0 (物理 GPU 6) +RERANK_GPU_ID=1 # 固定 reranker 到 cuda:1 (物理 GPU 7) +EMBED_BATCH_SIZE=16 # 覆盖 balanced 默认的 8 +``` diff --git a/docs/plans/2026-03-17-refactor-backend-comprehensive-optimization-plan.md b/docs/plans/2026-03-17-refactor-backend-comprehensive-optimization-plan.md new file mode 100644 index 0000000..b3f6470 --- /dev/null +++ b/docs/plans/2026-03-17-refactor-backend-comprehensive-optimization-plan.md @@ -0,0 +1,528 @@ +--- +title: "refactor: Backend Comprehensive Optimization" +type: refactor +status: active +date: 2026-03-17 +origin: docs/brainstorms/2026-03-17-backend-comprehensive-review-brainstorm.md +--- + +# refactor: Backend Comprehensive Optimization + +## Enhancement Summary + +**Deepened on:** 2026-03-17 +**Sections enhanced:** 6 +**Research agents used:** kieran-python-reviewer, architecture-strategist, learnings-researcher, repo-research-analyst, Context7 (FastAPI docs) + +### Key Improvements +1. Phase 1: 扩大 `asyncio.to_thread` 覆盖范围——补充 `get_stats`、`delete_paper`、`delete_index` 三处遗漏的 ChromaDB 同步调用 +2. Phase 2: 明确使用显式导入（非 `import *`），并修正 CHAT_QA / RAG_ANSWER 提示词合并策略 +3. Phase 3: `get_or_404` 改进——使用 `TypeVar` 泛型、`resource_id` 参数命名、`getattr` 替代 `hasattr`；并行 LLM 调用需加 semaphore 限流 +4. Phase 3: conversation 列表 `knowledge_base_id` 过滤改用 SQLite `json_each` 子查询确保 `total` 准确 + +### New Considerations Discovered +- `delete_paper`、`delete_index`、`get_stats` 中的 ChromaDB 同步调用也需要包裹 `asyncio.to_thread`（原计划遗漏） +- `update_conversation` 和 `delete_conversation` 无需 `flush`，直接移除 `commit` 即可 +- RAGService 的 count 缓存在 per-request 实例下只在单次请求内有效（仍有价值，因单次 query 调用 count 两次） +- 并行化 `summarize_papers` 时需考虑 LLM provider 速率限制，建议添加可配置的 semaphore + +--- + +## Overview + +对 Omelette 后端进行全面优化，涵盖四个阶段：修复异步阻塞和会话问题 → 提示词集中管理 → 架构级改进 → 代码清理。每个阶段独立可交付，可逐步推进。 + +## Problem Statement / Motivation + +通过全面复盘（see brainstorm: `docs/brainstorms/2026-03-17-backend-comprehensive-review-brainstorm.md`）发现以下核心问题： + +1. **运行时稳定性**：4 处同步阻塞调用在 async 代码中直接执行，阻塞事件循环 +2. **数据一致性**：`conversations.py` 和 `persist_node` 中的手动 commit 与 `get_session()` 自动 commit 冲突 +3. **可维护性**：16 个 LLM 提示词散落在 10+ 个文件中，存在重复和语言混用 +4. **代码卫生**：API 层包含业务逻辑、缺少分页、硬编码配置、异常吞没 + +## Proposed Solution + +分 4 个 Phase 依次修复，每个 Phase 完成后可独立提交和部署。 + +## Technical Approach + +### Phase 1: 关键修复（运行时稳定性） + +**目标**: 消除阻塞调用、修复双重提交、修复异常吞没 + +#### 1.1 修复异步中的同步阻塞 + +**文件**: `backend/app/services/subscription_service.py` +```python +# Before +feed = feedparser.parse(resp.text) + +# After +feed = await asyncio.to_thread(feedparser.parse, resp.text) +``` + +**文件**: `backend/app/services/pdf_metadata.py` +```python +# Before (in async extract_metadata) +result = _extract_local(pdf_path) + +# After +result = await asyncio.to_thread(_extract_local, pdf_path) +``` + +**文件**: `backend/app/services/rag_service.py` +- 将 `collection.count()` 调用包裹 `asyncio.to_thread()` +- 在 `query()` 和 `retrieve_only()` 中各出现两次（lines 220, 230, 299, 305） +- 考虑添加简单的实例级缓存（TTL=60s），避免每次 query 都访问两次 count +- **补充**: `get_stats()` (line 391)、`delete_paper()` (line 380)、`delete_index()` (line 371) 中的 ChromaDB 同步调用也需要包裹 + +```python +# rag_service.py — 添加 count 缓存 +import time + +class RAGService: + def __init__(self, ...): + ... + self._count_cache: dict[int, tuple[int, float]] = {} + + async def _get_count(self, project_id: int) -> int: + now = time.monotonic() + cached = self._count_cache.get(project_id) + if cached and now - cached[1] < 60.0: + return cached[0] + collection = self._get_collection(project_id) + count = await asyncio.to_thread(collection.count) + self._count_cache[project_id] = (count, now) + return count + + def _invalidate_count(self, project_id: int) -> None: + self._count_cache.pop(project_id, None) +``` + +在 `index_chunks`、`delete_index`、`delete_paper` 后调用 `_invalidate_count`。 + +> **Research Insight**: RAGService 是 per-request 实例，缓存在单次请求内生效。虽然 60s TTL 对 per-request 实例无意义，但缓存仍有价值——单次 `query()` 会调用 `count()` 两次。`get_stats()` 也应使用 `_get_count()` 而非直接调用 `collection.count()`。 + +#### 1.2 修复会话双重提交 + +**文件**: `backend/app/api/v1/conversations.py` + +| 端点 | 当前 | 修复 | +|------|------|------| +| `create_conversation` (line 115) | `await db.commit()` | 改为 `await db.flush()` — 需要 flush 使 `conv.id` 可用 | +| `update_conversation` (line 154) | `await db.commit()` | 直接移除 — 无需 flush，`get_session` 自动 commit | +| `delete_conversation` (line 175) | `await db.commit()` | 直接移除 — 无需 flush，`get_session` 自动 commit | + +```python +# conversations.py — create_conversation +db.add(conv) +await db.flush() # ID available for the follow-up query +# await db.commit() ← REMOVE +``` + +> **Research Insight** (FastAPI docs): FastAPI 官方推荐 `yield` 依赖管理 session 生命周期。`get_session` 的 try/yield/commit/except/rollback 模式完全符合最佳实践。手动 commit 只在需要提前获取 ID 时用 `flush` 替代。 + +**文件**: `backend/app/pipelines/chat/nodes.py` +- `persist_node` (line 430)：移除 `await db.commit()`，保留现有的 `await db.flush()` (line 413) +- 注意：persist_node 使用的 db session 来自 `Depends(get_db)` → `get_session()`，所以自动 commit 生效 +- **边界情况**：如果 persist_node 在 flush 后抛异常，`get_session` 会自动 rollback，这是正确行为 + +#### 1.3 修复异常吞没 + +| 文件 | 行号 | 修复 | +|------|------|------| +| `rag_service.py` | 206-207 | `except Exception: pass` → `except Exception: logger.debug("Adjacent chunk fetch failed", exc_info=True)` | +| `rag_service.py` | 393-395 | 添加 `logger.warning("Failed to get stats for project %d", project_id, exc_info=True)` | +| `completion_service.py` | 69-71 | 已有 `logger.warning`，保留 | +| `main.py` | 74-75 | MCP mount 失败：`logger.warning` → `logger.error` | + +### Phase 2: 提示词集中管理 + +**目标**: 建立 `app/prompts/` 模块，统一语言为英文，消除重复 + +#### 2.1 创建 `app/prompts/` 目录结构 + +``` +backend/app/prompts/ +├── __init__.py # 统一导出所有 prompt 常量 +├── chat.py # Chat 管道 system prompts (5 个) +├── writing.py # 写作助手 prompts (4 个 system + user 模板) +├── rag.py # RAG 知识库 prompt (1 个) +├── dedup.py # 去重 prompts (2 个: verify + auto_resolve) +├── keyword.py # 关键词扩展 prompt (1 个) +├── completion.py # 写作补全 prompt (1 个) +└── rewrite.py # 文本改写 prompts (4 个) +``` + +#### 2.2 各文件内容 + +**`app/prompts/chat.py`**: +```python +CHAT_QA_SYSTEM = ( + "You are a scientific research assistant. Answer the question based on the provided context. " + "Use inline citations like [1], [2] to reference source papers. " + "If the context doesn't contain enough information, say so honestly. " + "Structure your answer with clear paragraphs. " + "Respond in the same language as the user's question." +) + +CHAT_CITATION_SYSTEM = ( + "You are a citation finder. Given the user's text, identify and list the most relevant " + "references from the provided context. Format as a numbered list with paper titles, authors, " + "and brief explanations of relevance. Include DOI when available. " + "Keep your own commentary minimal." +) + +CHAT_OUTLINE_SYSTEM = ( + "You are a literature review expert. Based on the provided context, generate a structured " + "review outline with sections, subsections, and key points. Use markdown headers for sections. " + "Use citations like [1], [2] to reference sources. Suggest a logical flow and highlight key themes." +) + +CHAT_GAP_SYSTEM = ( + "You are a research gap analyst. Based on the provided literature context, identify " + "research gaps, unexplored areas, and potential future directions. Cite existing work " + "using [1], [2] format. Organize by theme, not by individual papers. " + "Be specific about what has been studied and what remains open." +) + +CHAT_FALLBACK_SYSTEM = ( + "You are a scientific research assistant specializing in academic literature analysis. " + "Answer questions clearly and accurately based on your knowledge. " + "When the user's question is outside your expertise or you are uncertain, say so honestly. " + "Respond in the same language as the user's question." +) + +EXCERPT_CLEAN_SYSTEM = ( + "Clean up the following text extracted from an academic PDF. " + "Fix OCR errors, add missing spaces between words, restore formatting. " + "Keep the original meaning intact. Output only the cleaned text, nothing else." +) +``` + +**`app/prompts/writing.py`**: +```python +WRITING_SECTION_SYSTEM = ( + "You are an academic review writing expert. Write a review paragraph for the given section. " + "Requirements: " + "1. Use academic language with clear logic. " + "2. Use [1][2] format for citations at appropriate positions. " + "3. Every citation must correspond to a provided reference — do not fabricate. " + "4. Paragraph length: 200-400 words." +) + +WRITING_SUMMARIZE_SYSTEM = ( + "You are a scientific paper analyst. Provide structured, accurate summaries. " + "Focus on empirical findings and methodology. " + "Do not hallucinate information not present in the provided metadata." +) + +WRITING_OUTLINE_SYSTEM = ( + "You are a scientific writing expert. Generate well-structured review outlines " + "organized by research themes with clear section hierarchy." +) + +WRITING_GAP_SYSTEM = ( + "You are a research gap analyst. Identify unexplored areas and innovation opportunities " + "based on the provided literature." +) +``` + +**`app/prompts/rag.py`**: +```python +RAG_ANSWER_SYSTEM = ( + "You are a scientific research assistant. " + "Answer questions based strictly on the provided context. " + "Cite sources accurately using the format provided. " + "Respond in the same language as the user's question." +) +``` + +**`app/prompts/dedup.py`**: +```python +DEDUP_VERIFY_SYSTEM = ( + "You are a scientific literature deduplication expert. " + "Compare papers carefully based on title, authors, DOI, and journal. " + "Return valid JSON only." +) + +DEDUP_RESOLVE_SYSTEM = ( + "You are a scientific literature deduplication expert. " + "Determine the best resolution for duplicate candidates. " + "Return valid JSON only." +) +``` + +**`app/prompts/keyword.py`**: +```python +KEYWORD_EXPAND_SYSTEM = ( + "You are a scientific terminology expert. " + "Generate related terms including synonyms, abbreviations, technical variants, " + "and cross-disciplinary application terms. " + "Return valid JSON only." +) +``` + +**`app/prompts/completion.py`**: +```python +COMPLETION_SYSTEM = ( + "You are a scientific writing assistant. Predict and complete the user's text. " + "Return only the completion (do not repeat the user's input), max 50 characters. " + "If you cannot reasonably predict, return an empty string. " + "Return plain text only — no quotes, explanations, or formatting." +) +``` + +**`app/prompts/rewrite.py`**: +```python +REWRITE_SIMPLIFY = ( + "Rewrite the following academic text in plain, accessible language. " + "Keep the core meaning and key concepts intact, but make it understandable " + "to a general audience. Output only the rewritten text, no explanations." +) + +REWRITE_ACADEMIC = ( + "Rewrite the following text in formal academic style. " + "Use precise terminology, passive voice where appropriate, and proper " + "academic conventions. Maintain the original meaning. Output only the rewritten text." +) + +REWRITE_TRANSLATE_EN = ( + "Translate the following text into English. " + "Preserve academic terminology and the original meaning. " + "Output only the translation, no explanations." +) + +REWRITE_TRANSLATE_ZH = ( + "Translate the following text into Chinese. " + "Preserve academic terminology and the original meaning. " + "Output only the translation, no explanations." +) +``` + +#### 2.3 迁移步骤 + +1. 创建 `app/prompts/` 目录和所有文件 +2. 逐个文件替换（使用显式导入，**不要用 `import *`**）： + - `pipelines/chat/nodes.py`: 导入 `from app.prompts.chat import CHAT_QA_SYSTEM, CHAT_CITATION_SYSTEM, ...`，删除本地 `TOOL_MODE_PROMPTS` 和 `EXCERPT_CLEAN_PROMPT` + - `services/writing_service.py`: 导入 writing prompts，删除 `SECTION_SYSTEM_PROMPT` 和内联 system prompt + - `services/rag_service.py`: 导入 `RAG_ANSWER_SYSTEM` + - `services/dedup_service.py`: 导入 `DEDUP_VERIFY_SYSTEM` + - `services/keyword_service.py`: 导入 `KEYWORD_EXPAND_SYSTEM` + - `services/completion_service.py`: 导入 `COMPLETION_SYSTEM` + - `api/v1/rewrite.py`: 导入 rewrite prompts +3. 将 `api/v1/dedup.py` 中的 `auto_resolve_conflict` LLM 逻辑移到 `DedupService.auto_resolve()` +4. 将 `api/v1/keywords.py` 中的 `expand_keywords` LLM 逻辑移到 `KeywordService.expand_keywords_with_llm()`（已有此方法，统一调用即可） + +### Phase 3: 架构级改进 + +#### 3.1 添加分页 + +**文件**: `backend/app/api/v1/keywords.py` +```python +@router.get("", response_model=ApiResponse[PaginatedData[KeywordRead]]) +async def list_keywords( + project_id: int, + page: int = 1, + page_size: int = 50, + db: AsyncSession = Depends(get_db), +): + stmt = select(Keyword).where(Keyword.project_id == project_id) + count = (await db.execute(select(func.count()).select_from(stmt.subquery()))).scalar_one() + items = (await db.execute( + stmt.order_by(Keyword.level, Keyword.id) + .offset((page - 1) * page_size).limit(page_size) + )).scalars().all() + + return ApiResponse(data=PaginatedData( + items=[KeywordRead.model_validate(k) for k in items], + total=count, + page=page, + page_size=page_size, + total_pages=(count + page_size - 1) // page_size or 1, + )) +``` + +同理应用到 `subscription.py` 和 `tasks.py`。 + +#### 3.2 并行化写作服务 LLM 调用 + +**文件**: `backend/app/services/writing_service.py` +```python +_summarize_semaphore = asyncio.Semaphore(5) + +async def summarize_papers(self, paper_ids: list[int], language: str = "en") -> list[dict]: + # ... load papers ... + + async def _summarize_one(paper: Paper) -> dict: + async with _summarize_semaphore: + prompt = f"Summarize this scientific paper in {language}: ..." + summary = await self.llm.chat(messages=[...], temperature=0.3, task_type="summarize") + return {"paper_id": paper.id, "title": paper.title, "summary": summary} + + tasks = [_summarize_one(papers[pid]) for pid in paper_ids if pid in papers] + results = await asyncio.gather(*tasks, return_exceptions=True) + return [r for r in results if not isinstance(r, Exception)] +``` + +> **Research Insight**: 添加 semaphore 防止并行 LLM 调用过多导致 provider 速率限制。使用 `return_exceptions=True` 实现部分成功——单篇摘要失败不影响其他。 + +#### 3.3 抽取通用 404 依赖 + +**文件**: `backend/app/api/deps.py` — 添加： +```python +from typing import TypeVar +from app.database import Base + +T = TypeVar("T", bound=Base) + +async def get_or_404( + db: AsyncSession, + model: type[T], + resource_id: int, + *, + project_id: int | None = None, + detail: str = "Resource not found", +) -> T: + obj = await db.get(model, resource_id) + if not obj: + raise HTTPException(status_code=404, detail=detail) + obj_project_id = getattr(obj, "project_id", None) + if project_id is not None and obj_project_id is not None and obj_project_id != project_id: + raise HTTPException(status_code=404, detail=detail) + return obj +``` + +> **Research Insight**: 使用 `TypeVar` 保持返回类型信息，`resource_id` 避免遮蔽内置 `id`，`getattr` 替代 `hasattr` 更安全。 + +#### 3.4 修复 conversation 列表的内存过滤 + +**文件**: `backend/app/api/v1/conversations.py` + +将 `knowledge_base_id` 过滤移到 SQL 层。使用 SQLite `json_each` (需要 SQLite 3.38+): + +```python +from sqlalchemy import text + +if knowledge_base_id is not None: + kb_filter = text( + "EXISTS (SELECT 1 FROM json_each(conversations.knowledge_base_ids) WHERE value = :kb_id)" + ) + stmt = stmt.where(kb_filter).params(kb_id=knowledge_base_id) + count_stmt = select(func.count()).select_from(stmt.subquery()) + total = (await db.execute(count_stmt)).scalar() or 0 +``` + +> **Research Insight**: 不要使用 `contains(f"[{kb_id}]")`，因为 `"[1]"` 会匹配 `"[12]"` 或 `"[21]"`。`json_each` 是精确匹配的正确方案。同时修复 `total` 计算，确保分页准确。 + +#### 3.5 健康检查免认证 + +**文件**: `backend/app/middleware/auth.py` +```python +EXEMPT_PATHS = {"/health", "/api/v1/settings/health"} +``` + +### Phase 4: 代码清理 + +#### 4.1 硬编码配置移入 config + +**文件**: `backend/app/config.py` — 添加（使用 `Field` 进行验证）： +```python +from pydantic import Field + +max_upload_size_mb: int = Field(default=50, ge=1, le=500) +rate_limit: str = Field(default="120/minute", description="API rate limit") +clean_semaphore_limit: int = Field(default=3, ge=1) +rewrite_semaphore_limit: int = Field(default=3, ge=1) +llm_parallel_limit: int = Field(default=5, ge=1, description="Max parallel LLM calls for batch operations") +``` + +#### 4.2 清理未使用依赖 + +**文件**: `backend/pyproject.toml` — 移除 `aiohttp>=3.11.0` + +#### 4.3 统一 MCP 挂载错误处理 + +**文件**: `backend/app/main.py` +```python +try: + from app.mcp_server import mcp + app.mount("/mcp", mcp.streamable_http_app()) +except Exception: + logger.error("Failed to mount MCP server", exc_info=True) +``` + +## System-Wide Impact + +- **API 兼容性**: Phase 1-2 不改变任何 API 接口，Phase 3 的分页为向后兼容的新增参数 +- **Error propagation**: 修复异常吞没后，某些之前静默失败的场景会开始记录日志，但不会改变 API 响应 +- **State lifecycle**: 修复双重 commit 后，事务边界更清晰，不会有意外的提前提交 +- **Pipeline behavior**: Chat pipeline 的 persist_node 行为不变（flush 保证 ID 可用） + +## Acceptance Criteria + +### Phase 1 +- [ ] `subscription_service.py`: `feedparser.parse` 包裹 `asyncio.to_thread` +- [ ] `pdf_metadata.py`: `_extract_local` 调用包裹 `asyncio.to_thread` +- [ ] `rag_service.py`: `collection.count()` 包裹 `asyncio.to_thread` + 添加 `_get_count` 缓存 +- [ ] `rag_service.py`: `delete_paper`、`delete_index`、`get_stats` 中的 ChromaDB 同步调用也包裹 `asyncio.to_thread` +- [ ] `rag_service.py`: `get_stats` 使用 `_get_count` 复用缓存 +- [ ] `conversations.py`: `create` 改 `commit` 为 `flush`；`update`/`delete` 直接移除 `commit` +- [ ] `pipelines/chat/nodes.py`: `persist_node` 移除手动 `db.commit()` +- [ ] `rag_service.py`: `except Exception: pass` 添加日志 +- [ ] `main.py`: MCP 挂载失败改为 `logger.error` +- [ ] 所有现有测试通过 + +### Phase 2 +- [ ] 创建 `app/prompts/` 目录，包含 8 个文件 +- [ ] 所有提示词统一为英文 +- [ ] 消除所有重复的提示词定义 +- [ ] `dedup.py` 的 `auto_resolve_conflict` 逻辑移入 `DedupService` +- [ ] `keywords.py` 的 `expand_keywords` 统一使用 `KeywordService` +- [ ] 所有现有测试通过 + +### Phase 3 +- [ ] `keywords.py`: 添加分页（`page`, `page_size` 参数） +- [ ] `subscription.py`: 添加分页 +- [ ] `tasks.py`: 添加分页 +- [ ] `writing_service.py`: `summarize_papers` 并行化（含 semaphore 限流 + `return_exceptions=True`） +- [ ] `deps.py`: 添加 `get_or_404` 通用依赖（TypeVar 泛型 + `resource_id` 命名） +- [ ] `conversations.py`: `knowledge_base_id` 过滤改用 `json_each` 子查询，修复 `total` 计算 +- [ ] `auth.py`: 健康检查端点免认证 + +### Phase 4 +- [ ] 硬编码配置值移入 `config.py` +- [ ] `pyproject.toml`: 移除 `aiohttp` +- [ ] `main.py`: MCP 挂载错误处理统一 + +## Success Metrics + +- 所有 178+ 现有测试保持通过 +- 无 ruff lint 错误 +- 提示词文件数从 10+ 减少到 `app/prompts/` 下的 8 个 +- 无重复的 LLM system prompt 定义 +- 零同步阻塞调用在 async 代码路径中 + +## Dependencies & Risks + +| 风险 | 缓解措施 | +|------|---------| +| 双重 commit 修复可能影响事务行为 | 先跑完整测试，重点关注 conversation CRUD 和 chat pipeline | +| 提示词措辞变化可能影响 LLM 输出质量 | 改动尽量保守，主要是语言统一和微调，不做大幅重写 | +| 分页添加需要前端配合 | 新增参数有默认值，不影响现有前端调用 | +| `asyncio.to_thread` 增加线程池压力 | `feedparser` 和 `fitz` 调用频率低，不会成为瓶颈 | + +## Sources & References + +### Origin + +- **Brainstorm document**: [docs/brainstorms/2026-03-17-backend-comprehensive-review-brainstorm.md](docs/brainstorms/2026-03-17-backend-comprehensive-review-brainstorm.md) — Key decisions: prompts unified to English, centralized to `app/prompts/`, no user customization needed, API layer should not call LLM directly + +### Internal References + +- **Async pattern**: `docs/solutions/performance-issues/blocking-sync-calls-asyncio-to-thread.md` +- **Chat rewrite**: `docs/solutions/integration-issues/2026-03-12-chat-routing-chain-langgraph-aisdk-rewrite.md` +- **Quality audit**: `docs/solutions/compound-issues/codebase-quality-audit-4-batch-remediation.md` +- **Pagination pattern**: `backend/app/api/v1/projects.py:18-19`, `backend/app/schemas/common.py:17-29` +- **Session lifecycle**: `backend/app/database.py:45-51` diff --git a/docs/plans/2026-03-17-refactor-config-rag-api-testing-SECURITY-AUDIT.md b/docs/plans/2026-03-17-refactor-config-rag-api-testing-SECURITY-AUDIT.md new file mode 100644 index 0000000..d199a70 --- /dev/null +++ b/docs/plans/2026-03-17-refactor-config-rag-api-testing-SECURITY-AUDIT.md @@ -0,0 +1,294 @@ +# Security Audit: Config + RAG + API Testing Plan + +**Plan**: [2026-03-17-refactor-config-rag-api-testing-plan.md](./2026-03-17-refactor-config-rag-api-testing-plan.md) +**Auditor**: security-sentinel +**Date**: 2026-03-17 + +--- + +## Executive Summary + +| Severity | Count | Status | +|----------|-------|--------| +| Critical | 2 | Must fix before implementation | +| High | 3 | Fix in Phase 2/3 | +| Medium | 4 | Recommended | +| Low | 2 | Nice to have | + +**Overall**: The plan is implementable but requires specific security hardening before and during rollout. Critical issues center on resource exhaustion and test data isolation. + +--- + +## 1. Input Validation on ChatStreamRequest + +### 1.1 `rag_top_k` (Plan: `ge=1, le=50`) + +**Status**: ✅ Plan specifies correct bounds. + +**Recommendation**: Ensure the schema uses `Field(ge=1, le=50)` exactly. Current `ChatStreamRequest` does not yet have this field—add it per plan. + +**Gap**: The **RAG API** (`RAGQueryRequest` in `backend/app/api/v1/rag.py`) has `top_k: int = 10` with **no validation**. An attacker can send `top_k=999999`. + +**Action**: Add validation to `RAGQueryRequest`: + +```python +# backend/app/api/v1/rag.py +class RAGQueryRequest(BaseModel): + question: str + top_k: int = Field(default=10, ge=1, le=50) + use_reranker: bool = True + include_sources: bool = True +``` + +### 1.2 `knowledge_base_ids` — Unbounded List + +**Status**: ⚠️ **High risk** + +**Finding**: `ChatStreamRequest.knowledge_base_ids: list[int] = Field(default_factory=list)` has **no max length**. A client can send hundreds of IDs, triggering one RAG query per ID in parallel. + +**Impact**: With 50 KBs × 150 nodes (50×3 oversample) = 7,500 nodes per request. Combined with reranker calls, this can exhaust memory and GPU. + +**Recommendation**: + +```python +knowledge_base_ids: list[int] = Field( + default_factory=list, + max_length=20, # or configurable via settings.rag_max_knowledge_bases +) +``` + +Add `rag_max_knowledge_bases: int = Field(default=20, ge=1, le=50)` to `config.py` and use it in the schema. + +### 1.3 `message` Length + +**Status**: ✅ `min_length=1` present. Consider `max_length` (e.g. 32_000) to cap context size and prevent abuse. + +--- + +## 2. API Key Exposure in Test Fixtures + +### 2.1 `.env.example` + +**Status**: ✅ Safe + +**Finding**: `.env.example` uses placeholders (`sk-sp-xxxxx`, `your-volcengine-api-key`, `your-email@example.com`). No real secrets. + +**Recommendation**: Add a header comment: + +``` +# NEVER commit .env with real keys. .env is gitignored. +``` + +### 2.2 Real LLM Test Fixtures (Phase 3) + +**Status**: ⚠️ **Medium risk** + +**Finding**: Plan specifies `conftest_real_llm.py` and `LLM_PROVIDER=volcengine` for real LLM tests. Keys must come from environment, not from committed files. + +**Recommendation**: + +1. Document that `VOLCENGINE_API_KEY` must be set in the environment (or a local `.env` that is gitignored). +2. In `conftest_real_llm.py`, add a check: + + ```python + if REAL_LLM_AVAILABLE and not os.environ.get("VOLCENGINE_API_KEY"): + pytest.skip("VOLCENGINE_API_KEY required for real LLM tests") + ``` + +3. Never log or print API keys. Ensure `LLMConfig` and provider clients do not log key values. + +### 2.3 `test_llm_settings.py` + +**Status**: ✅ Uses fake values (`sk-test`, `sk-ant-test`). Safe. + +--- + +## 3. Resource Exhaustion via `rag_top_k=50` + Oversampling + +### 3.1 Per-Query Cost + +**Status**: ⚠️ **High risk** + +**Finding**: With `rag_top_k=50` and `use_reranker=True`: +- `fetch_k = 50 × 3 = 150` nodes retrieved +- Reranker processes 150 query–document pairs (GPU-heavy) +- Adjacent chunk expansion adds more I/O per node + +**Impact**: A single request can be expensive. Under 120 req/min global rate limit, sustained `rag_top_k=50` + `use_reranker=True` can saturate GPU and memory. + +**Recommendations**: + +1. **Cap effective top_k for reranker path** when oversampling is used: + + ```python + # In rag_service.py + EFFECTIVE_TOP_K_MAX = 20 # or from config + if use_reranker and top_k > EFFECTIVE_TOP_K_MAX: + top_k = min(top_k, EFFECTIVE_TOP_K_MAX) + ``` + + Or make `rag_default_top_k` and `rag_max_top_k` configurable (e.g. `rag_max_top_k=25`). + +2. **Add `rag_oversample_factor` cap** in config: `ge=1, le=5` to avoid `top_k × 10` explosion. + +3. **Consider stricter rate limits** for Chat/RAG endpoints (see Section 8). + +--- + +## 4. GPU Resource Abuse via Repeated Reranker Calls + +### 4.1 No Throttling on Reranker + +**Status**: ⚠️ **High risk** + +**Finding**: Plan uses `get_reranker()` with `lru_cache(maxsize=1)`. There is no semaphore or queue limiting concurrent reranker calls. Multiple concurrent Chat requests with `use_reranker=True` can overload the GPU. + +**Recommendation**: Add a reranker semaphore: + +```python +# backend/app/services/rag_service.py or reranker_service.py +_reranker_semaphore = asyncio.Semaphore(2) # or from config.reranker_concurrency_limit + +async def _rerank_nodes(self, nodes, query, top_n): + async with _reranker_semaphore: + # ... existing logic +``` + +Add `reranker_concurrency_limit: int = Field(default=2, ge=1, le=8)` to `config.py`. + +### 4.2 Reranker `lru_cache` and `top_n` + +**Finding**: `get_reranker(*, model_name, top_n)` is cached with `top_n` as part of the key. Different `top_n` values create separate instances. Consider caching only by `model_name` and passing `top_n` at inference time if the library supports it, to avoid multiple model loads. + +--- + +## 5. Test Data Isolation + +### 5.1 Unit/Integration Tests (pytest with ASGITransport) + +**Status**: ✅ Isolated + +**Finding**: `conftest.py` uses `tempfile.mkdtemp()` for `DATA_DIR` and `DATABASE_URL`. Tests use an ephemeral DB and data dir. + +### 5.2 E2E Tests Against Live Server + +**Status**: ⚠️ **Critical risk** + +**Finding**: Plan says E2E tests "interact with live server" on port 8000. If the server uses production `.env` and `/data0/djx/omelette/`, tests will: +- Create real projects and papers +- Write to production ChromaDB +- Consume real LLM API credits + +**Recommendation**: + +1. **Dedicated E2E mode**: Start the server with `APP_ENV=testing`, `DATA_DIR=/tmp/omelette_e2e_XXX`, `DATABASE_URL=sqlite:////tmp/omelette_e2e.db`, and `CHROMA_DB_DIR` pointing to a temp dir. +2. **E2E startup script**: + + ```bash + # scripts/run_e2e_server.sh + export APP_ENV=testing + export DATA_DIR=$(mktemp -d) + export DATABASE_URL="sqlite:///$(mktemp -u)_e2e.db" + export CHROMA_DB_DIR="$DATA_DIR/chroma_db" + uvicorn app.main:app --host 0.0.0.0 --port 8000 + ``` + +3. **Document**: E2E tests must never run against a production server. Add a check in E2E conftest: + + ```python + if os.environ.get("APP_ENV") == "production": + pytest.skip("E2E tests must not run against production") + ``` + +### 5.3 Test PDFs at `/data0/djx/omelette_pdf_test/` + +**Status**: ✅ Acceptable if path is read-only and used only for test fixtures. Ensure E2E does not write to this path; use a copy in a temp dir if needed. + +--- + +## 6. `.env.example` — Secrets Check + +**Status**: ✅ No real secrets + +| Variable | Value | Verdict | +|----------|-------|---------| +| `APP_SECRET_KEY` | `change-me-to-a-random-secret-key` | Placeholder ✅ | +| `API_SECRET_KEY` | (empty) | Safe ✅ | +| `ALIYUN_API_KEY` | `sk-sp-xxxxx` | Placeholder ✅ | +| `VOLCENGINE_API_KEY` | `your-volcengine-api-key` | Placeholder ✅ | +| `OPENAI_API_KEY` | (empty) | Safe ✅ | +| `UNPAYWALL_EMAIL` | `your-email@example.com` | Placeholder ✅ | +| `HTTP_PROXY` | `http://127.0.0.1:20171/` | Environment-specific; consider removing or using placeholder | + +**Recommendation**: Replace `HTTP_PROXY` with a placeholder like `http://localhost:PORT/` to avoid leaking local proxy config. Add a comment that proxy values are user-specific. + +--- + +## 7. HNSW Parameter Injection via API + +**Status**: ✅ No injection risk + +**Finding**: HNSW parameters (`hnsw:construction_ef`, `hnsw:search_ef`, `hnsw:M`) are hardcoded in `RAGService._get_collection()`. No API endpoint accepts or forwards these values. + +**Recommendation**: Keep HNSW params server-side only. If future config is needed, use `config.py` only, never request body or query params. + +--- + +## 8. Rate Limiting on New Chat API Params + +### 8.1 Current State + +**Finding**: Global rate limit is `120/minute` (all endpoints). Chat and RAG use the same limit as lightweight CRUD endpoints. + +### 8.2 Risk + +**Status**: ⚠️ **Medium risk** + +Chat with `rag_top_k=50` + `use_reranker=True` is far more expensive than a simple GET. An attacker can consume most of the budget with a few heavy Chat requests. + +**Recommendation**: + +1. **Stricter limit for Chat/RAG**: + + ```python + # In chat.py and rag.py + from app.middleware.rate_limit import limiter + + @router.post("/stream") + @limiter.limit("30/minute") # Lower than global for expensive ops + async def chat_stream(...): + ... + ``` + +2. **Or** use a tiered approach: e.g. 120/min for normal endpoints, 20/min for Chat stream, 30/min for RAG query. + +3. **Cost-aware limiting** (future): Weight requests by `rag_top_k` and `use_reranker` (e.g. 1 point for simple, 5 for heavy RAG). + +--- + +## Remediation Roadmap + +| Phase | Action | Priority | +|-------|--------|----------| +| Before Phase 2 | Add `rag_top_k`/`use_reranker` to ChatStreamRequest with `Field(ge=1, le=50)` | P0 | +| Before Phase 2 | Add `top_k` validation to RAGQueryRequest | P0 | +| Phase 2 | Cap `knowledge_base_ids` (max_length or config) | P0 | +| Phase 2 | Add reranker semaphore | P1 | +| Phase 2 | Consider `rag_max_top_k` when `use_reranker=True` | P1 | +| Phase 3 | E2E test isolation (dedicated env, temp dirs) | P0 | +| Phase 3 | Real LLM tests: require env vars, no key logging | P1 | +| Phase 4 | Stricter rate limits for Chat/RAG | P2 | +| Phase 4 | Clean `.env.example` proxy placeholder | P3 | + +--- + +## Checklist for Implementation + +- [ ] `ChatStreamRequest`: `rag_top_k` with `ge=1, le=50`; `use_reranker: bool` +- [ ] `ChatStreamRequest`: `knowledge_base_ids` with `max_length` or config-driven cap +- [ ] `RAGQueryRequest`: `top_k` with `Field(ge=1, le=50)` +- [ ] Reranker: semaphore for concurrent calls +- [ ] Config: `rag_max_knowledge_bases`, `reranker_concurrency_limit`, optionally `rag_max_top_k` +- [ ] E2E: Document and enforce test-only server env (APP_ENV=testing, temp DATA_DIR) +- [ ] Real LLM tests: Skip if API key missing; never log keys +- [ ] Rate limiting: Consider stricter limits for Chat/RAG diff --git a/docs/plans/2026-03-17-refactor-config-rag-api-testing-plan.md b/docs/plans/2026-03-17-refactor-config-rag-api-testing-plan.md new file mode 100644 index 0000000..31e19e6 --- /dev/null +++ b/docs/plans/2026-03-17-refactor-config-rag-api-testing-plan.md @@ -0,0 +1,752 @@ +--- +title: "refactor: 配置修复 + RAG 召回优化 + 全接口测试" +type: refactor +status: completed +date: 2026-03-17 +origin: docs/brainstorms/2026-03-17-config-rag-api-testing-brainstorm.md +--- + +# 配置修复 + RAG 召回优化 + 全接口测试 + +## Enhancement Summary + +**Deepened on:** 2026-03-17 +**Sections enhanced:** 6 (Phase 1 config, Phase 2 reranker/MMR/HNSW/Chat/tests, Phase 3 testing) +**Research agents used:** framework-docs-researcher, best-practices-researcher, performance-oracle, architecture-strategist, security-sentinel, kieran-python-reviewer, learnings-researcher + +### Key Improvements + +1. **[CRITICAL] Qwen3-Reranker 兼容性修正**: 原生 `Qwen/Qwen3-Reranker-0.6B` 使用 CausalLM 格式，与 `SentenceTransformerRerank` / `CrossEncoder` **不兼容**。必须使用 `tomaarsen/Qwen3-Reranker-0.6B-seq-cls` 变体。 +2. **N+1 相邻 chunk 查询修复**: `_get_adjacent_chunks` 每个 node 调用 2 次 ChromaDB，过采样时可达 300 次/请求。需批量化为单次 `collection.get()`。 +3. **Reranker 缓存策略修正**: `lru_cache(maxsize=1)` + `top_n` 作为缓存键会导致缓存抖动。改为不含 `top_n`，在调用侧截断。 +4. **Reranker 并发安全**: 添加 `asyncio.Semaphore(1)` 防止 PyTorch 推理竞态。 +5. **测试基础设施修正**: `conftest_real_llm.py` 不会被 pytest 自动加载 → 合并到 `conftest.py`。 +6. **安全强化**: 限制 `knowledge_base_ids` 长度、RAG `top_k` 边界验证、E2E 测试数据隔离。 + +### New Considerations Discovered + +- ChromaDB MMR 支持需要 LlamaIndex PR #19731（2025.08+），需确认当前版本 +- HNSW `ef_construction` 和 `M` 为**不可变参数**，仅对新建 collection 生效 +- Reranker 0.6B 约占 1.2 GB GPU，与 Embedding 0.6B 共计约 2.5 GB +- `reranker.postprocess_nodes` 需要 `QueryBundle` 而非 `query_str` + +--- + +## Overview + +三阶段后端改进：(1) 修复配置三端不一致问题，(2) 实现 RAG 向量召回的 reranking + MMR + HNSW 调优，(3) 对全部 API 端点进行文档化和 mock + 真实 LLM 双轨测试。 + +## Problem Statement / Motivation + +1. **配置脱节**: `config.py` 默认值（`BAAI/bge-m3`）与实际使用的 Qwen3 模型不一致，`.env.example` 混入环境特定配置 +2. **RAG 召回质量差**: reranking 是死代码、无结果多样性控制、HNSW 未调优、Chat 端无法控制检索参数 +3. **测试覆盖不足**: 229 个测试全用 mock LLM，未验证真实 LLM 端到端行为；上轮重构改动 20 文件未做 E2E 验证 + +## Proposed Solution + +按顺序执行三个阶段，每个阶段独立可测。 + +## Technical Approach + +### Phase 1: 配置一致性修复 + +**目标**: 同步 `config.py` 默认值、`.env.example`、`.env` + +#### Step 1.1: 更新 `config.py` 默认值 + +**文件**: `backend/app/config.py` + +```python +embedding_model: str = "Qwen/Qwen3-Embedding-0.6B" # was: BAAI/bge-m3 +reranker_model: str = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls" # was: BAAI/bge-reranker-v2-m3 +pdf_parser: str = "mineru" # was: auto +mineru_timeout: int = 8000 # was: 300 +cuda_visible_devices: str = "5,6,7" # was: 0,3 +``` + +#### Step 1.2: 清理 `.env.example` + +**文件**: `.env.example` + +- `APP_DEBUG=true`（当前 debug 阶段，保持 true） +- `EMBEDDING_MODEL=Qwen/Qwen3-Embedding-8B`（推荐较大模型） +- `RERANKER_MODEL=tomaarsen/Qwen3-Reranker-8B-seq-cls`（推荐较大模型） +- `VOLCENGINE_MODEL=doubao-seed-2-0-mini-260215`（更新模型名） +- `HTTP_PROXY=` 改为通用占位符 `# HTTP_PROXY=http://your-proxy:port` +- 其余保持与当前 `.env.example` 一致 + +> **Decision**: config.py 默认值 = 实际最小可用（0.6B-seq-cls）；.env.example = 推荐配置（8B-seq-cls） + +### Research Insights (Phase 1) + +**Qwen3-Reranker 兼容性 [CRITICAL]**: +- 原生 `Qwen/Qwen3-Reranker-*` 使用 `AutoModelForCausalLM` + yes/no token logits，**不能**直接用于 `CrossEncoder` / `SentenceTransformerRerank` +- 社区提供了 seq-cls 转换版本：`tomaarsen/Qwen3-Reranker-0.6B-seq-cls`（和 8B 版本），兼容 `sentence-transformers` `CrossEncoder` API +- 依赖版本要求：`transformers>=4.51.0`，`sentence-transformers>=4.0.0` +- 参考：https://huggingface.co/tomaarsen/Qwen3-Reranker-8B-seq-cls + +#### Step 1.3: 验证 + +- 运行 `pytest` 确认 229 测试全部通过 +- 确认 mock 模式不受影响 + +--- + +### Phase 2: RAG 向量召回优化 + +#### Step 2.1: 添加 Reranker 依赖 + +**文件**: `backend/pyproject.toml` + +```toml +[project.optional-dependencies] +ml = [ + # ... existing ... + "llama-index-postprocessor-sbert-rerank>=0.4.0", + "sentence-transformers>=4.0.0", + "transformers>=4.51.0", +] +``` + +### Research Insights (Step 2.1) + +**包名修正**: PyPI 包名为 `llama-index-postprocessor-sbert-rerank`（非 `sentence-transformer-rerank`）。对应导入路径： + +```python +from llama_index.postprocessor.sbert_rerank import SentenceTransformerRerank +# 或从 core 导入（如果版本足够新）： +from llama_index.core.postprocessor import SentenceTransformerRerank +``` + +#### Step 2.2: 创建 Reranker 服务 + +**文件**: `backend/app/services/reranker_service.py`（新建） + +```python +"""Reranker model loading and caching.""" +from __future__ import annotations + +import asyncio +import logging +from functools import lru_cache +from typing import TYPE_CHECKING + +from app.config import settings + +if TYPE_CHECKING: + from llama_index.core.schema import NodeWithScore, QueryBundle + +logger = logging.getLogger(__name__) + +_reranker_semaphore = asyncio.Semaphore(1) + + +@lru_cache(maxsize=1) +def _load_reranker(model_name: str): + """Load and cache a SentenceTransformerRerank by model name only.""" + from llama_index.postprocessor.sbert_rerank import SentenceTransformerRerank + from app.services.embedding_service import _inject_hf_env + + _inject_hf_env() + logger.info("Loading reranker model=%s", model_name) + return SentenceTransformerRerank( + model=model_name, + top_n=50, + device="cuda", + keep_retrieval_score=True, + ) + + +def get_reranker(*, model_name: str | None = None): + """Return a cached reranker. top_n is controlled at call site.""" + name = model_name or settings.reranker_model + return _load_reranker(name) + + +async def rerank_nodes( + nodes: list[NodeWithScore], + query: str, + top_n: int, +) -> list[NodeWithScore]: + """Apply reranker with concurrency control and graceful fallback.""" + try: + from llama_index.core.schema import QueryBundle as QB + + reranker = get_reranker() + query_bundle = QB(query_str=query) + async with _reranker_semaphore: + reranked = await asyncio.to_thread( + reranker.postprocess_nodes, + nodes, + query_bundle=query_bundle, + ) + return reranked[:top_n] + except (ImportError, OSError, RuntimeError): + logger.warning("Reranking failed, returning original nodes", exc_info=True) + return nodes[:top_n] +``` + +### Research Insights (Step 2.2) + +**关键设计改进（综合 performance-oracle + architecture-strategist + python-reviewer）**: + +1. **缓存策略**: `lru_cache` 只以 `model_name` 为键，`top_n` 在 `rerank_nodes()` 中通过 `[:top_n]` 截断。避免不同 `rag_top_k` 导致缓存抖动和重复加载模型。 +2. **并发安全**: `asyncio.Semaphore(1)` 序列化 reranker 推理，避免 PyTorch 模型在多线程下竞态。 +3. **HF 环境注入**: 复用 `embedding_service._inject_hf_env()` 设置 HuggingFace 镜像和缓存路径。 +4. **`keep_retrieval_score=True`**: 保留原始检索分数，方便调试和质量对比。 +5. **`device="cuda"`**: 利用 GPU 加速推理。0.6B 模型约 1.2 GB 显存。 +6. **`QueryBundle`**: LlamaIndex `postprocess_nodes` 接受 `QueryBundle` 对象而非 `query_str` 关键字。 +7. **异常粒度**: 仅捕获 `ImportError`（包未安装）、`OSError`（模型加载失败）、`RuntimeError`（推理失败），不吞没其他异常。 +8. **可测试性**: 测试中调用 `_load_reranker.cache_clear()` 清除缓存。 + +**GPU 显存估算**: + +| 模型 | 显存 | 推荐 | +|------|------|------| +| Qwen3-Embedding-0.6B | ~1.2 GB | GPU 5 | +| Qwen3-Reranker-0.6B-seq-cls | ~1.2 GB | GPU 6 | +| 8B 版本 | ~16 GB 各 | 需独占 GPU | + +#### Step 2.3: 实现 Reranking 逻辑 + +**文件**: `backend/app/services/rag_service.py` + +**修改 `query()`**（lines 226-303）: + +```python +async def query(self, ..., use_reranker: bool = False, ...): + oversample = settings.rag_oversample_factor if use_reranker else 1 + fetch_k = min(top_k * oversample, count) + retriever = index.as_retriever(similarity_top_k=fetch_k) + retrieved_nodes = await asyncio.to_thread(retriever.retrieve, question) + + if use_reranker and retrieved_nodes: + from app.services.reranker_service import rerank_nodes + retrieved_nodes = await rerank_nodes(retrieved_nodes, question, top_n=top_k) + + # 后续处理不变... +``` + +**修改 `retrieve_only()`**（lines 304-354）: + +```python +async def retrieve_only( + self, + project_id: int, + question: str, + top_k: int = 10, + use_reranker: bool = False, +) -> list[dict]: + oversample = settings.rag_oversample_factor if use_reranker else 1 + fetch_k = min(top_k * oversample, count) + retriever = index.as_retriever(similarity_top_k=fetch_k) + retrieved_nodes = await asyncio.to_thread(retriever.retrieve, question) + + if use_reranker and retrieved_nodes: + from app.services.reranker_service import rerank_nodes + retrieved_nodes = await rerank_nodes(retrieved_nodes, question, top_n=top_k) + + # 后续 adjacent chunk 处理... +``` + +### Research Insights (Step 2.3) + +**N+1 相邻 chunk 查询批量化 [P0 性能修复]**: + +当前 `_get_adjacent_chunks` 每个 node 调用 2 次 `collection.get()`。过采样时 `fetch_k = 150`，产生 **300 次 ChromaDB 调用**。 + +**批量替代方案**: + +```python +async def _get_adjacent_chunks_batch( + self, + collection: chromadb.Collection, + nodes: list, +) -> dict[tuple[int, int], tuple[str, str]]: + """Batch-fetch all adjacent chunks in one ChromaDB call.""" + all_ids: set[str] = set() + node_keys: list[tuple[int | None, int | None]] = [] + adj_map: dict[tuple, tuple[list[str], list[str]]] = {} + + for n in nodes: + meta = (n.node if hasattr(n, "node") else n).metadata or {} + pid, cidx = meta.get("paper_id"), meta.get("chunk_index") + key = (pid, cidx) + node_keys.append(key) + if pid is None or cidx is None: + adj_map[key] = ([], []) + continue + prev_id = f"paper_{pid}_chunk_{cidx - 1}" + next_id = f"paper_{pid}_chunk_{cidx + 1}" + all_ids.update([prev_id, next_id]) + adj_map[key] = ([prev_id], [next_id]) + + if not all_ids: + return {k: ("", "") for k in node_keys} + + result = await asyncio.to_thread( + collection.get, ids=list(all_ids), include=["documents"] + ) + id_to_doc = dict(zip(result["ids"] or [], result.get("documents") or [])) + + return { + k: ( + "\n".join(id_to_doc.get(i, "") or "" for i in adj_map.get(k, ([], []))[0]), + "\n".join(id_to_doc.get(i, "") or "" for i in adj_map.get(k, ([], []))[1]), + ) + for k in node_keys + } +``` + +**效果**: 300 次 ChromaDB 调用 → **1 次**。 + +**检索流程顺序（最佳实践确认）**: + +``` +Dense retrieval (oversample) → Reranking (relevance) → Adjacent chunk expansion → 返回 +``` + +> MMR 在 retriever 层面通过 `vector_store_query_mode="mmr"` 实现，与 reranking 正交。 + +#### Step 2.4: MMR 多样性 + +**文件**: `backend/app/services/rag_service.py` + +```python +# 在 retriever 创建时使用 MMR +if use_mmr: + retriever = index.as_retriever( + similarity_top_k=fetch_k, + vector_store_query_mode="mmr", + vector_store_kwargs={"mmr_threshold": settings.rag_mmr_threshold}, + ) +else: + retriever = index.as_retriever(similarity_top_k=fetch_k) +``` + +**添加 `use_mmr` 参数**到 `query()` 和 `retrieve_only()`。 + +### Research Insights (Step 2.4) + +**MMR with ChromaDB**: +- LlamaIndex PR #19731（2025.08）为 ChromaVectorStore 添加了 Python 侧 MMR 实现 +- 工作方式：从 ChromaDB 获取更多结果，在客户端应用 MMR 算法 +- `mmr_threshold` 即 λ：0 = 最大多样性，1 = 纯相关性，**推荐 0.5（平衡）** +- **需确认当前 `llama-index-vector-stores-chroma` 版本 >= 0.4.0** + +**参数设计建议**: +- 全局默认通过 `settings.rag_mmr_threshold` 控制（0.0 = 关闭 MMR） +- 暂不暴露 `use_mmr` 到 `ChatStreamRequest`，仅通过配置控制 + +#### Step 2.5: HNSW 调优 + +**文件**: `backend/app/services/rag_service.py` + +```python +def _get_collection(self, project_id: int) -> chromadb.Collection: + return self._get_chroma_client().get_or_create_collection( + name=f"project_{project_id}", + metadata={ + "hnsw:space": "cosine", + "hnsw:construction_ef": 200, + "hnsw:search_ef": 100, + "hnsw:M": 32, + }, + ) +``` + +### Research Insights (Step 2.5) + +**HNSW 参数详解**: + +| 参数 | 值 | 说明 | 可变性 | +|------|-----|------|--------| +| `hnsw:space` | `cosine` | 距离度量 | 不可变 | +| `hnsw:construction_ef` | 200 | 构建时邻居搜索范围（越大越精但更慢） | 不可变 | +| `hnsw:M` | 32 | 每节点最大连接数（越大越精但更耗内存） | 不可变 | +| `hnsw:search_ef` | 100 | 查询时邻居搜索范围 | **可变**，可用 `collection.modify()` | + +**对已有 collection 的影响**: +- `space`、`construction_ef`、`M` 为**不可变参数**，仅在 `get_or_create_collection` 创建时生效 +- 已有 collection 保持原有参数（默认 `ef_construction=100`, `M=16`） +- **需提供索引重建脚本/说明**：删除旧 collection → 重新索引 + +**Chroma 版本注意**: Chroma 1.0+ 推荐使用 `configuration={"hnsw": {...}}` 而非 `metadata`。需检查当前安装版本。 + +#### Step 2.6: 暴露参数到 Chat API + +**文件**: `backend/app/schemas/conversation.py` + +```python +class ChatStreamRequest(BaseModel): + conversation_id: int | None = None + knowledge_base_ids: list[int] = Field( + default_factory=list, + max_length=20, + description="Knowledge base IDs for RAG retrieval", + ) + model: str | None = None + tool_mode: str = "qa" + message: str = Field(min_length=1) + rag_top_k: int = Field(default=10, ge=1, le=50, description="RAG retrieval top-k") + use_reranker: bool = Field(default=False, description="Apply reranker to retrieved nodes") +``` + +**文件**: `backend/app/api/v1/chat.py` + +```python +initial_state["rag_top_k"] = req.rag_top_k +initial_state["use_reranker"] = req.use_reranker +``` + +**文件**: `backend/app/pipelines/chat/nodes.py` (`retrieve_node`) + +```python +sources = await rag.retrieve_only( + project_id=kb_id, + question=question, + top_k=state.get("rag_top_k", 10), + use_reranker=state.get("use_reranker", False), +) +``` + +**文件**: `backend/app/api/v1/rag.py`（RAG API 对齐） + +```python +class RAGQueryRequest(BaseModel): + question: str + top_k: int = Field(default=10, ge=1, le=50) # 添加边界验证 + use_reranker: bool = True # RAG API 默认启用 +``` + +### Research Insights (Step 2.6) + +**API Surface Parity**: + +| 参数 | RAG API | Chat API | 说明 | +|------|---------|----------|------| +| `top_k` | `ge=1, le=50`，默认 10 | `ge=1, le=50`，默认 10 | 统一验证 | +| `use_reranker` | 默认 `True` | 默认 `False` | RAG API 面向精确查询（偏质量），Chat 面向流式交互（偏延迟） | +| `knowledge_base_ids` | N/A | `max_length=20` | 限制并行检索数量 | + +**安全强化（security-sentinel）**: +- `knowledge_base_ids` 无长度限制 → 50 个 KB × 150 nodes = 7,500 nodes → 添加 `max_length=20` +- RAG `top_k` 无边界 → `top_k=999999` 可能 → 添加 `Field(ge=1, le=50)` +- Chat/RAG 端点建议使用更严格的速率限制（20-30/min vs 全局 120/min） + +#### Step 2.7: 添加 RAG 配置项 + +**文件**: `backend/app/config.py` + +```python +# RAG retrieval +rag_default_top_k: int = Field(default=10, ge=1, le=100, description="Default retrieval top-k") +rag_oversample_factor: int = Field(default=3, ge=1, le=10, description="Multiplier for oversampling before rerank") +rag_mmr_threshold: float = Field(default=0.5, ge=0.0, le=1.0, description="MMR diversity threshold (0=off, 0.5=balanced)") +reranker_concurrency_limit: int = Field(default=1, ge=1, le=4, description="Max concurrent reranker calls") +``` + +#### Step 2.8: 运行测试验证 + +- 运行全部 229 测试确认无回归 +- 测试 `use_reranker=False` 路径（默认行为不变） +- 测试 `use_reranker=True` 路径（mock reranker via `_load_reranker.cache_clear()` + `@patch`） +- 所有 RAG sync 调用确认已包裹 `asyncio.to_thread()` + +--- + +### Phase 3: 全接口文档 + 测试 + +#### Step 3.1: 生成 API 端点文档 + +**文件**: `docs/api-endpoints.md`（新建） + +整理全部端点（~77-82 个），按模块分组，包含： +- HTTP 方法 + 路径 +- 简要描述 +- 关键参数 +- 是否涉及 LLM 调用 +- 测试优先级标记 + +#### Step 3.2: 扩展测试基础设施 + +**文件**: `backend/conftest.py`（扩展，不新建文件） + +```python +import pytest + +REAL_LLM_AVAILABLE = os.environ.get("LLM_PROVIDER", "mock") != "mock" + +real_llm = pytest.mark.skipif( + not REAL_LLM_AVAILABLE, + reason="Real LLM not configured (set LLM_PROVIDER=volcengine)" +) +``` + +**文件**: `backend/pyproject.toml`（注册 marker） + +```toml +[tool.pytest.ini_options] +markers = [ + "real_llm: marks tests requiring real LLM (deselect with -m 'not real_llm')", +] +``` + +### Research Insights (Step 3.2) + +**测试基础设施修正（architecture-strategist）**: +- `conftest_real_llm.py` **不会被 pytest 自动加载**（只有 `conftest.py` 会）→ 合并到 `conftest.py` +- `--override-ini="LLM_PROVIDER=volcengine"` 不设环境变量 → 改用 `LLM_PROVIDER=volcengine pytest ...` +- `conftest.py` 已使用 `tempfile.mkdtemp()` 做 DB 隔离，新测试自动受益 + +**Fixture 模式（learnings）**: + +```python +@pytest.fixture +async def client(): + """httpx AsyncClient with ASGITransport for in-process testing.""" + from httpx import ASGITransport, AsyncClient + from app.main import app + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac +``` + +**Chat/Stream 测试模式（learnings）**: +- Monkeypatch `_init_services` 注入 mock LLM/RAG，避免 DB 依赖的服务初始化 +- SSE 测试：await 完整响应 → 解析 `data:` 行 → 断言事件类型（start、text-delta、finish、[DONE]） +- 参考：`docs/solutions/integration-testing/2026-03-16-fastapi-langgraph-integration-testing-best-practices.md` + +#### Step 3.3: 按模块编写测试 + +**并行化策略**: 开多个 Agent，每个负责一组模块： + +| Agent | 模块 | 端点数 | 特点 | +|-------|------|--------|------| +| Agent 1 | Projects + Papers + Upload | ~17 | 纯 CRUD + 文件 I/O | +| Agent 2 | Keywords + Search + Dedup | ~14 | CRUD + LLM 调用 | +| Agent 3 | Chat + RAG + Writing + Completion + Rewrite | ~14 | 全部涉及 LLM，核心流程 | +| Agent 4 | Conversations + Subscriptions + Tasks + Settings + Pipelines | ~22 | 混合：CRUD + 管线 + 配置 | + +**每个测试文件结构**: + +```python +# tests/test__e2e.py +import pytest +from httpx import ASGITransport, AsyncClient +from app.main import app + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + +class TestModuleMock: + """Mock LLM tests — always run.""" + + async def test_endpoint_basic(self, client): + ... + +@pytest.mark.real_llm +class TestModuleRealLLM: + """Real LLM tests — require LLM_PROVIDER=volcengine.""" + + async def test_endpoint_with_volcengine(self, client): + ... +``` + +#### Step 3.4: E2E 测试（真实服务器） + +**启动服务器**: + +```bash +cd backend && conda run -n omelette uvicorn app.main:app --host 0.0.0.0 --port 8000 +``` + +**E2E 测试流程**: + +1. 创建项目 → 上传 8 篇 PDF → 触发 OCR/索引 +2. 创建对话 → Chat 流式交互（qa/citation 模式） +3. RAG query → 验证 reranking 效果 +4. Writing 服务 → 摘要/综述 +5. Keyword 扩展 → LLM 生成 +6. Dedup → LLM 验证/解决 + +**真实 LLM 断言策略**: +- **结构断言**: 检查响应格式（JSON schema、SSE 事件类型） +- **非空断言**: 检查 LLM 返回非空字符串 +- **关键词断言**: 检查输出包含问题中的关键术语 +- **不做精确匹配**: LLM 输出不确定，避免精确字符串比对 + +### Research Insights (Step 3.4) + +**E2E 安全（security-sentinel）**: +- E2E 测试必须使用 `APP_ENV=testing`，配置独立 `DATA_DIR`（`tempfile.mkdtemp()`） +- **严禁使用生产 `.env` 和 `/data0/djx/omelette/` 数据目录** +- 添加启动检查：`if APP_ENV == "production": sys.exit("E2E tests cannot run in production")` + +**真实 LLM 速率控制**: +- Volcengine 并发限制 → 添加 `asyncio.Semaphore(2)` 限制同时进行的 LLM 测试 + +```python +_real_llm_semaphore = asyncio.Semaphore(2) + +@pytest.fixture +async def real_llm_slot(): + async with _real_llm_semaphore: + yield +``` + +**RAG 质量度量（可选）**: +- nDCG@10 衡量 reranking 改进 +- MRR 衡量首个命中的排名 +- 对比 `use_reranker=True` vs `False` 的结果差异 + +#### Step 3.5: 运行完整测试套件 + +```bash +# 1. Mock 测试（快速，CI 用） +pytest tests/ -x -q + +# 2. 真实 LLM 测试（慢，手动触发） +LLM_PROVIDER=volcengine pytest tests/ -m real_llm -x -v + +# 3. E2E 测试（需启动服务器 + APP_ENV=testing） +APP_ENV=testing pytest tests/test_e2e_full.py -x -v +``` + +--- + +## System-Wide Impact + +### Interaction Graph + +- Config 变更 → 影响所有使用 `settings.embedding_model` / `settings.reranker_model` 的服务 +- Reranker 添加 → `reranker_service.py` → `rag_service.py` → `chat/nodes.py` → `ChatStreamRequest` +- HNSW 调优 → 仅影响新建 collection（旧索引需重建） + +### Error & Failure Propagation + +- Reranker 包未安装 → `ImportError` → `rerank_nodes` 捕获 → 返回原始 top_k 结果 +- Reranker 模型加载失败 → `OSError` → 同上 fallback +- GPU OOM → `RuntimeError` → 同上 fallback +- Volcengine API 限流 → 测试 semaphore 控制并发 + +### State Lifecycle Risks + +- HNSW 参数变更 → 旧 collection 不受影响 → 需提供重建脚本 +- Reranker 加载是全局单例 + semaphore → 进程内序列化，无并发问题 +- adjacent chunk 批量查询 → 单次 ChromaDB 调用，事务完整 + +### API Surface Parity + +- `rag.query()` 和 `rag.retrieve_only()` 同时支持 `use_reranker` 和过采样 +- RAG API 默认 `use_reranker=True`（偏质量），Chat API 默认 `False`（偏延迟） +- `top_k` 统一 `ge=1, le=50` 验证 + +### Integration Test Scenarios + +| 场景 | 单元测试无法覆盖的原因 | +|------|------------------------| +| Reranker + Embedding GPU 争用 | 需真实 GPU 负载 | +| RAG 索引 → Chat → RAG query 跨端点 | 跨 endpoint 流程 | +| MinerU 超时 / OCR 失败 | 外部服务行为 | +| Volcengine 速率限制 | 并行请求 | +| ChromaDB 删除后重建索引 | 索引生命周期 | +| Chat 流式 + 多 KB 并行检索 | 并发 + SSE | + +--- + +## Acceptance Criteria + +### Phase 1 + +- [x] `config.py` 默认值使用 `tomaarsen/Qwen3-Reranker-0.6B-seq-cls`（非原生 Qwen3-Reranker） +- [x] `.env.example` 使用 `tomaarsen/Qwen3-Reranker-8B-seq-cls` +- [x] `.env.example` 中 HTTP_PROXY 改为通用占位符 +- [x] 229 测试全部通过 + +### Phase 2 + +- [x] `llama-index-postprocessor-sbert-rerank` + `sentence-transformers>=4.0.0` 已安装 +- [x] `reranker_service.py` 使用 `lru_cache(maxsize=1)` 仅以 `model_name` 为键 +- [x] `rerank_nodes()` 使用 `Semaphore(1)` 序列化推理 +- [x] `rerank_nodes()` 使用 `QueryBundle` 而非 `query_str` +- [x] `_get_adjacent_chunks` 替换为批量 `_get_adjacent_chunks_batch` +- [x] 过采样使用 `settings.rag_oversample_factor` +- [x] MMR 通过 `settings.rag_mmr_threshold` 控制 +- [x] HNSW 参数已调优（ef_construction=200, M=32, ef_search=100） +- [x] `ChatStreamRequest` 支持 `rag_top_k`、`use_reranker`，`knowledge_base_ids` 限制 `max_length=20` +- [x] `RAGQueryRequest.top_k` 添加 `ge=1, le=50` +- [x] Chat pipeline 传递 `rag_top_k` 和 `use_reranker` 到 `initial_state` +- [x] `rank_node` 维持批量 Paper 查询（无 N+1 回归） +- [x] 所有 RAG sync 调用包裹 `asyncio.to_thread()` +- [x] reranker 失败时 graceful fallback（不影响请求） +- [x] 229 + 新增 reranker 测试全部通过（370 passed, 2 skipped） + +### Phase 3 + +- [x] `docs/api-endpoints.md` 文档覆盖全部端点 +- [x] `real_llm` marker 注册到 `pyproject.toml` +- [x] real LLM 逻辑合并到 `conftest.py`（非 conftest_real_llm.py） +- [x] 每个端点至少有一个 mock LLM 测试 +- [x] 涉及 LLM 的端点有 `@pytest.mark.real_llm` 真实测试 +- [x] E2E 使用 `APP_ENV=testing` 和独立 DATA_DIR +- [x] 真实 LLM 测试使用 Volcengine（25 passed, 1 skipped） +- [x] SSE 测试验证事件类型（start、text-delta、finish、[DONE]） +- [x] E2E 流程（上传 → 索引 → 聊天 → 写作）通过 + +--- + +## Dependencies & Risks + +| 风险 | 严重度 | 缓解措施 | +|------|--------|----------| +| Qwen3-Reranker seq-cls 版本兼容性 | 高 | 先在 notebook 中验证模型加载和推理 | +| GPU 显存不足（Embedding + Reranker） | 中 | 0.6B 共计 ~2.5GB，CUDA 5,6,7 各 80GB 足够 | +| ChromaDB MMR 版本要求 | 中 | 检查 `llama-index-vector-stores-chroma` 版本 ≥ 0.4.0 | +| HNSW 参数仅对新 collection 生效 | 低 | 提供重建脚本，记录迁移步骤 | +| MinerU 服务不可用 | 低 | E2E 测试标记 `@pytest.mark.skipif(not MINERU_AVAILABLE)` | +| Volcengine 限流 | 低 | Semaphore + 重试 | +| `sentence-transformers>=4.0.0` 与现有依赖冲突 | 中 | 安装时检查，必要时固定版本 | + +--- + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-03-17-config-rag-api-testing-brainstorm.md](docs/brainstorms/2026-03-17-config-rag-api-testing-brainstorm.md) + - 决策：RAG 优先做 reranking + MMR，BM25 留后续 + - 决策：.env.example 更新为当前实际配置 + - 决策：全部 77 端点 pytest + E2E 双轨测试 + +### Internal References + +- RAG 技术报告: `docs/research/llamaindex-rag-technical-report.md` +- 原始 RAG 计划 Phase 4: `docs/plans/2026-03-11-feat-llamaindex-rag-engine-plan.md` +- Async 最佳实践: `docs/solutions/performance-issues/blocking-sync-calls-asyncio-to-thread.md` +- 测试模式: `docs/solutions/integration-testing/2026-03-16-fastapi-langgraph-integration-testing-best-practices.md` +- N+1 优化: `docs/solutions/performance-issues/2026-03-12-rag-rich-citation-performance-analysis.md` +- Chat 路由: `docs/solutions/integration-issues/2026-03-12-chat-routing-chain-langgraph-aisdk-rewrite.md` + +### External References + +- Qwen3-Reranker-seq-cls: https://huggingface.co/tomaarsen/Qwen3-Reranker-8B-seq-cls +- LlamaIndex Node Postprocessors: https://docs.llamaindex.ai/en/stable/module_guides/querying/node_postprocessors/ +- ChromaDB HNSW Config: https://docs.trychroma.com/docs/collections/configure +- LlamaIndex MMR PR #19731: https://github.com/run-llama/llama_index/pull/19731 +- llama-index-postprocessor-sbert-rerank: https://pypi.org/project/llama-index-postprocessor-sbert-rerank/ + +### Key Files + +- `backend/app/services/rag_service.py` — RAG 核心 +- `backend/app/services/embedding_service.py` — Embedding 加载 +- `backend/app/services/reranker_service.py` — Reranker 加载（新建） +- `backend/app/config.py` — 配置中心 +- `backend/app/pipelines/chat/nodes.py` — Chat pipeline +- `backend/app/schemas/conversation.py` — Chat 请求 schema +- `backend/app/api/v1/rag.py` — RAG API +- `backend/conftest.py` — 测试环境设置 diff --git a/docs/plans/2026-03-18-feat-gpu-cleanup-on-exit-plan.md b/docs/plans/2026-03-18-feat-gpu-cleanup-on-exit-plan.md new file mode 100644 index 0000000..f179ddb --- /dev/null +++ b/docs/plans/2026-03-18-feat-gpu-cleanup-on-exit-plan.md @@ -0,0 +1,97 @@ +--- +title: "feat(backend): 程序退出时 GPU 资源自动释放" +type: feat +status: active +date: 2026-03-18 +origin: docs/brainstorms/2026-03-18-gpu-cleanup-on-exit-brainstorm.md +--- + +# 程序退出时 GPU 资源自动释放 + +## Overview + +实现两层防护确保程序退出时释放所有 GPU 资源： +1. **进程内安全网** — atexit、signal handler、lifespan 增强 +2. **外部看门狗** — 独立脚本监控主进程，退出后清理残留 + +## Implementation Phases + +### Phase 1: 进程内安全网 + +#### 1.1 atexit 同步清理 + +**修改文件**: `backend/app/main.py` + +- [ ] 注册 `atexit` 回调函数 `_sync_cleanup()`，在 Python 解释器退出前执行： + - 调用 `gpu_model_manager.unload_all()` 释放所有 GPU 模型 + - 杀死 MinerU 子进程（如有） + - 杀死端口 8010 上的外部 MinerU（通过 PID 查找） +- [ ] `_sync_cleanup()` 必须是**同步函数**（atexit 不支持 async） +- [ ] 在 lifespan startup 中注册 `atexit.register(_sync_cleanup)` + +#### 1.2 SIGHUP 处理 + +**修改文件**: `backend/app/main.py` + +- [ ] 注册 `signal.signal(signal.SIGHUP, _handle_sighup)` 处理终端关闭 +- [ ] `_handle_sighup` 中调用 `_sync_cleanup()` 后退出 +- [ ] 注意：SIGHUP 在 uvicorn 运行时可能被覆盖，需在 lifespan startup 中注册 + +#### 1.3 Lifespan 增强 — 杀外部 MinerU + +**修改文件**: `backend/app/services/mineru_process_manager.py` + +- [ ] 新增方法 `kill_external_by_port()` — 通过 `lsof -ti:{port}` 或 `ss` 查找并杀死占用 MinerU 端口的进程 +- [ ] 在 `stop()` 方法中，如果 `_is_external` 为 True，也调用 `kill_external_by_port()` +- [ ] 安全检查：不杀自己的 PID + +#### 1.4 PID 文件 + +**修改文件**: `backend/app/main.py`, `backend/app/config.py` + +- [ ] `config.py`: 添加 `pid_file: str` 配置（默认 `{data_dir}/omelette.pid`） +- [ ] lifespan startup: 写入当前 PID 到文件 +- [ ] lifespan shutdown / atexit: 删除 PID 文件 + +### Phase 2: 外部看门狗脚本 + +**新增文件**: `backend/scripts/gpu_watchdog.py` + +- [ ] 独立 Python 脚本，不依赖 Omelette 代码 +- [ ] 读取 PID 文件，监控主进程是否存活 +- [ ] 主进程退出后执行清理： + 1. 查找并杀死 MinerU（通过端口） + 2. 查找并杀死其他占用 `CUDA_VISIBLE_DEVICES` 的 Python 进程（可选） + 3. 删除 PID 文件 +- [ ] 支持命令行参数：`--pid-file`, `--interval`(检查间隔，默认 5s), `--mineru-port` +- [ ] 后台运行：`python scripts/gpu_watchdog.py --daemon` + +### Phase 3: 集成 + 测试 + +- [ ] 测试 Ctrl+C 场景 +- [ ] 测试 kill PID 场景 +- [ ] 测试关闭终端场景 +- [ ] 测试看门狗 kill -9 场景 +- [ ] `ruff check` + `ruff format` + +## Acceptance Criteria + +- [ ] Ctrl+C 后 `nvidia-smi` 无 Omelette/MinerU 进程 +- [ ] kill PID 后同上 +- [ ] 关闭终端后同上 +- [ ] kill -9 后看门狗 5s 内清理完毕 +- [ ] 正常 TTL 功能不受影响 + +## Risk & Mitigation + +| 风险 | 缓解措施 | +|------|---------| +| `signal.signal(SIGHUP)` 被 uvicorn 覆盖 | 在 lifespan startup 中注册（uvicorn 已完成信号设置后） | +| `lsof` 不可用 | fallback 到 `ss -tlnp` 或 `/proc/net/tcp` + `/proc/{pid}/cmdline` | +| 看门狗自身挂掉 | lifespan startup 自动启动看门狗；看门狗自带简单心跳 | +| atexit 在 SIGKILL 时不执行 | 已知限制，由看门狗覆盖 | +| 杀错进程（端口被其他服务占用） | 杀之前检查进程 cmdline 是否包含 `mineru` | + +## Sources + +- **Brainstorm**: [docs/brainstorms/2026-03-18-gpu-cleanup-on-exit-brainstorm.md](../brainstorms/2026-03-18-gpu-cleanup-on-exit-brainstorm.md) diff --git a/docs/plans/2026-03-18-feat-gpu-resource-auto-management-plan.md b/docs/plans/2026-03-18-feat-gpu-resource-auto-management-plan.md new file mode 100644 index 0000000..ce354e6 --- /dev/null +++ b/docs/plans/2026-03-18-feat-gpu-resource-auto-management-plan.md @@ -0,0 +1,354 @@ +--- +title: "feat(backend): GPU 资源自动管理 + MinerU 子进程自治" +type: feat +status: completed +date: 2026-03-18 +origin: docs/brainstorms/2026-03-18-gpu-resource-auto-management-brainstorm.md +--- + +# GPU 资源自动管理 + MinerU 子进程自治 + +## Overview + +当前 Omelette 后端 GPU 模型（Embedding、Reranker、PaddleOCR）加载后永不释放，MinerU 需要手动启动/停止。 +本计划实现三个改进： + +1. **GPU 模型 TTL 自动卸载** — 空闲 5 分钟后自动释放显存 +2. **MinerU 子进程自动管理** — 按需启动/空闲 10 分钟后自动停止 +3. **GPU 监控 API** — 查看模型状态、显存占用、手动卸载 + +## Problem Statement + +实际观察（`nvidia-smi`）： + +| PID | GPU | 显存 | 来源 | +|-----|-----|------|------| +| 3808494 | GPU 6 | 2712 MiB | MinerU 外部进程 | +| 848294 | GPU 6 | 500 MiB | Embedding 模型 | +| 848294 | GPU 7 | 20714 MiB | PaddleOCR + Reranker + CUDA context | + +根因： +- `embedding_service.py` 全局 `_cached_embed_model` 永不释放 +- `reranker_service.py` `@lru_cache(maxsize=1)` 永不清除 +- `ocr_service.py` PaddleOCR 实例无 `close()`、无 `torch.cuda.empty_cache()` +- MinerU 是外部手动进程，无自治能力 + +## Technical Considerations + +### Key Decisions (see brainstorm: docs/brainstorms/2026-03-18-gpu-resource-auto-management-brainstorm.md) + +| 决策 | 选择 | 理由 | +|------|------|------| +| GPU 模型释放策略 | TTL 5 分钟自动释放 | 平衡首次加载延迟（3-5s）和资源节约 | +| MinerU 管理方式 | 子进程 (`subprocess.Popen`) | 不依赖 Docker，使用已有 conda env | +| MinerU TTL | 10 分钟 | 启动慢（30-60s），应保持更久 | +| MinerU conda env | 独立 `mineru` 环境 | 与 omelette 环境隔离，避免依赖冲突 | +| PaddleOCR 处理 | 实例销毁时显式 `torch.cuda.empty_cache()` | 当前无清理机制 | +| 并发加载保护 | `asyncio.Lock` per model | 单用户系统，防重复加载即可 | +| 向后兼容 | 可通过配置禁用 | `model_ttl_seconds=0` 禁用 TTL, `mineru_auto_manage=False` 禁用自动管理 | + +### Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ main.py lifespan │ +│ startup: gpu_model_manager.start() │ +│ mineru_process_manager.start() │ +│ shutdown: gpu_model_manager.stop() │ +│ mineru_process_manager.stop() │ +└──────────────┬──────────────────────┬───────────────┘ + │ │ + ┌──────────▼──────────┐ ┌───────▼────────────────┐ + │ GPUModelManager │ │ MinerUProcessManager │ + │ │ │ │ + │ _models: dict │ │ _process: Popen | None │ + │ _locks: dict │ │ _lock: asyncio.Lock │ + │ _cleanup_task │ │ _cleanup_task │ + │ │ │ │ + │ acquire(name) → │ │ ensure_running() → │ + │ load + touch TTL │ │ start if needed │ + │ release(name) → │ │ touch() → │ + │ touch TTL │ │ update last_used │ + │ unload_all() │ │ stop() │ + │ get_status() │ │ get_status() │ + └──────────┬──────────┘ └───────┬────────────────┘ + │ │ + ┌──────────▼──────────┐ ┌───────▼────────────────┐ + │ embedding_service │ │ mineru_client │ + │ reranker_service │ │ ocr_service │ + │ ocr_service │ │ │ + └─────────────────────┘ └────────────────────────┘ +``` + +### Implementation Phases + +#### Phase 1: GPUModelManager 核心 + config 配置 + +**新增文件**: `backend/app/services/gpu_model_manager.py` + +**任务**: +- [ ] 在 `config.py` 新增 TTL 配置字段：`model_ttl_seconds`（默认 300）、`model_ttl_check_interval`（默认 30） +- [ ] 创建 `GPUModelManager` 类，管理模型生命周期 + - `_models: dict[str, ModelEntry]` 存储模型名 → (model, last_used_at, loader_fn) + - `_locks: dict[str, asyncio.Lock]` 每个模型一把锁 + - `start()` 启动后台清理任务 + - `stop()` 停止清理任务，卸载所有模型 + - `acquire(name, loader_fn)` 获取模型（按需加载 + 更新 TTL） + - `_cleanup_loop()` 定期检查并卸载超时模型 + - `_unload(name)` 卸载单个模型：`del model` + `gc.collect()` + `torch.cuda.empty_cache()` + - `unload_all()` 卸载所有模型 + - `get_status()` 返回每个模型的加载状态 +- [ ] `model_ttl_seconds=0` 时禁用 TTL（模型永不自动卸载，保持现有行为） + +**预估**: ~150 行代码 + +#### Phase 2: 改造 embedding_service + reranker_service + +**修改文件**: +- `backend/app/services/embedding_service.py` +- `backend/app/services/reranker_service.py` + +**任务**: +- [ ] `embedding_service.py`: 移除全局 `_cached_embed_model`，改为通过 `GPUModelManager.acquire("embedding", _build_local_embedding)` 获取模型 +- [ ] `reranker_service.py`: 移除 `@lru_cache`，改为通过 `GPUModelManager.acquire("reranker", _load_reranker_fn)` 获取模型 +- [ ] 保持 `get_embedding_model()` 和 `get_reranker()` 的公共 API 不变，调用方无需修改 +- [ ] `force_reload` 参数保持有效：通过 `GPUModelManager._unload` + 重新 `acquire` 实现 + +**关键约束**: `GPUModelManager` 必须是全局单例，通过 `main.py lifespan` 初始化 + +#### Phase 3: OCRService 显存清理 + +**修改文件**: `backend/app/services/ocr_service.py` + +**任务**: +- [ ] 为 `OCRService` 添加 `close()` 方法，显式释放 PaddleOCR 实例并调用 `torch.cuda.empty_cache()` +- [ ] 为 `OCRService` 添加 `__del__()` 作为安全网，调用 `close()` +- [ ] 在所有创建 `OCRService` 实例的地方确保使用后调用 `close()` 或使用 context manager +- [ ] `ocr_service.py`: 添加 `__enter__` / `__exit__` 支持 `with` 语句 + +**修改调用方**: +- `paper_processor.py`: OCR 完成后调用 `close()` +- `pipelines/nodes.py`: OCR 节点完成后调用 `close()` +- `pipeline_service.py`: OCR 完成后调用 `close()` +- `api/v1/ocr.py`: API 处理完成后调用 `close()` + +#### Phase 4: MinerUProcessManager + +**新增文件**: `backend/app/services/mineru_process_manager.py` + +**任务**: +- [ ] 在 `config.py` 新增配置字段： + - `mineru_auto_manage: bool = True` + - `mineru_conda_env: str = "mineru"` + - `mineru_ttl_seconds: int = 600` + - `mineru_startup_timeout: int = 120` + - `mineru_gpu_ids: str = ""` +- [ ] 创建 `MinerUProcessManager` 类： + - `_process: subprocess.Popen | None` + - `_lock: asyncio.Lock` + - `_last_used_at: float` + - `start()` 启动后台 TTL 检查任务 + - `stop()` 停止清理任务，终止 MinerU 子进程 + - `ensure_running()` 确保 MinerU 正在运行： + 1. 检查端口是否已被外部进程占用 → 使用已有进程 + 2. 否则启动子进程：`conda run -n {env} python -m mineru.cli.fast_api --host 0.0.0.0 --port {port}` + 3. 设置 `CUDA_VISIBLE_DEVICES` 环境变量 + 4. 轮询 health check 直到可用（超时 120s） + - `touch()` 更新 `_last_used_at` + - `_cleanup_loop()` 定期检查是否超时 + - `_kill_process()` 安全终止：`terminate()` → `wait(10)` → `kill()` + - `get_status()` 返回进程状态 +- [ ] `mineru_auto_manage=False` 时全部跳过，保持现有行为 +- [ ] 失败 fallback：启动失败时记录 warning，自动 fallback 到 pdfplumber + +**预估**: ~180 行代码 + +#### Phase 5: 集成到 OCRService + MinerUClient + +**修改文件**: +- `backend/app/services/ocr_service.py` +- `backend/app/services/mineru_client.py` + +**任务**: +- [ ] `ocr_service.py` `_extract_with_mineru()`: 调用前先 `await mineru_manager.ensure_running()`，调用后 `mineru_manager.touch()` +- [ ] `mineru_client.py`: 移除 `health_check()` 中的硬编码 5s timeout，使用 `settings` 值 +- [ ] 确保 `mineru_auto_manage=False` 时走原有流程（直接 health_check → 失败则跳过） + +#### Phase 6: Lifespan 集成 + +**修改文件**: `backend/app/main.py` + +**任务**: +- [ ] 导入 `GPUModelManager` 和 `MinerUProcessManager` 单例 +- [ ] `lifespan` startup: 调用 `gpu_model_manager.start()` 和 `mineru_process_manager.start()` +- [ ] `lifespan` shutdown: 调用 `gpu_model_manager.stop()` 和 `mineru_process_manager.stop()` +- [ ] shutdown 时确保：卸载所有 GPU 模型 → 停止 MinerU 子进程 → 日志记录 + +#### Phase 7: GPU 监控 API + +**新增文件**: `backend/app/api/v1/gpu.py` + +**任务**: +- [ ] `GET /api/v1/gpu/status` — 返回： + - 已加载模型列表（名称、模型名、设备、最后使用时间、TTL 剩余秒数） + - MinerU 状态（running/stopped/external、PID、端口、TTL 剩余） + - GPU 显存信息（通过 `torch.cuda.mem_get_info()` 获取） +- [ ] `POST /api/v1/gpu/unload` — 立即卸载所有模型并释放显存 +- [ ] 在 `api/v1/__init__.py` 注册 `gpu.router` + +**响应 schema**: + +```python +class ModelStatus(BaseModel): + name: str + model_name: str + loaded: bool + device: str | None + last_used_at: datetime | None + ttl_remaining_seconds: int | None + +class MinerUStatus(BaseModel): + status: Literal["running", "stopped", "external"] + pid: int | None + port: int + last_used_at: datetime | None + ttl_remaining_seconds: int | None + +class GpuMemory(BaseModel): + gpu_id: int + total_mb: int + used_mb: int + free_mb: int + +class GpuStatusResponse(BaseModel): + models: list[ModelStatus] + mineru: MinerUStatus + gpu_memory: list[GpuMemory] +``` + +#### Phase 8: 测试 + +**新增文件**: +- `backend/tests/test_gpu_model_manager.py` +- `backend/tests/test_mineru_process_manager.py` +- `backend/tests/test_gpu_api.py` + +**GPUModelManager 测试**: +- [ ] `test_acquire_loads_model` — 首次获取触发 loader_fn +- [ ] `test_acquire_returns_cached` — 第二次获取返回缓存模型 +- [ ] `test_ttl_expires_unloads` — TTL 过期后模型被卸载（mock 时间） +- [ ] `test_acquire_resets_ttl` — 使用后 TTL 重置 +- [ ] `test_unload_all` — 手动卸载所有模型 +- [ ] `test_concurrent_acquire_single_load` — 并发 acquire 只加载一次 +- [ ] `test_ttl_zero_disables_cleanup` — `model_ttl_seconds=0` 禁用 TTL +- [ ] `test_get_status` — 状态返回正确 + +**MinerUProcessManager 测试**: +- [ ] `test_ensure_running_starts_subprocess` — 第一次调用启动子进程 +- [ ] `test_ensure_running_reuses_existing` — 已运行时不重启 +- [ ] `test_ttl_expires_stops_process` — TTL 过期后停止子进程 +- [ ] `test_ensure_running_detects_external` — 检测到已有外部进程时使用它 +- [ ] `test_start_failure_logs_warning` — 启动失败不抛异常，只记录 warning +- [ ] `test_auto_manage_false_skips` — `mineru_auto_manage=False` 时跳过 +- [ ] `test_stop_kills_subprocess` — stop() 终止子进程 + +**GPU API 测试**: +- [ ] `test_gpu_status_no_models` — 无模型时返回空列表 +- [ ] `test_gpu_status_with_models` — 有模型时返回正确信息 +- [ ] `test_gpu_unload` — POST 调用后所有模型被卸载 +- [ ] `test_gpu_status_mineru_stopped` — MinerU 停止时返回 "stopped" +- [ ] `test_gpu_status_mineru_running` — MinerU 运行时返回 "running" + +**已有测试适配**: +- [ ] `test_embedding.py`: 适配新的模型获取方式 +- [ ] `test_reranker_service.py`: 适配移除 `lru_cache` 后的行为 +- [ ] `test_ocr.py`: 适配 `close()` 方法 + +## System-Wide Impact + +### Interaction Graph + +1. API 请求 → `RAGService.search()` → `get_embedding_model()` → `GPUModelManager.acquire("embedding")` → 加载或返回缓存 → 更新 TTL +2. API 请求 → `rerank_nodes()` → `get_reranker()` → `GPUModelManager.acquire("reranker")` → 加载或返回缓存 → 更新 TTL +3. API 请求 → `OCRService.process_pdf_async()` → `_extract_with_mineru()` → `MinerUProcessManager.ensure_running()` → 启动或复用 MinerU → `MinerUClient.parse_pdf()` → 更新 TTL +4. 后台清理任务 → `_cleanup_loop()` → 每 30 秒检查 → TTL 过期 → `_unload()` 释放显存 +5. `lifespan` shutdown → `gpu_model_manager.stop()` → 卸载所有模型 → `mineru_process_manager.stop()` → 终止 MinerU + +### Error & Failure Propagation + +| 场景 | 处理 | +|------|------| +| 模型加载失败（OOM） | `acquire()` 抛出异常，由调用方（service 层）处理 | +| 清理任务异常 | `_cleanup_loop()` 内部 catch，记录 error，继续运行 | +| MinerU 启动超时 | `ensure_running()` 记录 warning，返回 False，fallback 到 pdfplumber | +| MinerU 进程意外退出 | 下次 `ensure_running()` 检测到进程已退出，重新启动 | +| `torch.cuda.empty_cache()` 失败 | catch 异常，记录 warning，继续 | +| shutdown 时模型卸载失败 | 记录 error，继续 shutdown 其他组件 | + +### State Lifecycle Risks + +| 风险 | 缓解措施 | +|------|---------| +| 模型正在使用时被 TTL 卸载 | `acquire()` 返回模型引用，即使 manager 卸载了缓存，已持有的引用仍有效；下次 `acquire` 重新加载 | +| 并发 `acquire` + `_unload` 竞态 | 每个模型一把 `asyncio.Lock`，`acquire` 和 `_unload` 都需持锁 | +| MinerU 进程残留（shutdown 失败） | `_kill_process()` 先 `terminate()`，超时后 `kill()`；`atexit` 注册最后防线 | +| PaddleOCR 显存泄漏 | `OCRService.close()` 显式 `del self._paddle_ocr` + `torch.cuda.empty_cache()` | + +## Acceptance Criteria + +### Functional Requirements + +- [ ] GPU 模型（Embedding、Reranker）在无 API 调用 5 分钟后自动从显存中卸载 +- [ ] MinerU 在需要时自动启动，空闲 10 分钟后自动停止 +- [ ] `GET /api/v1/gpu/status` 返回正确的模型状态和显存信息 +- [ ] `POST /api/v1/gpu/unload` 立即卸载所有模型 +- [ ] `model_ttl_seconds=0` 禁用自动卸载（向后兼容） +- [ ] `mineru_auto_manage=False` 禁用 MinerU 自动管理（向后兼容） +- [ ] 应用退出时所有 GPU 模型被卸载、MinerU 子进程被终止 + +### Non-Functional Requirements + +- [ ] TTL 检查间隔 30 秒，对 event loop 性能影响可忽略 +- [ ] 模型重新加载延迟：Embedding ~3-5s, Reranker ~3-5s +- [ ] MinerU 冷启动延迟 ~30-60s（首次请求可能感知延迟） +- [ ] 无 API 调用时 GPU 显存占用接近 0 + +### Quality Gates + +- [ ] 所有新代码通过 `ruff check` + `ruff format` +- [ ] 新增 ≥ 20 个单元测试 +- [ ] 已有测试全部通过（~498 个） +- [ ] 手动验证：启动 → API 调用 → 等待 TTL → 确认 `nvidia-smi` 显存释放 + +## Dependencies & Prerequisites + +- Python `asyncio` — 后台任务管理 +- `subprocess.Popen` — MinerU 进程管理 +- `torch.cuda` — 显存查询和清理（可选依赖，无 torch 时降级） +- `conda` CLI — MinerU 环境管理（需要 conda 安装且 `mineru` 环境存在） + +## Risk Analysis & Mitigation + +| 风险 | 可能性 | 影响 | 缓解 | +|------|--------|------|------| +| TTL 过短导致频繁重新加载 | 中 | 性能下降 | 默认 5 分钟，可配置 `model_ttl_seconds` | +| MinerU 子进程启动失败 | 低 | PDF 解析降级 | fallback 到 pdfplumber，记录 warning | +| 并发请求在模型卸载后重新加载延迟 | 低 | 首次请求延迟 3-5s | 使用 Lock 确保只加载一次 | +| CUDA context 即使 `empty_cache` 后仍占用少量显存 | 中 | ~100-200 MiB 残留 | 可接受，相比现状（20 GB）大幅改善 | + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-03-18-gpu-resource-auto-management-brainstorm.md](docs/brainstorms/2026-03-18-gpu-resource-auto-management-brainstorm.md) + - 关键决策：TTL 5 分钟、MinerU 子进程管理、独立 conda 环境 + +### Internal References + +- `backend/app/services/embedding_service.py` — 全局 `_cached_embed_model` 和 `_cleanup_gpu_memory()` +- `backend/app/services/reranker_service.py` — `@lru_cache(maxsize=1)` 和 `_load_reranker()` +- `backend/app/services/ocr_service.py` — PaddleOCR 实例缓存 +- `backend/app/services/mineru_client.py` — MinerU HTTP 客户端 +- `backend/app/config.py:152-159` — 现有 GPU 配置 +- `backend/app/main.py:25-33` — 现有 lifespan（仅 init_db） +- `docs/solutions/deployment/mineru-setup-guide.md` — MinerU 部署指南 diff --git a/docs/plans/2026-03-18-feat-gpu-scheduling-modes-plan.md b/docs/plans/2026-03-18-feat-gpu-scheduling-modes-plan.md new file mode 100644 index 0000000..dc9a177 --- /dev/null +++ b/docs/plans/2026-03-18-feat-gpu-scheduling-modes-plan.md @@ -0,0 +1,147 @@ +--- +title: "feat: GPU 资源调度模式化配置" +type: feat +status: active +date: 2026-03-18 +origin: docs/brainstorms/2026-03-18-gpu-scheduling-modes-brainstorm.md +--- + +# GPU 资源调度模式化配置 + +## Overview + +为 Omelette 后端引入 `GPU_MODE` 全局预设 + 按服务细粒度覆盖的 GPU 资源调度系统。三档预设（conservative / balanced / aggressive）一键控制所有 GPU 相关参数，同时允许用户通过 `.env` 按需覆盖任意参数。 + +## Problem Statement + +当前 GPU 参数分散在多个服务中，部分硬编码（embed_batch_size=8、reranker top_n=50），无法适配不同显存环境： +- 双卡 32G 环境下默认值可能 OOM +- 大显存环境下默认值太保守，吞吐量不足 +- 用户需要手动定位并修改多处代码才能调整参数 + +## Proposed Solution + +（see brainstorm: `docs/brainstorms/2026-03-18-gpu-scheduling-modes-brainstorm.md`） + +1. 新增 `GPU_MODE` 配置项，三档预设自动填充所有 GPU 相关参数 +2. 新增 6 个细粒度配置项，用户可按需覆盖预设 +3. GPU 固定 pin 功能：允许将 Embedding/Reranker/OCR 固定到指定 GPU +4. 优先级：用户显式设置 > GPU_MODE 预设 > 硬编码默认值 + +## Technical Approach + +### Phase 1: 配置层（config.py） + +- [ ] 1.1 定义 `GpuMode` 枚举（conservative / balanced / aggressive） +- [ ] 1.2 新增配置项到 `Settings` 类： + +```python +# backend/app/config.py +class GpuMode(str, Enum): + CONSERVATIVE = "conservative" + BALANCED = "balanced" + AGGRESSIVE = "aggressive" + +# 新增字段 +gpu_mode: GpuMode = Field(default=GpuMode.BALANCED) +embed_batch_size: int = Field(default=0, ge=0, le=128, description="0=follow GPU_MODE") +rerank_batch_size: int = Field(default=0, ge=0, le=128, description="0=follow GPU_MODE") +embed_gpu_id: int = Field(default=-1, ge=-1, le=15, description="-1=auto select") +rerank_gpu_id: int = Field(default=-1, ge=-1, le=15, description="-1=auto select") +ocr_gpu_ids: str = Field(default="", description="Comma-separated GPU IDs for OCR, empty=all") +``` + +- [ ] 1.3 实现 `resolve_gpu_param()` 辅助函数： + +```python +# backend/app/config.py + +GPU_MODE_PRESETS = { + GpuMode.CONSERVATIVE: { + "ocr_parallel_limit": 1, + "embed_batch_size": 1, + "rerank_batch_size": 1, + "reranker_concurrency_limit": 1, + }, + GpuMode.BALANCED: { + "ocr_parallel_limit": 0, # auto + "embed_batch_size": 8, + "rerank_batch_size": 16, + "reranker_concurrency_limit": 1, + }, + GpuMode.AGGRESSIVE: { + "ocr_parallel_limit": 0, # auto * 2 handled in resolver + "embed_batch_size": 32, + "rerank_batch_size": 50, + "reranker_concurrency_limit": 2, + }, +} +``` + +- [ ] 1.4 实现 `model_post_init` 或 `@model_validator` 在配置加载后解析参数：当 `embed_batch_size == 0` 时从预设填充 + +### Phase 2: 服务层适配 + +- [ ] 2.1 **embedding_service.py**： + - `_build_local_embedding` 读取 `settings.embed_batch_size`（已解析）替代硬编码 + - `_pick_best_gpu` 在 `settings.embed_gpu_id >= 0` 时短路返回 `cuda:N` + +- [ ] 2.2 **reranker_service.py**： + - `_load_reranker` 读取 `settings.rerank_batch_size` 替代硬编码 `top_n=50` + - GPU 选择同样支持 `settings.rerank_gpu_id` pin + +- [ ] 2.3 **paper_processor.py**： + - OCR GPU 轮转范围支持 `settings.ocr_gpu_ids` 限制 + - `_resolve_parallel_limit` 在 aggressive 模式下倍增 + +### Phase 3: 配置文件更新 + +- [ ] 3.1 更新 `.env.example`：新增所有配置项及注释说明 +- [ ] 3.2 更新 `.env`：设置 `GPU_MODE=conservative`（当前调试阶段） + +### Phase 4: 测试 + +- [ ] 4.1 单元测试：`test_gpu_mode_config.py` + - 三种模式的参数解析正确性 + - 用户覆盖优先于模式预设 + - 边界值（gpu_id=-1, batch_size=0） +- [ ] 4.2 集成测试：验证各服务读取解析后的配置 +- [ ] 4.3 E2E 测试：`CUDA_VISIBLE_DEVICES=6,7 GPU_MODE=conservative` 下全套 E2E 通过 + +### Phase 5: 收尾 + +- [ ] 5.1 ruff lint + format +- [ ] 5.2 回归测试（394+ tests） +- [ ] 5.3 提交代码 + +## Acceptance Criteria + +- [ ] `GPU_MODE=conservative` 时 `CUDA_VISIBLE_DEVICES=6,7` 全套 E2E 不 OOM +- [ ] `GPU_MODE=balanced` 时行为与当前默认一致（向后兼容） +- [ ] 所有 6 个新配置项可通过 `.env` 设置 +- [ ] 用户显式设置的参数优先于模式预设 +- [ ] `EMBED_GPU_ID=0` 可将 embedding 固定到 cuda:0 +- [ ] `OCR_GPU_IDS=1` 可限制 OCR 只用 cuda:1 +- [ ] 394+ 单元测试不回归 +- [ ] 新增 ≥ 10 个配置解析单元测试 + +## System-Wide Impact + +- **Interaction graph**: config.py 解析 → embedding_service / reranker_service / paper_processor 读取 +- **Error propagation**: 无效的 gpu_id 应在配置加载时报错（Pydantic validation），不会传播到运行时 +- **State lifecycle risks**: 无状态变更，纯配置层改动 +- **API surface parity**: 无新 API，仅 `.env` 配置变更 + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-03-18-gpu-scheduling-modes-brainstorm.md](docs/brainstorms/2026-03-18-gpu-scheduling-modes-brainstorm.md) + - Key decisions: 三档预设 + 按服务覆盖, 默认 balanced, .env 为主无运行时切换 + +### Internal References + +- `backend/app/config.py` — 现有 GPU 配置 +- `backend/app/services/embedding_service.py:154` — 硬编码 embed_batch_size=8 +- `backend/app/services/reranker_service.py:42` — 硬编码 top_n=50 +- `backend/app/services/paper_processor.py` — OCR 并行和 GPU 轮转逻辑 diff --git a/docs/plans/2026-03-18-refactor-backend-comprehensive-review-plan.md b/docs/plans/2026-03-18-refactor-backend-comprehensive-review-plan.md new file mode 100644 index 0000000..99d0653 --- /dev/null +++ b/docs/plans/2026-03-18-refactor-backend-comprehensive-review-plan.md @@ -0,0 +1,341 @@ +--- +title: "refactor(backend): 全面代码审核改进 — 48 项修复" +type: refactor +status: completed +date: 2026-03-18 +origin: docs/brainstorms/2026-03-18-backend-comprehensive-review-brainstorm.md +--- + +# 全面代码审核改进 — 48 项修复 + +## Overview + +基于对整个后端代码的四维度审核（服务层、API 层、配置/数据库/Pipeline、测试），共发现 48 项改进点（P0: 10, P1: 18, P2: 20）。本计划将修复全部问题，包含 1 个真实 Bug（ResolvedConflict 字段不匹配导致数据丢失）、Pipeline checkpoint 持久化、Paper 唯一约束迁移等。 + +## Problem Statement + +详见 brainstorm: [docs/brainstorms/2026-03-18-backend-comprehensive-review-brainstorm.md](../brainstorms/2026-03-18-backend-comprehensive-review-brainstorm.md) + +核心问题： +1. **真实 Bug**: `keep_new` pipeline action 因字段不匹配导致论文丢失 +2. **安全隐患**: SSRF、未校验 project_id、GPU unload 无保护 +3. **性能**: 3 处异步上下文中的阻塞调用 +4. **资源泄漏**: fitz 文件句柄、Pipeline 全局状态 +5. **代码质量**: 重复逻辑、不统一的输入验证、缺失测试 + +## Technical Considerations + +### Key Decisions (see brainstorm) + +| 决策 | 选择 | 理由 | +|------|------|------| +| 修复范围 | 全部 48 项 | 一次性全面提升 | +| Pipeline checkpoint | MemorySaver → AsyncSqliteSaver | 重启后状态不丢失 | +| Paper 唯一约束 | 添加 + Alembic 迁移 | 防止重复导入 | + +### Implementation Phases + +#### Phase 1: P0 Bug 修复 + 安全漏洞 + +**目标**: 修复影响运行时正确性和安全的 10 个高优先级问题 + +**任务**: + +##### 1.1 ResolvedConflict 字段不匹配 Bug + +- [ ] `app/api/v1/pipelines.py`: 在 `ResolvedConflict` 模型中添加 `new_paper: dict | None = None` 字段 +- [ ] `app/pipelines/nodes.py`: `apply_resolution_node` 中同时支持 `new_paper` 和 `merged_paper` 字段（向后兼容） +- [ ] 添加测试：verify `keep_new` action 在 pipeline 中正确保留论文 + +##### 1.2 Pipeline 内存泄漏修复 + +- [ ] `app/api/v1/pipelines.py`: 在 `_run_pipeline` 完成/失败/取消时清理 `_cancelled[thread_id]` 和 `_running_tasks[thread_id]` +- [ ] 将 `_cancelled` dict 抽离到 `app/pipelines/cancellation.py` 共享模块，消除 nodes.py 对 API 层的反向依赖 +- [ ] 添加测试：verify task cleanup + +##### 1.3 异步阻塞调用修复 + +- [ ] `app/services/crawler_service.py:86`: `validate_url_safe(url)` → `await asyncio.to_thread(validate_url_safe, url)` +- [ ] `app/services/crawler_service.py:104`: 同上处理 `validate_url_safe(pdf_url)` +- [ ] `app/services/mineru_process_manager.py`: `_kill_process()` 中 `process.wait()` → `await asyncio.to_thread(...)` 或使用 `asyncio.create_subprocess_exec` +- [ ] `app/services/mineru_process_manager.py:193`: `process.stderr.read()` → `await asyncio.to_thread(...)` + +##### 1.4 资源泄漏修复 + +- [ ] `app/services/ocr_service.py:177-187`: `fitz.open(pdf_path)` 改为 `with fitz.open(pdf_path) as pdf_doc:` 确保异常时关闭 + +##### 1.5 安全漏洞修复 + +- [ ] `app/services/subscription_service.py`: `check_rss_feed` 中对 `feed_url` 调用 `validate_url_safe()` 做 SSRF 校验 +- [ ] `app/api/v1/rag.py`: 为 `rag_query`、`index_stats`、`delete_index`、`build_index_stream` 添加 `Depends(get_project)` 依赖 +- [ ] `app/api/v1/subscription.py`: 为 `check_rss`、`check_updates` 添加 `Depends(get_project)` 依赖 +- [ ] `app/api/v1/search.py`: 为 `list_search_sources` 添加 `Depends(get_project)` 依赖 +- [ ] `app/api/v1/gpu.py`: 为 `gpu_unload` 添加 API key 校验（复用现有 `ApiKeyMiddleware` 逻辑） + +**参考文件**: +- 现有 `Depends(get_project)` 用法: `app/api/v1/papers.py` +- SSRF 校验: `app/services/url_validator.py` +- API key 中间件: `app/main.py` + +**预估**: ~200 行代码修改 + ~100 行测试 + +--- + +#### Phase 2: 数据完整性 + Pipeline 持久化 + +**目标**: 添加 Paper 唯一约束 + Pipeline checkpoint 持久化 + +**任务**: + +##### 2.1 Paper (project_id, doi) 唯一约束 + +- [ ] `app/models.py`: 在 Paper 模型中添加 `UniqueConstraint("project_id", "doi", name="uq_paper_project_doi")` 条件约束（SQLite 不支持部分索引，使用 `Index` + 应用层校验） +- [ ] 创建 Alembic 迁移: `alembic revision --autogenerate -m "add paper project_doi unique constraint"` +- [ ] 验证迁移在已有数据上的安全性（处理重复数据场景） +- [ ] 添加测试：verify 同 project 下重复 DOI 被拒绝 + +##### 2.2 Pipeline Checkpoint 持久化 + +- [ ] `backend/pyproject.toml`: 确认 `langgraph[sqlite]` 依赖已安装（或添加） +- [ ] `app/pipelines/graphs.py`: 将 `MemorySaver()` 替换为 `AsyncSqliteSaver` (使用 `settings.data_dir / "pipeline_checkpoints.db"`) +- [ ] `app/config.py`: 添加 `pipeline_checkpoint_db: str` 配置项 +- [ ] `app/main.py` lifespan: 初始化 checkpoint DB 连接 +- [ ] 添加测试：verify pipeline 中断后恢复 + +**参考文件**: +- 现有 pipeline 创建: `app/pipelines/graphs.py` +- 数据库迁移: `alembic/versions/` + +**预估**: ~100 行代码 + 1 个 Alembic 迁移 + +--- + +#### Phase 3: 代码质量重构 + +**目标**: 消除重复、修复配置问题、改善代码结构 + +**任务**: + +##### 3.1 GPU 内存清理逻辑复用 + +- [ ] 新建 `app/services/gpu_utils.py`，抽取公共函数 `release_gpu_memory(logger_name: str) -> None` +- [ ] 修改 `embedding_service.py:_cleanup_gpu_memory()` → 调用 `release_gpu_memory()` +- [ ] 修改 `ocr_service.py:close()` → 调用 `release_gpu_memory()` +- [ ] 修改 `gpu_model_manager.py:_do_unload()` → 调用 `release_gpu_memory()` + +##### 3.2 OCR 调用一致性 + +- [ ] `app/services/pipeline_service.py:79-80`: 将 `asyncio.to_thread(ocr.process_pdf, ...)` 改为 `await ocr.process_pdf_async(...)` 以支持 MinerU 优先路径 + +##### 3.3 LLM 配置回退修复 + +- [ ] `app/services/llm_config_resolver.py:101-103`: temperature/max_tokens 从 `merged_settings` 获取而非 `settings` + +##### 3.4 OCR 临时路径修复 + +- [ ] `app/services/ocr_service.py:182`: 将 `/tmp/omelette_ocr_page_{page_num}.png` 改为 `tempfile.gettempdir() / f"omelette_ocr_page_{page_num}.png"` + +##### 3.5 Embedding service lambda 清理 + +- [ ] `app/services/embedding_service.py:124,127`: 将 `lambda` + `noqa: E731` 改为显式辅助函数 + +##### 3.6 应用生命周期完善 + +- [ ] `app/main.py`: shutdown 时添加 `await engine.dispose()` + +**预估**: ~80 行代码修改 + +--- + +#### Phase 4: 输入验证 + API 一致性 + +**目标**: 统一参数验证、强化输入校验 + +**任务**: + +##### 4.1 统一分页参数 + +- [ ] `app/schemas/common.py`（新建或使用现有）: 定义 `PaginationParams` 类 (`page: int = Query(1, ge=1)`, `page_size: int = Query(20, ge=1, le=100)`) +- [ ] 修改 `projects.py`、`papers.py`、`keywords.py`、`subscription.py`、`dedup.py` 使用统一 `PaginationParams` + +##### 4.2 Literal 类型约束 + +- [ ] `app/api/v1/dedup.py`: `strategy` 参数改为 `Literal["full", "doi_only", "title_only"]` +- [ ] `app/api/v1/crawler.py`: `priority` 参数改为 `Literal["high", "low"]` + +##### 4.3 Search 请求体建模 + +- [ ] `app/schemas/search.py`（或内联）: 定义 `SearchExecuteRequest` Pydantic 模型 +- [ ] `app/api/v1/search.py`: 将 `execute_search` 参数改为使用该模型 + +##### 4.4 Import 数据模型化 + +- [ ] `app/schemas/project.py`（或内联）: 为 `import_project` 的 papers、keywords、subscriptions 定义 Pydantic 模型 +- [ ] `app/api/v1/projects.py`: 使用新模型替代 `list[dict]` 解包 + +**预估**: ~120 行代码修改 + +--- + +#### Phase 5: 测试补充 + +**目标**: 补充缺失的关键测试，统一 fixture + +**任务**: + +##### 5.1 pdf_metadata 服务测试 + +- [ ] 新建 `tests/test_pdf_metadata.py`: 测试 `extract_metadata()` + - 正常 PDF 提取标题、DOI、作者 + - 损坏 PDF 返回 fallback + - Crossref 元数据查询成功/失败 + - DOI 清洗和年份解析 + +##### 5.2 Papers API 测试 + +- [ ] 新建或扩展 `tests/test_api_papers_extended.py`: + - `POST /papers/bulk` 批量导入 + - `GET /papers/{id}/chunks` 论文分块 + - 错误路径: 无效 PDF、不存在的 paper_id + +##### 5.3 Fixture 统一 + +- [ ] 将 `setup_db`、`client`、`project_id`、`minimal_pdf_bytes` 从各测试文件迁移到 `conftest.py` +- [ ] 移除各文件中的重复定义 + +##### 5.4 错误路径测试补充 + +- [ ] OCR: PDF 损坏、超时场景 +- [ ] Search: 网络超时、部分源失败 +- [ ] Subscription: RSS 解析失败、无效 URL + +**预估**: ~400 行新测试 + +--- + +#### Phase 6: P2 改进 + +**目标**: 文档完善、一致性、限流等低优先级改进 + +**任务**: + +##### 6.1 OpenAPI 文档 + +- [ ] 为所有 endpoint 添加 `summary` 和 `description` +- [ ] 统一 Tags（`rewrite` 从 `["rewrite"]` 改为 `["chat"]`） +- [ ] 为 streaming 接口添加 `responses` 描述 + +##### 6.2 SSE 错误格式统一 + +- [ ] 定义统一 SSE 错误格式: `event: error\ndata: {"code": xxx, "message": "..."}` +- [ ] 修改 `chat.py`、`rewrite.py`、`rag.py`、`writing.py` 使用统一格式 + +##### 6.3 test_connection 状态码修复 + +- [ ] `app/api/v1/settings_api.py`: 异常时返回 `JSONResponse(status_code=500)` 而非 `ApiResponse(code=500)` + HTTP 200 + +##### 6.4 限流扩展 + +- [ ] 为 `upload`、`search/execute`、`dedup/run`、`writing` 添加限流（复用现有 slowapi 机制） + +##### 6.5 列表接口分页 + +- [ ] `subscription.py:list_subscriptions` 添加分页 +- [ ] `dedup.py:list_dedup_candidates` 添加分页 + +##### 6.6 其他小改进 + +- [ ] `citation_graph_service.py`: 中文错误信息提取到常量或 i18n key +- [ ] `pdf_metadata.py`: Crossref User-Agent fallback 邮箱移到 config +- [ ] `reranker_service.py`: 注释说明 `top_n` 和 `batch_size` 关系 +- [ ] `config.py`: `cuda_visible_devices` 默认值改为 `""` 或 `"0"` +- [ ] `models.py`: 为 Keyword `parent_id` 添加索引 +- [ ] `test_new_features.py`: 改用 `monkeypatch` 替代直接修改全局状态 + +**预估**: ~300 行代码修改 + +--- + +#### Phase 7: Lint + 全量测试 + 提交 + +- [ ] `ruff check` + `ruff format` +- [ ] 运行全量 mock 测试 +- [ ] 运行 real_llm 测试（`LLM_PROVIDER=volcengine`） +- [ ] 运行 E2E live server 测试 +- [ ] 提交所有更改 + +## System-Wide Impact + +### Interaction Graph + +1. Phase 1.1 (ResolvedConflict): Pipeline HITL resume → `apply_resolution_node` → 论文保留/丢弃 +2. Phase 1.5 (SSRF): API request → `subscription_service.check_rss_feed` → `httpx.get(feed_url)` → 外部网络 +3. Phase 2.2 (Checkpoint): Pipeline start → `AsyncSqliteSaver.aput()` → SQLite → Pipeline resume → `AsyncSqliteSaver.aget()` + +### Error & Failure Propagation + +| 场景 | 处理 | +|------|------| +| 唯一约束冲突（重复 DOI） | `IntegrityError` → 返回 409 Conflict | +| AsyncSqliteSaver 连接失败 | 降级回 MemorySaver + warning | +| SSRF 校验阻塞（DNS 慢） | `asyncio.to_thread` 包装，不影响 event loop | + +### State Lifecycle Risks + +| 风险 | 缓解 | +|------|------| +| Alembic 迁移在有重复 DOI 数据时失败 | 迁移前先检测并清理重复数据 | +| AsyncSqliteSaver checkpoint 文件损坏 | SQLite WAL 模式 + 定期清理旧 checkpoint | + +## Acceptance Criteria + +### Functional Requirements + +- [ ] `keep_new` pipeline action 正确保留论文（修复前为空 dict） +- [ ] Pipeline 重启后可恢复中断的任务 +- [ ] 同一项目下重复 DOI 的 Paper 被拒绝 +- [ ] SSRF 攻击向量被阻断 +- [ ] RAG/subscription/search API 校验 project 存在性 +- [ ] 异步上下文中无阻塞调用 + +### Non-Functional Requirements + +- [ ] 所有 endpoint 有 OpenAPI summary/description +- [ ] 统一的分页、限流、SSE 错误格式 +- [ ] pdf_metadata、bulk_import、list_paper_chunks 有测试覆盖 + +### Quality Gates + +- [ ] `ruff check` + `ruff format` 通过 +- [ ] 全量 mock 测试通过 +- [ ] Real LLM 测试通过 +- [ ] E2E live server 测试通过 + +## Dependencies & Prerequisites + +- `langgraph[sqlite]` — AsyncSqliteSaver 所需 +- Alembic — Paper 唯一约束迁移 +- 现有 `slowapi` — 限流扩展 + +## Risk Analysis & Mitigation + +| 风险 | 可能性 | 影响 | 缓解 | +|------|--------|------|------| +| Alembic 迁移在已有重复数据上失败 | 中 | 部署阻断 | 迁移脚本先清理重复 | +| AsyncSqliteSaver API 与 MemorySaver 不兼容 | 低 | Pipeline 创建失败 | 查阅 LangGraph 文档确认 API | +| SSRF 校验误判合法 URL | 低 | 功能降级 | 白名单机制 | + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-03-18-backend-comprehensive-review-brainstorm.md](../brainstorms/2026-03-18-backend-comprehensive-review-brainstorm.md) + - 关键决策：全部 48 项修复、Pipeline 持久化、Paper 唯一约束 + +### Internal References + +- 现有 `Depends(get_project)`: `app/api/v1/papers.py` +- SSRF 校验: `app/services/url_validator.py` +- Pipeline 图: `app/pipelines/graphs.py` +- 数据库模型: `app/models.py` +- Alembic 配置: `alembic.ini`, `alembic/env.py` diff --git a/docs/plans/2026-03-18-refactor-backend-deep-audit-improvements-plan.md b/docs/plans/2026-03-18-refactor-backend-deep-audit-improvements-plan.md new file mode 100644 index 0000000..55c3754 --- /dev/null +++ b/docs/plans/2026-03-18-refactor-backend-deep-audit-improvements-plan.md @@ -0,0 +1,887 @@ +--- +title: "refactor: 后端深度审计改进 — 21 项全面优化" +type: refactor +status: completed +date: 2026-03-18 +origin: docs/brainstorms/2026-03-18-backend-deep-audit-brainstorm.md +--- + +# refactor: 后端深度审计改进 — 21 项全面优化 + +## Overview + +对 Omelette 后端进行全面质量提升，覆盖安全加固、Bug 修复、性能优化、API 补全、MCP 扩展和 WebSocket 引入。共 21 项改进，分 5 个实施阶段依次交付。 + +## Problem Statement + +经过对全部 17 个路由文件（76+ 端点）、18 个服务模块、10 个数据模型、12 个 Schema 文件、MCP Server 和 Pipeline 系统的深度审计，发现以下核心问题： + +1. **安全隐患** — SSRF 风险、DOI 注入、API Key 暴露 +2. **运行时 Bug** — OCR 阻塞事件循环、Pipeline 取消无效、Task 引用丢失 +3. **接口一致性** — 错误响应格式不统一、输入验证缺失 +4. **功能缺失** — 批量操作、Pipeline 列表、MCP 工具不全 +5. **架构短板** — Pipeline 状态仅存内存、无 WebSocket 支持 + +## Proposed Solution + +分 5 个 Phase 实施，每个 Phase 独立可交付、可测试： + +| Phase | 名称 | 改进项 | 关键交付 | +|-------|------|--------|---------| +| 1 | 紧急修复 | P0 #1-4 | OCR async、Pipeline 取消、SSRF 防护、Task 引用 | +| 2 | 接口规范化 | P1 #5-7 | 错误格式统一、Schema 验证、DOI 校验 | +| 3 | API 补全 | P1 #8-10 + P2 部分 | Pipeline 持久化/列表、批量删除、复合索引 | +| 4 | MCP 与中间件 | P2 #11-16 | MCP 工具扩展、Rate Limit、Subscription 增强 | +| 5 | WebSocket 与收尾 | P3 #17-21 | WebSocket、项目导出/导入、CORS、测试 | + +--- + +## Technical Approach + +### Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ main.py │ +│ ┌─────────┐ ┌──────────┐ ┌──────────────────────┐ │ +│ │ HTTPExc │ │ ValidErr │ │ Global Exception │ │ +│ │ Handler │ │ Handler │ │ Handler (existing) │ │ +│ └─────────┘ └──────────┘ └──────────────────────┘ │ +│ ┌──────────────────────────────────────────────┐ │ +│ │ ApiKeyMiddleware │ CORSMiddleware │ RateLimit│ │ +│ └──────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────┐ │ +│ │ API Router /api/v1 │ │ +│ │ ┌────────┐ ┌──────────┐ ┌────────────────┐ │ │ +│ │ │papers │ │pipelines │ │ ws/pipelines/* │ │ │ +│ │ │(batch) │ │(list,ws) │ │ (WebSocket) │ │ │ +│ │ └────────┘ └──────────┘ └────────────────┘ │ │ +│ └──────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────┐ │ +│ │ Services │ │ +│ │ ┌─────────┐ ┌──────────────┐ ┌───────────┐ │ │ +│ │ │url_safe │ │pipeline_mgr │ │ws_manager │ │ │ +│ │ │(SSRF) │ │(Task persist)│ │(rooms) │ │ │ +│ │ └─────────┘ └──────────────┘ └───────────┘ │ │ +│ └──────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────┐ │ +│ │ MCP Server (expanded tools) │ │ +│ └──────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────┘ +``` + +### Implementation Phases + +--- + +#### Phase 1: 紧急修复 (P0) + +**目标**: 修复影响安全和稳定性的 4 个关键问题。 + +##### 1.1 OCR 阻塞事件循环修复 + +**文件**: `backend/app/api/v1/ocr.py`, `backend/app/services/pipeline_service.py` + +```python +# ocr.py — 修复前 +ocr_result = service.process_pdf(paper.pdf_path, force_ocr=force_ocr) + +# ocr.py — 修复后 +ocr_result = await asyncio.to_thread(service.process_pdf, paper.pdf_path, force_ocr=force_ocr) +``` + +**验收标准**: +- [ ] `ocr.py:54` 使用 `asyncio.to_thread()` 包裹 `process_pdf` +- [ ] `pipeline_service.py` 中同理修复 OCR 调用 +- [ ] 并发 OCR 请求不阻塞 chat/RAG 端点 + +##### 1.2 Pipeline 取消机制修复 + +**文件**: `backend/app/api/v1/pipelines.py`, `backend/app/pipelines/nodes.py` + +**方案**: 使用共享的 `_cancelled` 字典，Pipeline 节点在开头检查。 + +```python +# pipelines.py +_cancelled: dict[str, bool] = {} + +@router.post("/{thread_id}/cancel") +async def cancel_pipeline(thread_id: str): + task = _running_tasks.get(thread_id) + if not task: + raise HTTPException(status_code=404, detail="Pipeline not found") + if task["status"] == "completed": + raise HTTPException(status_code=400, detail="Pipeline already completed") + _cancelled[thread_id] = True + task["status"] = "cancelled" + asyncio_task = task.get("asyncio_task") + if asyncio_task and not asyncio_task.done(): + asyncio_task.cancel() + return ApiResponse(data={"thread_id": thread_id, "status": "cancelled"}) +``` + +```python +# nodes.py — 每个节点开头添加 +def _check_cancelled(state: PipelineState) -> bool: + from app.api.v1.pipelines import _cancelled + thread_id = state.get("thread_id", "") + return _cancelled.get(thread_id, False) + +async def search_node(state: PipelineState) -> dict: + if _check_cancelled(state): + return {"stage": "cancelled", "cancelled": True} + # ... 原有逻辑 +``` + +**验收标准**: +- [ ] 取消后 Pipeline 实际停止运行 +- [ ] 取消已完成的 Pipeline 返回 400 +- [ ] 取消后尝试 resume 返回 400 +- [ ] `cancel_pipeline` 同时取消 asyncio Task + +##### 1.3 SSRF 防护 + +**文件**: `backend/app/services/crawler_service.py`, `backend/app/services/url_validator.py` (新建) + +**方案**: 创建 `url_validator.py` 工具模块，在所有 URL 请求前验证。 + +```python +# backend/app/services/url_validator.py +import ipaddress +import socket +from urllib.parse import urlparse + +BLOCKED_HOSTNAMES = frozenset({ + "metadata.google.internal", + "metadata.amazonaws.com", +}) + +def validate_url_safe(url: str) -> str: + """Validate URL is safe for server-side fetch. Raises ValueError if unsafe.""" + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + raise ValueError(f"Unsupported scheme: {parsed.scheme}") + hostname = parsed.hostname + if not hostname: + raise ValueError("Invalid URL: no hostname") + if hostname in BLOCKED_HOSTNAMES: + raise ValueError(f"Blocked hostname: {hostname}") + addrinfos = socket.getaddrinfo(hostname, None) + for info in addrinfos: + ip_str = info[4][0] + ip = ipaddress.ip_address(ip_str) + if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast: + raise ValueError(f"Blocked: {ip_str} resolves to private/reserved address") + return url +``` + +**集成点**: +- `crawler_service.py` — 在 `_download_pdf()` 前调用 +- `mcp_server.py` — 在 Crossref API 调用前验证 DOI 格式 +- `subscription_service.py` — 在 RSS feed URL 请求前调用 + +**验收标准**: +- [ ] 私有 IP / localhost / 元数据地址被拒绝 +- [ ] DOI 格式验证 `^10\.\d{4,9}/[-._;()/:A-Z0-9]+$` (大小写不敏感) +- [ ] Unpaywall / Semantic Scholar 返回的 URL 也经过验证 +- [ ] `ValueError` 被正确捕获并返回友好错误 + +##### 1.4 asyncio.create_task 引用保存 + +**文件**: `backend/app/api/v1/pipelines.py` + +```python +# 修复前 +asyncio.create_task(_run()) + +# 修复后 +task_ref = asyncio.create_task(_run()) +_running_tasks[thread_id]["asyncio_task"] = task_ref +``` + +**验收标准**: +- [ ] 所有 3 处 `create_task` 保存引用到 `_running_tasks` +- [ ] 大量并发 Pipeline 不会被 GC 静默取消 + +##### Phase 1 测试 + +```python +# tests/test_p0_fixes.py + +class TestOCRAsync: + async def test_concurrent_ocr_no_blocking(self, client, setup_db): + """5 concurrent OCR requests shouldn't block other endpoints.""" + +class TestPipelineCancel: + async def test_cancel_stops_pipeline(self, client, setup_db): + """Cancel mid-pipeline → status becomes cancelled, nodes stop.""" + async def test_cancel_completed_returns_400(self, client, setup_db): + """Cancel completed pipeline returns 400.""" + async def test_resume_cancelled_returns_400(self, client, setup_db): + """Resume cancelled pipeline returns 400.""" + +class TestSSRF: + async def test_private_ip_blocked(self): + """pdf_url pointing to 169.254.169.254 raises ValueError.""" + async def test_localhost_blocked(self): + """pdf_url pointing to 127.0.0.1 raises ValueError.""" + async def test_valid_url_passes(self): + """Valid academic URL passes validation.""" + +class TestTaskReference: + async def test_task_stored_in_running_tasks(self, client, setup_db): + """asyncio.Task stored in _running_tasks after pipeline start.""" +``` + +--- + +#### Phase 2: 接口规范化 (P1 前半) + +**目标**: 统一错误响应格式，加强输入验证。 + +##### 2.1 错误处理统一 + +**文件**: `backend/app/main.py` + +```python +from fastapi import HTTPException +from fastapi.exceptions import RequestValidationError + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + return JSONResponse( + status_code=exc.status_code, + content={"code": exc.status_code, "message": exc.detail, "data": None}, + ) + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + return JSONResponse( + status_code=422, + content={"code": 422, "message": "Validation error", "data": exc.errors()}, + ) +``` + +**附加修复**: +- `writing.py:87-91` — 返回 `HTTPException(400)` 而非 200 + `code=400` + +**验收标准**: +- [ ] 所有 HTTPException 返回 `{"code", "message", "data"}` 格式 +- [ ] 所有 422 验证错误返回统一格式，`data` 包含错误详情 +- [ ] 前端无需兼容 `detail` 字段 + +##### 2.2 Schema 输入验证加强 + +**文件**: 多个 schema 文件 + +| 文件 | 修改 | +|------|------| +| `schemas/paper.py` | `PaperCreate`: `title` 加 `max_length=2000`; `abstract` 加 `max_length=50000`; `year` 加 `ge=1800, le=2100`; `citation_count` 加 `ge=0`; `pdf_url` 用 `AnyHttpUrl \| str = ""`; `PaperBulkImport.papers` 加 `max_length=500`; `PaperUpdate.status` 用 `Literal[...]` | +| `schemas/subscription.py` | `frequency` 用 `Literal["daily", "weekly", "monthly"]` | +| `schemas/conversation.py` | `tool_mode` 用 `Literal["qa", "citation_lookup", "review_outline", "gap_analysis"]` | +| `schemas/knowledge_base.py` | `ResolveConflictRequest.action` 用 `Literal["keep_old", "keep_new", "merge", "skip"]`; `NewPaperData.title` 加 `max_length=2000` | +| `schemas/keyword.py` | `KeywordExpandRequest.seed_terms` 加 `max_length=50` | +| `schemas/llm.py` | API key 字段加 `max_length=500` | + +##### 2.3 DOI 格式验证 + +**文件**: `backend/app/services/url_validator.py`（已在 Phase 1 创建，添加 DOI 验证） + +```python +import re + +DOI_PATTERN = re.compile(r"^10\.\d{4,9}/[-._;()/:A-Za-z0-9]+$") + +def validate_doi(doi: str) -> str: + """Validate DOI format. Raises ValueError if invalid.""" + if not DOI_PATTERN.match(doi): + raise ValueError(f"Invalid DOI format: {doi}") + return doi +``` + +**集成点**: `mcp_server.py` 的 `add_paper_by_doi`、`_fetch_crossref_metadata` + +**验收标准**: +- [ ] 全部 15 个 Schema 字段添加约束 +- [ ] 无效枚举值返回 422 +- [ ] 超长字符串被拒绝 +- [ ] DOI 格式不合法返回清晰错误 + +##### Phase 2 测试 + +```python +# tests/test_p1_error_handling.py + +class TestErrorFormat: + async def test_404_returns_api_response(self, client): + resp = await client.get("/api/v1/projects/99999") + assert resp.json()["code"] == 404 + assert "message" in resp.json() + assert "data" in resp.json() + + async def test_422_returns_api_response(self, client): + resp = await client.post("/api/v1/projects", json={}) + assert resp.json()["code"] == 422 + assert resp.json()["data"] # contains error details + +class TestSchemaValidation: + async def test_paper_year_range(self, client, setup_db, project_id): + resp = await client.post(f"/api/v1/projects/{project_id}/papers", + json={"title": "Test", "year": 1000}) + assert resp.status_code == 422 + + async def test_invalid_tool_mode(self, client, setup_db): + resp = await client.post("/api/v1/chat/stream", + json={"message": "test", "tool_mode": "invalid"}) + assert resp.status_code == 422 + +class TestDOIValidation: + async def test_valid_doi_accepted(self): + assert validate_doi("10.1038/nature12373") == "10.1038/nature12373" + + async def test_invalid_doi_rejected(self): + with pytest.raises(ValueError): + validate_doi("not-a-doi") +``` + +--- + +#### Phase 3: API 补全与持久化 (P1 后半 + P2 部分) + +**目标**: 补全缺失接口，Pipeline 状态持久化，数据库索引优化。 + +##### 3.1 Pipeline 状态持久化 + +**文件**: `backend/app/api/v1/pipelines.py` + +**方案**: Pipeline 启动时创建 Task 记录，状态变更同步到 DB。 + +```python +async def start_search_pipeline(body: SearchPipelineRequest, db: AsyncSession = Depends(get_db)): + # ... 现有逻辑 ... + # 创建 Task 记录 + task_record = Task( + project_id=body.project_id, + task_type=TaskType.SEARCH, + status=TaskStatus.RUNNING, + progress=0, + result={"thread_id": thread_id, "pipeline_type": "search"}, + ) + db.add(task_record) + await db.flush() + _running_tasks[thread_id]["task_id"] = task_record.id + # ... 启动 pipeline ... +``` + +##### 3.2 Pipeline 列表端点 + +**文件**: `backend/app/api/v1/pipelines.py` + +```python +@router.get("", response_model=ApiResponse[list[dict]]) +async def list_pipelines( + status: str | None = None, + db: AsyncSession = Depends(get_db), +): + """List all pipelines (running, interrupted, completed, failed).""" + data = [] + for thread_id, task in _running_tasks.items(): + if status and task["status"] != status: + continue + data.append({ + "thread_id": thread_id, + "status": task["status"], + "task_id": task.get("task_id"), + }) + return ApiResponse(data=data) +``` + +##### 3.3 批量删除论文 + +**文件**: `backend/app/api/v1/papers.py`, `backend/app/schemas/paper.py` + +```python +# schemas/paper.py +class PaperBatchDeleteRequest(BaseModel): + paper_ids: list[int] = Field(..., min_length=1, max_length=500) + +# papers.py +@router.post("/{project_id}/papers/batch-delete", response_model=ApiResponse[dict]) +async def batch_delete_papers( + project_id: int, + body: PaperBatchDeleteRequest, + db: AsyncSession = Depends(get_db), + project: Project = Depends(get_project), +): + stmt = select(Paper).where( + Paper.project_id == project_id, + Paper.id.in_(body.paper_ids), + ) + result = await db.execute(stmt) + papers = list(result.scalars().all()) + for paper in papers: + await db.delete(paper) + await db.flush() + return ApiResponse(data={"deleted": len(papers), "requested": len(body.paper_ids)}) +``` + +##### 3.4 数据库复合索引 + +**文件**: `backend/app/models/paper.py`, `backend/app/models/task.py` + +```python +# paper.py — 添加到 Paper 类 +__table_args__ = ( + Index("ix_paper_project_status", "project_id", "status"), +) + +# task.py — 添加到 Task 类 +__table_args__ = ( + Index("ix_task_project_status", "project_id", "status"), +) +``` + +需要创建对应的 Alembic 迁移文件。 + +##### 3.5 ResumeRequest 类型约束 + +**文件**: `backend/app/api/v1/pipelines.py` + +```python +class ResolvedConflict(BaseModel): + conflict_id: str + action: Literal["keep_old", "keep_new", "merge", "skip"] + merged_paper: dict | None = None + +class ResumeRequest(BaseModel): + resolved_conflicts: list[ResolvedConflict] = [] +``` + +**验收标准**: +- [ ] Pipeline 状态写入 Task 表 +- [ ] `GET /pipelines` 返回所有 Pipeline 列表 +- [ ] 批量删除端点正常工作 +- [ ] 复合索引创建并有对应迁移文件 +- [ ] ResumeRequest 有严格类型约束 + +##### Phase 3 测试 + +```python +# tests/test_p1_api_completion.py + +class TestBatchDelete: + async def test_batch_delete_papers(self, client, setup_db, project_id): + """Create 5 papers, batch delete 3.""" + +class TestPipelineList: + async def test_list_pipelines(self, client, setup_db): + """Start pipeline, list shows it running.""" + +class TestPipelinePersistence: + async def test_pipeline_creates_task_record(self, client, setup_db, project_id): + """Starting pipeline creates Task in DB.""" +``` + +--- + +#### Phase 4: MCP 扩展与中间件 (P2) + +**目标**: 扩展 MCP 工具覆盖，优化中间件配置。 + +##### 4.1 MCP 新工具 + +**文件**: `backend/app/mcp_server.py` + +| 工具 | 调用的 Service | 功能 | +|------|---------------|------| +| `summarize_papers` | `WritingService.summarize` | 论文摘要总结 | +| `generate_review_outline` | `WritingService.review_outline` | 综述大纲 | +| `analyze_gaps` | `WritingService.gap_analysis` | 研究空白分析 | +| `manage_keywords` | `KeywordService` | 关键词 CRUD + 扩展 | + +```python +@mcp.tool() +async def summarize_papers(kb_id: int, paper_ids: list[int] | None = None, language: str = "en") -> str: + """Summarize papers in a knowledge base.""" + from app.services.writing_service import WritingService + svc = WritingService() + result = await svc.summarize(project_id=kb_id, paper_ids=paper_ids, language=language) + return f"## Summary\n\n{result.get('content', 'No summary generated.')}" +``` + +##### 4.2 MCP 输入验证 + +```python +@mcp.tool() +async def search_knowledge_base( + query: str, + kb_id: int, + top_k: int = 5, # 添加验证 +) -> str: + if top_k < 1 or top_k > 50: + return "Error: top_k must be between 1 and 50." + # ... +``` + +##### 4.3 Rate Limiting 分端点配置 + +**文件**: `backend/app/api/v1/rag.py`, `backend/app/api/v1/chat.py`, `backend/app/api/v1/upload.py` + +```python +from app.middleware.rate_limit import limiter + +@router.post("/index", response_model=ApiResponse[dict]) +@limiter.limit("5/minute") +async def build_index(...): + ... +``` + +| 端点类别 | 限速 | +|----------|------| +| RAG 建索引 | 5/minute | +| Chat stream | 30/minute | +| PDF 上传 | 10/minute | +| OCR 处理 | 5/minute | +| Pipeline 启动 | 10/minute | +| 其他 | 120/minute (全局默认) | + +##### 4.4 Subscription trigger 自动导入 + +**文件**: `backend/app/api/v1/subscription.py` + +```python +@router.post("/{sub_id}/trigger", response_model=ApiResponse[SubscriptionRunResult]) +async def trigger_subscription( + ... + auto_import: bool = Query(False, description="Auto-import new papers into project"), +): + # ... 现有检查逻辑 ... + if auto_import and new_papers: + for paper_data in new_papers: + paper = Paper(project_id=project_id, **paper_data) + db.add(paper) + await db.flush() + # ... +``` + +##### 4.5 LLM 客户端入口统一 + +**文件**: `backend/app/services/llm_client.py` (删除), `backend/app/api/deps.py` + +- 删除 `services/llm_client.py` shim +- `deps.py` 直接导入 `from app.services.llm.client import get_llm_client` +- 全局搜索替换所有 `from app.services.llm_client import` 引用 + +##### 4.6 Schema 导出完善 + +**文件**: `backend/app/schemas/__init__.py` + +统一导出所有公开 Schema。 + +**验收标准**: +- [ ] MCP 新增 4 个 writing/keyword 工具 +- [ ] MCP 输入参数有边界检查 +- [ ] 重操作端点有独立限速 +- [ ] Subscription trigger 支持 `auto_import` +- [ ] LLM 客户端入口唯一 +- [ ] Schema `__init__.py` 导出完整 + +--- + +#### Phase 5: WebSocket 与收尾 (P3) + +**目标**: 引入 WebSocket 支持 Pipeline 状态推送，完善剩余改进。 + +##### 5.1 WebSocket ConnectionManager + +**文件**: `backend/app/websocket/__init__.py`, `backend/app/websocket/manager.py` (新建) + +```python +# backend/app/websocket/manager.py +import asyncio +import logging +from collections import defaultdict +from fastapi import WebSocket + +logger = logging.getLogger(__name__) + +class PipelineConnectionManager: + def __init__(self): + self.rooms: dict[str, set[WebSocket]] = defaultdict(set) + + async def connect(self, websocket: WebSocket, thread_id: str): + await websocket.accept() + self.rooms[thread_id].add(websocket) + + def disconnect(self, websocket: WebSocket, thread_id: str): + if thread_id in self.rooms: + self.rooms[thread_id].discard(websocket) + if not self.rooms[thread_id]: + del self.rooms[thread_id] + + async def broadcast_to_room(self, thread_id: str, message: dict): + if thread_id not in self.rooms: + return + dead = [] + for conn in list(self.rooms[thread_id]): + try: + await conn.send_json(message) + except Exception: + dead.append(conn) + for conn in dead: + self.rooms[thread_id].discard(conn) + +pipeline_manager = PipelineConnectionManager() +``` + +##### 5.2 WebSocket 端点 + +**文件**: `backend/app/api/v1/pipelines.py` + +```python +from fastapi import WebSocket, WebSocketDisconnect, Query +from app.websocket.manager import pipeline_manager + +@router.websocket("/{thread_id}/ws") +async def pipeline_status_websocket( + websocket: WebSocket, + thread_id: str, + api_key: str | None = Query(default=None), +): + if settings.api_secret_key and api_key != settings.api_secret_key: + await websocket.close(code=4008) + return + await pipeline_manager.connect(websocket, thread_id) + try: + while True: + await websocket.receive_text() + except WebSocketDisconnect: + pipeline_manager.disconnect(websocket, thread_id) +``` + +**Pipeline 节点广播**: 在 `_run()` 中添加状态广播。 + +```python +async def _run(): + try: + result = await pipeline.ainvoke(initial_state, config=config) + # ... 状态更新 ... + await pipeline_manager.broadcast_to_room(thread_id, { + "type": "status", + "status": _running_tasks[thread_id]["status"], + "stage": result.get("stage", ""), + "progress": result.get("progress", 0), + }) + except Exception as e: + await pipeline_manager.broadcast_to_room(thread_id, { + "type": "error", "message": str(e), + }) +``` + +##### 5.3 健康检查路由统一 + +**文件**: `backend/app/main.py`, `backend/app/middleware/auth.py` + +```python +# main.py — 添加根级健康检查 +@app.get("/health") +async def health(): + return ApiResponse(data={"status": "ok"}) +``` + +Auth exempt 保持 `/health` 和 `/api/v1/settings/health` 并存。 + +##### 5.4 CORS 配置完善 + +**文件**: `backend/app/main.py` + +```python +app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origin_list, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + expose_headers=["X-Request-ID", "X-Process-Time"], + max_age=600, +) +``` + +##### 5.5 项目导出/导入 + +**文件**: `backend/app/api/v1/projects.py` + +```python +@router.get("/{project_id}/export", response_model=ApiResponse[dict]) +async def export_project(project_id: int, db: AsyncSession = Depends(get_db), project: Project = Depends(get_project)): + """Export project data as JSON (papers, keywords, subscriptions).""" + +@router.post("/import", response_model=ApiResponse[dict], status_code=201) +async def import_project(data: dict, db: AsyncSession = Depends(get_db)): + """Import a previously exported project.""" +``` + +##### 5.6 中间件专项测试 + +**文件**: `backend/tests/test_middleware.py` (新建) + +```python +class TestAuthMiddleware: + async def test_missing_api_key_returns_401(self, client): + """Request without API key when key is configured returns 401.""" + async def test_exempt_paths_no_auth(self, client): + """Health, docs, MCP paths don't require auth.""" + async def test_valid_api_key_passes(self, client): + """Valid X-API-Key header grants access.""" + async def test_query_param_api_key_removed(self, client): + """Query param api_key is no longer accepted.""" + +class TestRateLimiting: + async def test_rate_limit_exceeded(self, client): + """Exceeding rate limit returns 429.""" +``` + +##### 5.7 API Key 仅 Header 传递 + +**文件**: `backend/app/middleware/auth.py` + +```python +# 修复前 +api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key") + +# 修复后 +api_key = request.headers.get("X-API-Key") +``` + +WebSocket 端点单独处理 query param（浏览器不支持 WS header），但 REST API 仅接受 Header。 + +**验收标准**: +- [ ] WebSocket 端点可接收 Pipeline 状态推送 +- [ ] `/health` 路由存在且无需认证 +- [ ] CORS 配置完整 +- [ ] 项目导出/导入功能可用 +- [ ] 中间件有专项测试 +- [ ] REST API Key 仅通过 Header 传递 + +--- + +## System-Wide Impact + +### Interaction Graph + +1. **Error handler 添加** → 所有 HTTPException 路径变更响应格式 → 前端 `error.response?.data?.message` 始终可用 → 无需 fallback 到 `detail` +2. **Schema 验证加强** → 部分之前接受的请求会被拒绝（422）→ 前端需要处理新的验证错误 +3. **Pipeline 取消** → cancel API → `_cancelled` dict → 节点检查 → asyncio.Task.cancel() → 资源释放 +4. **WebSocket** → Pipeline 启动 → 节点广播 → ConnectionManager → 前端 WS 客户端 + +### Error & Failure Propagation + +| 层级 | 错误类型 | 处理 | +|------|---------|------| +| Schema 验证 | `RequestValidationError` | `validation_exception_handler` → 422 ApiResponse | +| 路由层 | `HTTPException` | `http_exception_handler` → 4xx ApiResponse | +| 服务层 | `ValueError` (URL/DOI) | 端点 try/except → HTTPException(400) | +| 服务层 | 其他 Exception | `global_exception_handler` → 500 ApiResponse | +| WebSocket | `WebSocketDisconnect` | `disconnect()` 清理连接 | + +### State Lifecycle Risks + +| 操作 | 风险 | 缓解 | +|------|------|------| +| Pipeline 取消 mid-crawl | 部分论文已下载 | 状态标记，不删除已下载文件 | +| 批量删除 | Paper chunks 和 ChromaDB 向量不一致 | cascade delete chunks；ChromaDB 需单独清理 | +| 项目导出 mid-write | 导出不完整 | 使用数据库快照/事务 | + +### API Surface Parity + +| 接口 | REST API | MCP Tool | 需要同步 | +|------|---------|----------|---------| +| 论文摘要 | `POST /writing/summarize` | `summarize_papers` (Phase 4) | ✅ | +| 综述大纲 | `POST /writing/review-outline` | `generate_review_outline` (Phase 4) | ✅ | +| 批量删除 | `POST /papers/batch-delete` (Phase 3) | — | 后续 | +| Pipeline 列表 | `GET /pipelines` (Phase 3) | — | 后续 | + +--- + +## Acceptance Criteria + +### Functional Requirements + +- [ ] OCR 端点使用 `asyncio.to_thread()`，不阻塞事件循环 +- [ ] Pipeline 取消后节点停止执行 +- [ ] SSRF 防护拦截私有 IP / localhost / 元数据地址 +- [ ] asyncio.Task 引用保存 +- [ ] HTTPException 和 ValidationError 统一 ApiResponse 格式 +- [ ] 15 个 Schema 字段添加验证约束 +- [ ] DOI 格式验证 +- [ ] Pipeline 状态持久化到 Task 表 +- [ ] 批量删除论文端点 +- [ ] Pipeline 列表端点 +- [ ] MCP 新增 4 个 writing/keyword 工具 +- [ ] Rate Limiting 分端点配置 +- [ ] Subscription trigger 支持 auto_import +- [ ] WebSocket Pipeline 状态推送 +- [ ] 项目导出/导入 +- [ ] 健康检查路由统一 +- [ ] CORS 配置完善 + +### Non-Functional Requirements + +- [ ] 并发 5 个 OCR 请求不影响 chat 响应时间 +- [ ] 所有新增端点有对应测试 +- [ ] SSRF 验证对正常请求增加 < 50ms 延迟 +- [ ] WebSocket 支持 100+ 并发连接 +- [ ] 零破坏性变更（向后兼容） + +### Quality Gates + +- [ ] `ruff check` 和 `ruff format` 通过 +- [ ] `mypy` 无新增错误 +- [ ] 全部现有测试通过 +- [ ] 新增测试 ≥ 30 个 +- [ ] Alembic 迁移文件可正常执行 + +--- + +## Dependencies & Prerequisites + +| 依赖 | 类型 | 用途 | +|------|------|------| +| 无新增外部依赖 (Phase 1-3) | — | SSRF 用 stdlib 实现 | +| `websockets` (已存在) | Python | WebSocket (Phase 5) | + +--- + +## Risk Analysis & Mitigation + +| 风险 | 影响 | 概率 | 缓解 | +|------|------|------|------| +| Schema 加严导致现有请求失败 | 高 | 中 | 宽松的 max_length 值；先部署后端再更新前端 | +| Pipeline 取消中断活跃数据库事务 | 高 | 低 | 节点在安全检查点检查取消状态 | +| SSRF 验证误拦正常 URL | 中 | 低 | 仅检查 IP，不做域名白名单 | +| WebSocket 连接泄露 | 中 | 低 | disconnect 清理 + 定时心跳 | +| Alembic 迁移冲突 | 低 | 低 | Phase 3 单独迁移，不修改现有列 | + +--- + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-03-18-backend-deep-audit-brainstorm.md](docs/brainstorms/2026-03-18-backend-deep-audit-brainstorm.md) — 21 项改进发现、优先级排序、WebSocket/单用户决策 + +### Internal References + +- Async blocking fix: `docs/solutions/performance-issues/blocking-sync-calls-asyncio-to-thread.md` +- HITL pattern: `docs/solutions/integration-issues/langgraph-hitl-interrupt-api-snapshot-next.md` +- Test DB isolation: `docs/solutions/test-failures/test-database-pollution-tempfile-mkdtemp.md` +- Integration testing: `docs/solutions/integration-testing/2026-03-16-fastapi-langgraph-integration-testing-best-practices.md` +- Backend rules: `.cursor/rules/python-backend.mdc` +- Pipeline rules: `.cursor/rules/langgraph-pipelines.mdc` +- MCP rules: `.cursor/rules/mcp-server.mdc` + +### External References + +- FastAPI WebSocket: raw WebSocket with room-based ConnectionManager，保持 SSE 用于 chat/RAG +- SSRF 防护: stdlib `ipaddress` + `socket.getaddrinfo()` 验证，阻止私有/保留/环回地址 +- OWASP SSRF Prevention Cheat Sheet diff --git a/docs/plans/2026-03-18-refactor-backend-quality-testing-gaps-plan.md b/docs/plans/2026-03-18-refactor-backend-quality-testing-gaps-plan.md new file mode 100644 index 0000000..3e22f01 --- /dev/null +++ b/docs/plans/2026-03-18-refactor-backend-quality-testing-gaps-plan.md @@ -0,0 +1,265 @@ +--- +title: "refactor(backend): 代码质量与测试缺口修补" +type: refactor +status: completed +date: 2026-03-18 +origin: docs/brainstorms/2026-03-18-backend-quality-testing-gaps-brainstorm.md +--- + +# refactor(backend): 代码质量与测试缺口修补 + +## Overview + +在完成 21 项后端综合优化后，深度审计发现 4 个领域仍有缺口：代码质量（硬编码值、错误处理不一致、路径安全）、MCP 工具测试（7/11 无测试）、Pipeline 真实 PDF 集成测试（HITL 流程未覆盖）、Chat tool_mode 覆盖不全（仅测了 qa 模式）。 + +## Problem Statement / Motivation + +- **MCP 工具**：11 个工具中 7 个无单元测试，任何改动都可能导致静默回归 +- **Pipeline**：HITL 中断→恢复流程是核心功能，但只有 mock 级别的 graph 测试，无 API 层集成测试 +- **代码质量**：硬编码值散落在多个文件中，`citation_graph_service` 返回 200 + `{"error": ...}` 违反统一的 `ApiResponse` 约定 +- **Chat**：4 种 tool_mode 仅测了 qa，citation_lookup / review_outline / gap_analysis 从未经过测试验证 + +## Proposed Solution + +分 4 个 Phase 实施，每个 Phase 独立可提交： + +1. **Phase 1: 代码质量改进** — 配置提取、错误处理统一、路径安全、缺失服务测试 +2. **Phase 2: MCP 工具测试补全** — 7 个工具的单元测试 +3. **Phase 3: Pipeline 真实 PDF 集成测试** — Upload Pipeline + HITL + 状态查询 +4. **Phase 4: Chat tool_mode 全覆盖** — citation_lookup / review_outline / gap_analysis + +## Technical Considerations + +### Architecture Impacts + +- `citation_graph_service.get_citation_graph()` 从返回 dict 改为抛出 `HTTPException` → `papers.py` 的 `get_citation_graph` 端点需要同步更新 +- 前端 citation graph 调用可能依赖 200 + error body → 需检查前端是否需要同步修改 + +### Key Decisions (see brainstorm: docs/brainstorms/2026-03-18-backend-quality-testing-gaps-brainstorm.md) + +| 决策 | 选择 | 理由 | +|------|------|------| +| Citation graph 错误处理 | 改为 `HTTPException(404/502)` | 符合统一 `ApiResponse` 约定；前端本身应处理非 200 响应 | +| add_paper_by_doi Crossref 失败 | 保持当前行为（用最少元数据创建） | DOI 已知时允许记录跟踪是有价值的 | +| 路径验证 | 使用 `Path.is_relative_to()` 替代 `startswith` | 避免前缀匹配 bug（`/data/omelette_sub` 绕过 `/data/omelette`） | +| S2_MAX_PER_REQUEST | 保持 50（代码现值） | S2 API 默认限制 | +| upload.py MAX_FILE_SIZE_MB | 统一使用 `settings.max_upload_size_mb` | config.py 已有该字段 | +| 版本号 | 提取为 `APP_VERSION` 常量到 config.py | 单一来源 | +| Rewrite 异常处理 | 缩窄为 `httpx.HTTPError`, `ValueError`, `RuntimeError` | `CancelledError` 和 `TimeoutError` 已单独处理 | +| Pipeline 测试 PDF | 使用 `/data0/djx/omelette_pdf_test/` 最小的 PDF | `pytest.mark.skipif` 跳过无数据环境 | + +## Acceptance Criteria + +### Phase 1: 代码质量改进 + +- [ ] **1.1** `config.py` 新增字段：`s2_api_base`, `s2_timeout`, `s2_max_per_request`, `title_similarity_threshold`, `rewrite_timeout`, `app_version` +- [ ] **1.2** `upload.py` 移除 `MAX_FILE_SIZE_MB`，使用 `settings.max_upload_size_mb`；移除 `TITLE_SIMILARITY_THRESHOLD`，使用 `settings.title_similarity_threshold` +- [ ] **1.3** `citation_graph_service.py` 移除 `S2_*` 常量，使用 settings；`get_citation_graph` 对 "paper not found" 抛出 `HTTPException(404)`，对 "S2 未收录" 抛出 `HTTPException(502)` +- [ ] **1.4** `papers.py` 更新 `get_citation_graph` 端点，移除 `ApiResponse` 包装中的 error dict 处理 +- [ ] **1.5** `rewrite.py` 移除 `REWRITE_TIMEOUT`，使用 `settings.rewrite_timeout`；缩窄 `except Exception` 为具体异常类型 +- [ ] **1.6** `main.py` 使用 `settings.app_version` 替代硬编码 `"0.1.0"` +- [ ] **1.7** `pipelines.py` 路径验证使用 `Path.is_relative_to()` 替代 `str.startswith()` +- [ ] **1.8** 新增 `tests/test_llm_config_resolver.py`：测试 `from_env()` 各 provider、`from_merged()` 优先级 +- [ ] **1.9** 新增 `tests/test_reranker_service.py`：mock `SentenceTransformerRerank`，测试 `get_reranker()`、`rerank_nodes()` 正常和降级路径 + +### Phase 2: MCP 工具测试补全 + +- [ ] **2.1** `test_mcp.py` 新增 `test_search_knowledge_base`：mock `RAGService.query`，验证正常返回和 top_k 验证 +- [ ] **2.2** `test_mcp.py` 新增 `test_find_citations`：mock `RAGService.query`，验证引文返回格式 +- [ ] **2.3** `test_mcp.py` 新增 `test_add_paper_by_doi`：mock `_fetch_crossref_metadata`，验证正常添加、重复检测、无效 DOI +- [ ] **2.4** `test_mcp.py` 新增 `test_search_papers_by_keyword`：mock `SearchService.search`，验证正常和 max_results 验证 +- [ ] **2.5** `test_mcp.py` 新增 `test_summarize_papers`：mock `WritingService.summarize` +- [ ] **2.6** `test_mcp.py` 新增 `test_generate_review_outline`：mock `WritingService.generate_review_outline` +- [ ] **2.7** `test_mcp.py` 新增 `test_analyze_gaps`：mock `WritingService.analyze_gaps` +- [ ] **2.8** `test_mcp.py` 新增 `test_manage_keywords`：测试 list / add / delete（DB 操作）、expand（mock `KeywordService`）、无效 action + +### Phase 3: Pipeline 真实 PDF 集成测试 + +- [ ] **3.1** 新增 `tests/test_pipeline_real_pdf.py`，含 `@pytest.mark.skipif` 当 PDF 目录不存在时跳过 +- [ ] **3.2** Upload Pipeline 完整流程：选最小 PDF → 上传 → extract_metadata → 验证论文记录入库 +- [ ] **3.3** HITL 流程测试：准备已有论文 → 上传同名 PDF → dedup 中断 → 解决冲突（skip/keep_new）→ 恢复 +- [ ] **3.4** Pipeline 列表与状态查询：启动 pipeline → GET /pipelines → 验证列表包含该 pipeline +- [ ] **3.5** 路径安全测试：传入含 `..` 的路径 → 验证 400 拒绝 + +### Phase 4: Chat tool_mode 全覆盖 + +- [ ] **4.1** `test_chat_pipeline.py` 新增 `test_stream_citation_lookup_mode`：发送 `tool_mode="citation_lookup"` + `knowledge_base_ids=[kb_id]` → 验证 SSE 事件序列 +- [ ] **4.2** `test_chat_pipeline.py` 新增 `test_stream_review_outline_mode`：发送 `tool_mode="review_outline"` → 验证 SSE 事件序列 +- [ ] **4.3** `test_chat_pipeline.py` 新增 `test_stream_gap_analysis_mode`：发送 `tool_mode="gap_analysis"` → 验证 SSE 事件序列 +- [ ] **4.4** 每个模式验证：`start` → `text-delta`（至少1个）→ `finish` → `[DONE]`，无 `error` 事件 + +## Implementation Details + +### Phase 1 文件变更 + +**`backend/app/config.py`** — 新增字段： + +```python +# Semantic Scholar API +s2_api_base: str = "https://api.semanticscholar.org/graph/v1" +s2_timeout: int = Field(default=15, ge=1, le=60) +s2_max_per_request: int = Field(default=50, ge=1, le=100) + +# Upload +title_similarity_threshold: float = Field(default=0.85, ge=0.0, le=1.0) + +# Rewrite +rewrite_timeout: float = Field(default=30.0, ge=5.0, le=120.0) + +# App version +app_version: str = "0.1.0" +``` + +**`backend/app/services/citation_graph_service.py`** — 改用 settings + HTTPException： + +```python +from fastapi import HTTPException +from app.config import settings + +class CitationGraphService: + async def get_citation_graph(self, paper_id, project_id, ...): + paper = await self._db.get(Paper, paper_id) + if not paper or paper.project_id != project_id: + raise HTTPException(status_code=404, detail="Paper not found") + + s2_id = await self._resolve_s2_id(paper) + if not s2_id: + raise HTTPException( + status_code=502, + detail="无法获取引用数据：Semantic Scholar 未收录此论文" + ) + # ... rest unchanged +``` + +**`backend/app/api/v1/pipelines.py`** — 路径验证： + +```python +allowed_root = _Path(settings.pdf_dir).resolve() +for p in body.pdf_paths: + resolved = _Path(p).resolve() + try: + resolved.relative_to(allowed_root) + except ValueError: + raise HTTPException(status_code=400, detail=f"Path not within allowed directory: {p}") + safe_paths.append(str(resolved)) +``` + +**`backend/tests/test_llm_config_resolver.py`**： + +```python +async def test_from_env_default_mock(): + config = LLMConfigResolver.from_env() + assert config.provider == "mock" + +async def test_from_env_override_provider(): + config = LLMConfigResolver.from_env(provider="volcengine") + assert config.provider == "volcengine" + assert config.model == settings.volcengine_model + +async def test_from_merged_user_overrides(): + merged = MergedSettings(llm_provider="anthropic", ...) + config = LLMConfigResolver.from_merged(merged) + assert config.provider == "anthropic" +``` + +**`backend/tests/test_reranker_service.py`**： + +```python +async def test_rerank_nodes_empty(): + result = await rerank_nodes([], "query", top_n=5) + assert result == [] + +async def test_rerank_nodes_fallback_on_import_error(monkeypatch): + monkeypatch.setattr(..., side_effect=ImportError) + result = await rerank_nodes(mock_nodes, "query", top_n=3) + assert len(result) == 3 # falls back to original order + +async def test_get_reranker_caches(monkeypatch): + # mock _load_reranker, call get_reranker twice, assert loaded once +``` + +### Phase 2 Mock 策略 + +| MCP Tool | Mock Target | 返回值 | +|----------|-------------|--------| +| search_knowledge_base | `RAGService.query` | `{"answer": "...", "sources": [...]}` | +| find_citations | `RAGService.query` | 同上 | +| add_paper_by_doi | `_fetch_crossref_metadata` | `{"title": "...", "authors": [...], ...}` | +| search_papers_by_keyword | `SearchService.search` | `{"papers": [...], "total": N}` | +| summarize_papers | `WritingService.summarize` | `{"content": "..."}` | +| generate_review_outline | `WritingService.generate_review_outline` | `{"outline": "..."}` | +| analyze_gaps | `WritingService.analyze_gaps` | `{"analysis": "..."}` | +| manage_keywords (expand) | `KeywordService.expand_keywords` | `{"expanded_terms": [...]}` | + +### Phase 3 测试结构 + +```python +PDF_TEST_DIR = "/data0/djx/omelette_pdf_test" +PDF_DIR_EXISTS = os.path.isdir(PDF_TEST_DIR) + +pytestmark = pytest.mark.skipif(not PDF_DIR_EXISTS, reason="Test PDF directory not available") + +@pytest.fixture +def smallest_pdf(): + """Find the smallest PDF in the test directory.""" + pdfs = sorted(Path(PDF_TEST_DIR).glob("*.pdf"), key=lambda p: p.stat().st_size) + return str(pdfs[0]) if pdfs else pytest.skip("No PDFs found") +``` + +### Phase 4 测试模式 + +```python +@pytest.mark.parametrize("tool_mode", ["citation_lookup", "review_outline", "gap_analysis"]) +async def test_stream_tool_modes(client, tool_mode): + resp = await client.post( + "/api/v1/chat/stream", + json={"message": "分析这个主题", "knowledge_base_ids": [], "tool_mode": tool_mode}, + ) + assert resp.status_code == 200 + # 解析 SSE 事件序列 + event_types = parse_sse_events(resp.text) + assert "start" in event_types + assert "text-delta" in event_types + assert "finish" in event_types + assert "[DONE]" in event_types + assert "error" not in event_types +``` + +## Dependencies & Risks + +| 风险 | 影响 | 缓解 | +|------|------|------| +| Citation graph 改为 HTTPException 可能影响前端 | 前端 citation graph 面板报错 | 检查前端代码，必要时同步更新 | +| 真实 PDF 测试依赖特定路径 | CI 环境无 PDF 目录 | `pytest.mark.skipif` + 环境变量 `E2E_PDF_DIR` | +| RerankerService mock 可能不够精确 | 测试通过但真实行为不同 | mock SentenceTransformerRerank 的 `postprocess_nodes` | +| MCP WritingService/KeywordService mock | LLM 行为变化可能导致集成问题 | 现有 E2E 测试覆盖真实 LLM 路径 | + +## Success Metrics + +- 全部新测试通过（`pytest tests/ -v`） +- 现有 409+ 测试不回归 +- `ruff check` 零报错 +- MCP 工具测试覆盖从 4/11 提升到 11/11 +- Chat tool_mode 测试覆盖从 1/4 提升到 4/4 + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-03-18-backend-quality-testing-gaps-brainstorm.md](../brainstorms/2026-03-18-backend-quality-testing-gaps-brainstorm.md) + - 关键决策：4 个改进方向、真实 PDF 使用已有测试数据、MCP 使用 mock + +### Internal References + +- Config 模式: `backend/app/config.py` (Settings with Field) +- MCP 测试模式: `backend/tests/test_mcp.py` (setup_db + sample_kb + direct call) +- Pipeline 测试模式: `backend/tests/test_pipelines.py` (monkeypatch + MemorySaver + snapshot.next) +- Chat 测试模式: `backend/tests/test_chat_pipeline.py` (mock_services + SSE parsing) +- 错误处理约定: `backend/app/main.py` (HTTPException + RequestValidationError handlers) +- 路径验证: `backend/app/api/v1/pipelines.py:178-184` + +### Institutional Learnings + +- `docs/solutions/integration-testing/` — AsyncClient + ASGITransport, 不用 TestClient +- `docs/solutions/integration-issues/langgraph-hitl-interrupt-api-snapshot-next.md` — 用 `snapshot.next` 检测 HITL 中断 +- `docs/solutions/test-failures/test-database-pollution-tempfile-mkdtemp.md` — 测试 DB 使用 tempfile diff --git a/docs/research/2026-03-17-rag-retrieval-optimization-best-practices.md b/docs/research/2026-03-17-rag-retrieval-optimization-best-practices.md new file mode 100644 index 0000000..2ed1cc8 --- /dev/null +++ b/docs/research/2026-03-17-rag-retrieval-optimization-best-practices.md @@ -0,0 +1,308 @@ +# RAG Retrieval Optimization Best Practices (2025–2026) + +**Context**: Academic paper retrieval, ChromaDB + HNSW, Qwen3-Embedding-0.6B (768 dim), Qwen3-Reranker-0.6B, LlamaIndex + FastAPI. + +**Date**: 2026-03-17 + +--- + +## 1. Reranking Pipeline Design + +### 1.1 Oversample → Rerank → MMR → Expand Pattern + +**Recommended pipeline order** (community consensus + Vectara/OpenSearch docs): + +``` +Dense retrieval (oversample) → Rerank → MMR (optional) → Adjacent chunk expansion +``` + +- **Dense retrieval**: Fetch `top_k × oversample_factor` candidates. +- **Rerank**: Cross-encoder scores query–document pairs, returns top-k. +- **MMR**: Optional diversity pass on reranked results. +- **Adjacent expansion**: Expand selected chunks with prev/next chunks (your current design). + +### 1.2 MMR: Before or After Reranking? + +**Apply MMR after reranking.** + +| Order | Rationale | +|-------|-----------| +| **Retrieve → Rerank → MMR** | Reranker provides relevance scores; MMR uses them to balance relevance vs diversity. | +| Retrieve → MMR → Rerank | MMR would operate on weaker similarity scores; reranker would then reorder, partially undoing diversity. | + +**Source**: Mixpeek, OpenSearch, Elasticsearch Labs — MMR is a *reranking* stage that needs a candidate pool and relevance scores. + +### 1.3 Oversampling Factor + +| Factor | Use case | Latency | Quality | +|-------|----------|---------|---------| +| **2x** | Low latency | Lower | Good | +| **3–5x** | Balanced (recommended) | Medium | Better | +| **5–10x** | High precision | Higher | Best | + +**Recommendation**: Start with **3x** (as in your plan). Research suggests 2–5x; 3x is a common default. Tune based on nDCG@10 and latency. + +**Source**: Ailog RAG Guide 2025, TopK docs — 10–40% accuracy gains with oversampling. + +### 1.4 Error Handling: Graceful Degradation + +**Pattern** (matches your plan): + +```python +async def _rerank_nodes( + self, + nodes: list[NodeWithScore], + query: str, + top_n: int, +) -> list[NodeWithScore]: + """Apply reranker. Falls back to original order on any error.""" + try: + reranker = get_reranker(top_n=top_n) + return await asyncio.to_thread( + reranker.postprocess_nodes, nodes, query_str=query + ) + except Exception: + logger.warning("Reranking failed, returning original nodes", exc_info=True) + return nodes[:top_n] # Truncate to top_n, preserve order +``` + +**Fallback strategies** (ChatNexus, Grizzly Peak): + +1. **Primary**: Return original retrieval order (no reranking). +2. **Optional**: Retry with smaller batch (e.g. halve batch size on OOM). +3. **Optional**: Cache successful reranker results for reuse. +4. **Optional**: Fallback to lighter model (e.g. BGE-reranker if Qwen OOM). + +**Must**: Catch and log; never fail the request. + +--- + +## 2. MMR (Maximum Marginal Relevance) in Practice + +### 2.1 Lambda Parameter + +| λ | Effect | Use case | +|---|--------|----------| +| **0.0** | Max diversity | Exploratory search | +| **0.5** | Balanced | General (recommended) | +| **0.7–0.8** | Relevance-heavy | Factual QA | +| **1.0** | Max relevance | Pure relevance | + +**Recommendation for academic papers**: **0.5–0.7** — diversity helps avoid redundant chunks from the same paper/section. + +### 2.2 Paper-Level vs Chunk-Level Diversity + +| Level | Strategy | +|-------|----------| +| **Chunk-level** | MMR on embedding similarity between chunks. | +| **Paper-level** | Deduplicate by `paper_id` before/after MMR; cap chunks per paper. | + +**Recommendation**: Combine both: + +1. **Chunk-level MMR**: Use LlamaIndex `vector_store_kwargs={"mmr_threshold": 0.5}`. +2. **Paper-level**: After MMR, optionally cap chunks per paper (e.g. max 2 per paper) or deduplicate by `paper_id` before final ranking. + +### 2.3 Integration with Adjacent Chunk Expansion + +**Order**: Rerank → MMR → Adjacent expansion. + +- Adjacent expansion uses **paper_id** and **chunk_index** from selected chunks. +- MMR selects diverse chunks; expansion adds context around them. +- No conflict: MMR operates on chunk selection; expansion adds context to selected chunks. + +--- + +## 3. HNSW Parameter Tuning for Academic Retrieval + +### 3.1 768-Dim Vectors: Typical Values + +| Parameter | Default | Recommended (academic) | Effect | +|-----------|---------|------------------------|--------| +| `ef_construction` | 100 | **150–200** | Higher recall, slower build | +| `ef_search` | 100 | **80–150** | Higher recall, slower queries | +| `max_neighbors` (M) | 16 | **24–32** | Higher recall, more memory | + +**Recommendation for 768-dim academic retrieval**: + +```python +configuration={ + "hnsw": { + "space": "cosine", + "ef_construction": 200, + "ef_search": 100, + "max_neighbors": 32, + } +} +``` + +### 3.2 Trade-offs + +| Increase | Recall | Latency | Memory | +|----------|--------|---------|--------| +| ef_construction | ↑ | Build time ↑ | ↑ | +| ef_search | ↑ | Query time ↑ | — | +| max_neighbors | ↑ | — | ↑ | + +### 3.3 ChromaDB Configuration + +**Chroma 1.0+** uses `configuration` (not `metadata`): + +```python +collection = client.get_or_create_collection( + name=f"project_{project_id}", + configuration={ + "hnsw": { + "space": "cosine", + "ef_construction": 200, + "ef_search": 100, + "max_neighbors": 32, + } + }, +) +``` + +**Note**: Chroma 0.6.x may use `metadata={"hnsw:space": "cosine"}`. Check your Chroma version; `configuration` is for Chroma 1.0+. + +### 3.4 When to Rebuild Index + +| Parameter | Mutable after creation? | Rebuild? | +|-----------|--------------------------|----------| +| `space` | No | Yes | +| `ef_construction` | No | Yes | +| `max_neighbors` | No | Yes | +| `ef_search` | **Yes** | No | + +**Rebuild**: Delete collection and re-index when changing `space`, `ef_construction`, or `max_neighbors`. + +--- + +## 4. Qwen3-Reranker Model Specifics + +### 4.1 Sentence-Transformers CrossEncoder Compatibility + +**Qwen3-Reranker does NOT work with standard SentenceTransformer CrossEncoder.** + +- Architecture: `AutoModelForCausalLM` (yes/no token logits) +- Format: `": ...\n: ...\n: ..."` +- Uses `token_true_id` / `token_false_id` ("yes"/"no") for relevance scoring + +**Source**: [HuggingFace Qwen3-Reranker-0.6B](https://huggingface.co/Qwen/Qwen3-Reranker-0.6B), [LlamaIndex #19790](https://github.com/run-llama/llama_index/issues/19790). + +### 4.2 Options for LlamaIndex Integration + +| Option | Effort | Notes | +|-------|--------|-------| +| **Custom wrapper** | Medium | Extend `BaseNodePostprocessor`, implement Qwen format + `postprocess_nodes` | +| **Llama-server** | High | Convert to GGUF, run via `/v1/rerank` | +| **BGE-reranker** | Low | Use `BAAI/bge-reranker-v2-m3` with SentenceTransformerRerank | + +**Custom wrapper sketch**: + +```python +from llama_index.core.postprocessor import BaseNodePostprocessor +from transformers import AutoModelForCausalLM, AutoTokenizer + +class Qwen3RerankerPostprocessor(BaseNodePostprocessor): + def __init__(self, model_name: str = "Qwen/Qwen3-Reranker-0.6B", top_n: int = 10): + self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + self.model = AutoModelForCausalLM.from_pretrained(model_name).eval() + self.top_n = top_n + # ... format_instruction, compute_logits logic from HuggingFace README + + def _postprocess_nodes(self, nodes, query_bundle): + # Format pairs, call model, return top_n by score + ... +``` + +### 4.3 GPU Memory & Batch + +| Model | Approx. GPU memory (fp16) | Batch size (24GB) | +|-------|---------------------------|-------------------| +| **0.6B** | ~1.5–2 GB | 32–64 | +| **8B** | ~16–18 GB | 4–8 | + +**0.6B**: Suitable for single GPUs; batch 16–32 for typical queries. + +### 4.4 Input Format + +```python +def format_instruction(instruction, query, doc): + return f": {instruction}\n: {query}\n: {doc}" + +# Default instruction: "Given a web search query, retrieve relevant passages that answer the query" +# Custom instruction for academic: "Given a research question, retrieve relevant passages from academic papers" +``` + +--- + +## 5. Testing Reranking Quality + +### 5.1 Metrics + +| Metric | Use case | +|--------|----------| +| **nDCG@10** | Primary for reranking; ranking quality across top 10 | +| **MRR** | When first relevant result matters | +| **Precision@K** | % of top-K that are relevant | +| **Recall@K** | % of relevant docs in top-K | + +**Recommendation**: Use **nDCG@10** as main metric for reranking; optionally MRR for QA-style evaluation. + +### 5.2 A/B Testing + +| Approach | Description | +|----------|-------------| +| **Dual pipeline** | Same query → baseline vs variant → compare nDCG@10 and latency | +| **User split** | Traffic split (e.g. 90/10) between variants | +| **Offline** | Evaluate on labeled query–document pairs before production | + +**Metrics to track**: nDCG@10, MRR, latency p95, error rate. + +### 5.3 Mock Reranker for Unit Tests + +```python +from unittest.mock import MagicMock +from llama_index.core.schema import NodeWithScore, TextNode + +def make_mock_reranker(): + """Returns a callable that mimics reranker.postprocess_nodes.""" + def mock_postprocess(nodes: list, query_str: str): + # Return nodes in reverse order (simulates reranking) or with shuffled scores + return list(reversed(nodes))[:10] if nodes else [] + return mock_postprocess + +# In test: +with patch("app.services.reranker_service.get_reranker") as mock_get: + mock_reranker = MagicMock() + mock_reranker.postprocess_nodes.side_effect = lambda nodes, query_str: nodes[:5] + mock_get.return_value = mock_reranker + result = await rag.retrieve_only(project_id=1, question="test", top_k=5, use_reranker=True) +``` + +**Alternative**: Pass a `reranker` dependency to `RAGService` and inject a no-op or deterministic mock in tests. + +--- + +## 6. Summary: Recommendations for Omelette + +| Area | Recommendation | +|------|----------------| +| **Pipeline** | Retrieve (3× oversample) → Rerank → MMR (λ=0.5) → Adjacent expand | +| **Error handling** | Try/except in `_rerank_nodes`, return original order on failure | +| **Qwen3-Reranker** | Use custom wrapper or BGE-reranker; SentenceTransformerRerank not compatible | +| **HNSW** | ef_construction=200, ef_search=100, max_neighbors=32 | +| **Chroma** | Use `configuration` when on Chroma 1.0+ | +| **Testing** | Mock reranker in unit tests; nDCG@10 for evaluation | + +--- + +## 7. Documentation Links + +- [Chroma Configure Collections](https://docs.trychroma.com/docs/collections/configure) +- [Chroma Configuration Cookbook](https://cookbook.chromadb.dev/core/configuration/) +- [Qwen3-Reranker-0.6B HuggingFace](https://huggingface.co/Qwen/Qwen3-Reranker-0.6B) +- [Qwen3 Embedding Blog](https://qwenlm.github.io/blog/qwen3-embedding/) +- [LlamaIndex SentenceTransformerRerank](https://docs.llamaindex.ai/en/stable/examples/node_postprocessor/SentenceTransformerRerank/) +- [LlamaIndex #19790](https://github.com/run-llama/llama_index/issues/19790) — Qwen + LlamaIndex +- [Ailog RAG Reranking Guide 2025](https://app.ailog.fr/en/blog/guides/reranking) +- [Shaped A/B Testing Retrieval](https://www.shaped.ai/blog/ab-testing-retrieval-how-to-prove-your-agent-is-getting-better) diff --git a/docs/solutions/deployment/mineru-setup-guide.md b/docs/solutions/deployment/mineru-setup-guide.md new file mode 100644 index 0000000..8559f59 --- /dev/null +++ b/docs/solutions/deployment/mineru-setup-guide.md @@ -0,0 +1,138 @@ +--- +title: MinerU PDF 解析服务部署指南 +category: deployment +tags: [mineru, ocr, pdf, gpu] +created: 2026-03-17 +--- + +# MinerU PDF 解析服务部署指南 + +MinerU 是 Omelette 的高质量 PDF 解析引擎，支持公式、表格和图片识别，输出结构化 Markdown。 + +## 环境安装 + +MinerU 运行在独立的 conda 环境中，避免与 Omelette 主环境的依赖冲突。 + +```bash +# 创建 conda 环境（Python 3.10，MinerU 官方推荐） +conda create -n mineru python=3.10 -y +conda activate mineru + +# 安装 MinerU +pip install "mineru[full]>=2.7" +``` + +## 启动服务 + +```bash +# 指定可用 GPU 并启动 FastAPI 服务 +CUDA_VISIBLE_DEVICES=5,6,7 conda run -n mineru python -m mineru.cli.fast_api \ + --host 0.0.0.0 --port 8010 +``` + +首次启动会自动下载模型（约 2-3 GB），需要等待数分钟。后续启动仅加载已缓存的模型。 + +### 验证服务状态 + +```bash +# 检查 Swagger 文档页面 +curl -s -o /dev/null -w "%{http_code}" http://localhost:8010/docs +# 预期输出：200 + +# 测试 PDF 解析 +curl -X POST http://localhost:8010/file_parse \ + -F "files=@/path/to/test.pdf" \ + -F "backend=pipeline" \ + -F "return_md=true" \ + -F "formula_enable=true" \ + -F "table_enable=true" +``` + +### 使用 systemd 管理（生产环境） + +```ini +# /etc/systemd/system/mineru.service +[Unit] +Description=MinerU PDF Parsing Service +After=network.target + +[Service] +Type=simple +User=djx +Environment=CUDA_VISIBLE_DEVICES=5,6,7 +ExecStart=/path/to/conda/envs/mineru/bin/python -m mineru.cli.fast_api --host 0.0.0.0 --port 8010 +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target +``` + +## 后端配置对接 + +在 `.env` 中配置以下参数： + +```bash +# 解析器选择：mineru | pdfplumber | auto +PDF_PARSER=mineru + +# MinerU 服务地址 +MINERU_API_URL=http://localhost:8010 + +# 解析后端：pipeline（推荐）| hybrid-auto-engine | vlm-auto-engine +MINERU_BACKEND=pipeline + +# 单个 PDF 解析超时（秒），大文件建议设大 +MINERU_TIMEOUT=8000 +``` + +### 配置项说明 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `PDF_PARSER` | `mineru` | 解析引擎选择。`mineru` 优先使用 MinerU，不可用时回退到 pdfplumber | +| `MINERU_API_URL` | `http://localhost:8010` | MinerU FastAPI 服务地址 | +| `MINERU_BACKEND` | `pipeline` | `pipeline` 最稳定；`vlm-auto-engine` 需要 VLM 模型 | +| `MINERU_TIMEOUT` | `8000` | 每个 PDF 解析的超时秒数 | + +## GPU 配置 + +MinerU 服务端自行管理 GPU 分配。Omelette 后端通过 HTTP 调用 MinerU，不直接占用 GPU。 + +```bash +# MinerU 使用的 GPU（通过 CUDA_VISIBLE_DEVICES 控制） +CUDA_VISIBLE_DEVICES=5,6,7 conda run -n mineru python -m mineru.cli.fast_api --host 0.0.0.0 --port 8010 +``` + +如需限制 MinerU 只使用特定 GPU： + +```bash +# 仅使用物理 GPU 7 +CUDA_VISIBLE_DEVICES=7 conda run -n mineru python -m mineru.cli.fast_api --host 0.0.0.0 --port 8010 +``` + +## 回退机制 + +当 MinerU 不可用时（服务未启动/网络不通/解析失败），Omelette 自动回退： + +1. **MinerU**（首选）→ 高质量 Markdown，支持公式和表格 +2. **pdfplumber**（回退）→ 原生文本提取，轻量无 GPU 要求 +3. **PaddleOCR**（二次回退）→ 扫描件 OCR，需 GPU + +## 常见问题 + +### Q: 首次解析很慢？ + +A: MinerU 首次请求需下载和加载模型（~2GB），通常耗时 1-3 分钟。后续请求正常。 + +### Q: 出现 CUDA out of memory？ + +A: MinerU 的 pipeline 后端需要约 3-4 GB 显存。确保指定的 GPU 有足够空闲显存。 + +### Q: 如何切换回 pdfplumber？ + +A: 设置 `PDF_PARSER=pdfplumber`，无需启动 MinerU 服务。 + +### Q: MinerU 服务端口被占用？ + +A: 修改启动命令中的 `--port` 参数，并同步更新 `.env` 中的 `MINERU_API_URL`。 diff --git a/frontend/src/components/knowledge-base/SubscriptionManager.tsx b/frontend/src/components/knowledge-base/SubscriptionManager.tsx index 0463a0f..18401c5 100644 --- a/frontend/src/components/knowledge-base/SubscriptionManager.tsx +++ b/frontend/src/components/knowledge-base/SubscriptionManager.tsx @@ -123,7 +123,7 @@ export function SubscriptionManager({ projectId }: SubscriptionManagerProps) { invalidateKeys: [['subscriptions', projectId]], }); - const subscriptions: Subscription[] = data ?? []; + const subscriptions: Subscription[] = data?.items ?? []; const resetForm = () => { setForm({ diff --git a/frontend/src/services/subscription-api.ts b/frontend/src/services/subscription-api.ts index 936b652..637a6d1 100644 --- a/frontend/src/services/subscription-api.ts +++ b/frontend/src/services/subscription-api.ts @@ -1,4 +1,5 @@ import { api } from '@/lib/api'; +import type { PaginatedData } from '@/lib/api'; export interface Subscription { id: number; @@ -25,7 +26,9 @@ export interface SubscriptionCreate { export const subscriptionApi = { list: (projectId: number) => - api.get(`/projects/${projectId}/subscriptions`).then(r => r.data), + api + .get>(`/projects/${projectId}/subscriptions`) + .then(r => r.data), create: (projectId: number, data: SubscriptionCreate) => api.post(`/projects/${projectId}/subscriptions`, data).then(r => r.data), update: ( diff --git a/frontend/src/test/mocks/handlers.ts b/frontend/src/test/mocks/handlers.ts index 4e7b349..3c30fd8 100644 --- a/frontend/src/test/mocks/handlers.ts +++ b/frontend/src/test/mocks/handlers.ts @@ -303,7 +303,15 @@ export const handlers = [ // Subscriptions http.get(`${apiBase}/projects/:id/subscriptions`, () => - HttpResponse.json(mockResponse(mockSubscriptionList)), + HttpResponse.json( + mockResponse({ + items: mockSubscriptionList, + total: mockSubscriptionList.length, + page: 1, + page_size: 20, + total_pages: 1, + }), + ), ), http.post(`${apiBase}/projects/:id/subscriptions`, async ({ request, params }) => { const body = (await request.json()) as Record;