From bf22842aac68a738c3b09daa818130b6d4d6940b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Jan 2026 21:34:55 +0000 Subject: [PATCH] Add Prompt Rewriter Agent system for autonomous prompt improvement This adds a comprehensive system for dynamically improving prompts based on user feedback. The system includes: - Technical proposal document explaining the architecture and workflow - Implementation roadmap with detailed task breakdowns - Database migration for tracking proposals, experiments, and knowledge base - Initial module structure with models, config, and base agent class - Feedback Intake Agent implementation for classifying user feedback - Orchestrator Prefect flow coordinating the multi-agent pipeline The Prompt Rewriter Agent will: 1. Capture user feedback (thumbs down, comments, label disputes) 2. Research disputed claims via web search 3. Generate structured prompt modification proposals 4. Test proposals experimentally against real snippets 5. Evaluate results and decide whether to deploy 6. Update prompts and reprocess affected historical snippets --- docs/PROMPT_REWRITER_AGENT_PROPOSAL.md | 552 ++++++++++++++++++ .../PROMPT_REWRITER_IMPLEMENTATION_ROADMAP.md | 518 ++++++++++++++++ src/prompt_rewriter/__init__.py | 37 ++ src/prompt_rewriter/agents/__init__.py | 17 + src/prompt_rewriter/agents/base.py | 250 ++++++++ src/prompt_rewriter/agents/feedback_intake.py | 164 ++++++ src/prompt_rewriter/config.py | 119 ++++ src/prompt_rewriter/main.py | 407 +++++++++++++ src/prompt_rewriter/models.py | 205 +++++++ ...124000000_prompt_rewriter_agent_schema.sql | 429 ++++++++++++++ 10 files changed, 2698 insertions(+) create mode 100644 docs/PROMPT_REWRITER_AGENT_PROPOSAL.md create mode 100644 docs/PROMPT_REWRITER_IMPLEMENTATION_ROADMAP.md create mode 100644 src/prompt_rewriter/__init__.py create mode 100644 src/prompt_rewriter/agents/__init__.py create mode 100644 src/prompt_rewriter/agents/base.py create mode 100644 src/prompt_rewriter/agents/feedback_intake.py create mode 100644 src/prompt_rewriter/config.py create mode 100644 src/prompt_rewriter/main.py create mode 100644 src/prompt_rewriter/models.py create mode 100644 supabase/migrations/20260124000000_prompt_rewriter_agent_schema.sql diff --git a/docs/PROMPT_REWRITER_AGENT_PROPOSAL.md b/docs/PROMPT_REWRITER_AGENT_PROPOSAL.md new file mode 100644 index 0000000..8a3a653 --- /dev/null +++ b/docs/PROMPT_REWRITER_AGENT_PROPOSAL.md @@ -0,0 +1,552 @@ +# Prompt Rewriter Agent: Technical Proposal + +## Executive Summary + +VERDAD's current fact-checking relies primarily on web search during analysis. This approach has proven insufficient for reliably correcting misinformation—search results are inconsistent, may not surface authoritative sources, and leave critical knowledge gaps. + +We propose a **Prompt Rewriter Agent** system that: +1. Dynamically augments prompts with verified factual information +2. Automatically triggers on user feedback (thumbs down, comments) +3. Researches, proposes, tests, and deploys prompt improvements +4. Reprocesses historical snippets affected by new knowledge + +This system is designed to be **extensible** for future applications (Contextualizer/Docs-AI). + +--- + +## Problem Statement + +### Current Limitations + +1. **Search Unreliability**: Stage 3's Google Search grounding returns inconsistent results. The same query may yield different sources at different times, leading to varying analysis quality. + +2. **Knowledge Gaps**: The model lacks domain-specific knowledge about recurring misinformation narratives (e.g., specific election fraud claims, COVID vaccine myths with precise statistics). + +3. **No Learning Loop**: User feedback (labels, comments, reactions) is captured but never feeds back into prompt improvements. The system doesn't learn from corrections. + +4. **Manual Prompt Updates**: Currently, prompt changes require developer intervention—reading the prompt file, making edits, deploying. No systematic testing of changes. + +--- + +## Proposed Solution + +### Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ USER FEEDBACK TRIGGERS │ +│ 👎 Thumbs Down │ 💬 Comment │ 🏷️ Label Dispute │ 📝 Report │ +└─────────────────────────────┬───────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PROMPT REWRITER AGENT ORCHESTRATOR │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Feedback Intake Agent │ │ +│ │ • Parse feedback type and content │ │ +│ │ • Extract snippet context │ │ +│ │ • Classify intent (factual error, missing context, wrong category) │ │ +│ └───────────────────────────────────┬──────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Research Agent │ │ +│ │ • Multi-source web research with retry logic │ │ +│ │ • Source credibility scoring │ │ +│ │ • Fact extraction and verification │ │ +│ │ • Fail noisily with detailed error context │ │ +│ └───────────────────────────────────┬──────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Proposal Writer Agent │ │ +│ │ • Generate structured prompt modification proposals │ │ +│ │ • Specify: target prompt, section, addition type, content │ │ +│ │ • Include rationale and expected impact │ │ +│ └───────────────────────────────────┬──────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Experiment Runner Agent │ │ +│ │ • Run proposal against original snippet (N times) │ │ +│ │ • Compare outputs: before vs after │ │ +│ │ • Statistical significance testing │ │ +│ └───────────────────────────────────┬──────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Evaluation Agent │ │ +│ │ • Assess if proposal resolves reported issue │ │ +│ │ • Check for regressions on control snippets │ │ +│ │ • Score improvement confidence │ │ +│ └───────────────────────────────────┬──────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Refinement Loop (if needed) │ │ +│ │ • Adjust proposal based on experiment results │ │ +│ │ • Re-run experiments │ │ +│ │ • Max 3 refinement iterations │ │ +│ └───────────────────────────────────┬──────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Semantic Search Agent │ │ +│ │ • Find historically similar snippets via embedding similarity │ │ +│ │ • Test proposal against similar snippets │ │ +│ │ • Identify snippets for reprocessing │ │ +│ └───────────────────────────────────┬──────────────────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Deployment Agent │ │ +│ │ • Update prompt_versions table with new version │ │ +│ │ • Set is_active=true for new prompt │ │ +│ │ • Queue historical snippets for reprocessing │ │ +│ │ • Notify team via Slack │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Knowledge Base Design + +### Structure + +The knowledge base is a **structured addition to existing prompts**, not a separate retrieval system. For now, we inject the entire knowledge base into prompts. If it grows large enough to warrant sharding, we can add RAG later. + +```markdown +## Verified Facts Knowledge Base + +### Election Claims +- **Claim**: "2020 election had millions of illegal votes" + **Fact**: Multiple audits, court cases (60+), and election officials from both + parties confirmed the 2020 election results. No evidence of widespread fraud + was found. Source: CISA, state election officials, court records. + +- **Claim**: "Dominion voting machines switched votes" + **Fact**: Hand recounts in Georgia matched machine counts. Dominion machines + produce paper ballots for verification. Multiple audits found no manipulation. + Source: Georgia Secretary of State audit, CISA. + +### COVID-19 / Vaccines +- **Claim**: "mRNA vaccines alter your DNA" + **Fact**: mRNA cannot enter the cell nucleus where DNA is stored. mRNA is + processed in the cytoplasm and degrades within days. Source: CDC, peer-reviewed + studies in Nature. + +### Immigration +- **Claim**: "Immigrants commit more crimes than citizens" + **Fact**: Studies consistently show immigrants (including undocumented) commit + crimes at lower rates than native-born citizens. Source: Cato Institute, + multiple peer-reviewed criminology studies. +``` + +### Storage Options + +**Option A: Inline in Prompt Files (Recommended for MVP)** +- Add `## Verified Facts Knowledge Base` section to `Stage_1_heuristics.md` and `Stage_3_heuristics.md` +- Simple to implement, version-controlled with git +- Easy to audit and review changes + +**Option B: Separate Knowledge Base File** +- Create `prompts/knowledge_base.md` +- Imported into prompts via `constants.py` +- Better separation of concerns + +**Option C: Database-Stored (Future)** +- Store facts in `knowledge_facts` table +- Query and inject at runtime +- Enables RAG when knowledge base grows large + +--- + +## Detailed Agent Specifications + +### 1. Feedback Intake Agent + +**Purpose**: Parse and classify user feedback to determine appropriate action. + +**Input**: +```json +{ + "feedback_type": "thumbs_down" | "comment" | "label_dispute", + "snippet_id": "uuid", + "user_id": "uuid", + "content": "The analysis says this is false but it's actually true...", + "metadata": { "label_id": "uuid", "thread_id": "string" } +} +``` + +**Output**: +```json +{ + "intent": "factual_error" | "missing_context" | "wrong_category" | "unclear", + "confidence": 0.85, + "extracted_claim": "Vaccine causes autism", + "user_correction": "Multiple studies disprove this", + "affected_prompts": ["stage_1", "stage_3"], + "priority": "high" | "medium" | "low" +} +``` + +**Implementation Notes**: +- Uses LLM to parse free-form comments +- Maps label disputes to specific claim types +- Determines which pipeline stages need modification + +--- + +### 2. Research Agent + +**Purpose**: Conduct thorough, reliable web research on the disputed claim. + +**Key Requirements**: +- **Multi-source verification**: Minimum 3 independent sources +- **Source credibility scoring**: Prioritize .gov, .edu, peer-reviewed +- **Retry logic**: Exponential backoff, max 5 retries per search +- **Fail noisily**: If research cannot be completed, halt pipeline with detailed error + +**Search Strategy**: +```python +search_queries = [ + f"{claim} fact check", + f"{claim} evidence", + f"{claim} debunked OR confirmed", + f"{claim} site:gov OR site:edu", + f"{claim} peer reviewed study" +] +``` + +**Output**: +```json +{ + "claim": "Vaccine causes autism", + "verdict": "false", + "confidence": 0.95, + "sources": [ + { + "url": "https://www.cdc.gov/...", + "title": "Vaccines Do Not Cause Autism", + "credibility_score": 0.95, + "relevant_excerpt": "...", + "publication_date": "2023-01-15" + } + ], + "summary": "Multiple large-scale studies...", + "research_complete": true, + "research_attempts": 2 +} +``` + +--- + +### 3. Proposal Writer Agent + +**Purpose**: Generate structured, testable prompt modification proposals. + +**Proposal Types**: +1. **Factual Addition**: Add verified fact to knowledge base +2. **Heuristic Update**: Modify detection heuristics +3. **Instruction Clarification**: Improve analysis instructions +4. **Category Addition**: Add new disinformation category + +**Output Schema**: +```json +{ + "proposal_id": "uuid", + "type": "factual_addition", + "target_prompts": ["Stage_1_heuristics.md", "Stage_3_heuristics.md"], + "changes": [ + { + "file": "Stage_1_heuristics.md", + "section": "## COVID-19 / Vaccines", + "action": "append", + "content": "- **Claim**: \"Vaccines cause autism\"\n **Fact**: ...", + "rationale": "User reported incorrect analysis of autism-vaccine claim" + } + ], + "expected_impact": "Snippets mentioning vaccine-autism link will correctly identify this as debunked misinformation", + "test_snippet_ids": ["uuid1", "uuid2"] +} +``` + +--- + +### 4. Experiment Runner Agent + +**Purpose**: Test proposals against real snippets with statistical rigor. + +**Methodology**: +1. Run original prompt against test snippet (5 times) +2. Run modified prompt against test snippet (5 times) +3. Compare outputs for consistency and correctness +4. Calculate improvement metrics + +**Output**: +```json +{ + "experiment_id": "uuid", + "proposal_id": "uuid", + "snippet_id": "uuid", + "runs": { + "baseline": [ + { "run_id": 1, "output": {...}, "correct": false }, + { "run_id": 2, "output": {...}, "correct": false } + ], + "proposal": [ + { "run_id": 1, "output": {...}, "correct": true }, + { "run_id": 2, "output": {...}, "correct": true } + ] + }, + "metrics": { + "baseline_accuracy": 0.0, + "proposal_accuracy": 1.0, + "improvement": 1.0, + "consistency": 0.95 + } +} +``` + +--- + +### 5. Evaluation Agent + +**Purpose**: Determine if proposal should be accepted, refined, or rejected. + +**Acceptance Criteria**: +- Resolves the originally reported issue +- No regressions on control snippets (unrelated topics) +- Improvement is statistically significant (p < 0.05) +- Consistency across runs > 80% + +**Output**: +```json +{ + "decision": "accept" | "refine" | "reject", + "confidence": 0.92, + "issues_resolved": true, + "regressions_detected": false, + "refinement_suggestions": null, + "human_review_required": false +} +``` + +--- + +### 6. Semantic Search Agent + +**Purpose**: Find similar historical snippets that may benefit from the improvement. + +**Implementation**: +- Use existing Stage 5 embeddings +- Cosine similarity search against snippet collection +- Filter by disinformation categories mentioned in proposal + +**Output**: +```json +{ + "query_snippet_id": "uuid", + "similar_snippets": [ + { "id": "uuid", "similarity": 0.92, "title": "..." }, + { "id": "uuid", "similarity": 0.87, "title": "..." } + ], + "reprocess_candidates": ["uuid1", "uuid2", "uuid3"], + "estimated_impact": 47 +} +``` + +--- + +### 7. Deployment Agent + +**Purpose**: Apply approved changes and trigger reprocessing. + +**Actions**: +1. Create new prompt version in `prompt_versions` table +2. Update prompt files (if using file-based storage) +3. Set new version as active +4. Queue similar snippets for Stage 3 reprocessing +5. Send Slack notification to team + +--- + +## Database Schema Extensions + +```sql +-- Track prompt rewrite proposals +CREATE TABLE prompt_rewrite_proposals ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + triggered_by_feedback_type TEXT NOT NULL, -- 'thumbs_down', 'comment', 'label_dispute' + triggered_by_snippet_id UUID REFERENCES snippets(id), + triggered_by_user_id UUID REFERENCES auth.users(id), + + -- Feedback analysis + intent_classification TEXT, -- 'factual_error', 'missing_context', etc. + extracted_claim TEXT, + user_correction TEXT, + + -- Research results + research_summary JSONB, + sources JSONB, + research_completed_at TIMESTAMPTZ, + + -- Proposal details + proposal_type TEXT, -- 'factual_addition', 'heuristic_update', etc. + proposal_changes JSONB, + expected_impact TEXT, + + -- Experiment results + experiment_results JSONB, + baseline_accuracy FLOAT, + proposal_accuracy FLOAT, + + -- Evaluation + evaluation_decision TEXT, -- 'accept', 'refine', 'reject' + refinement_count INT DEFAULT 0, + human_review_required BOOLEAN DEFAULT false, + + -- Deployment + deployed_at TIMESTAMPTZ, + prompt_version_id UUID REFERENCES prompt_versions(id), + + -- Status tracking + status TEXT DEFAULT 'pending', -- 'pending', 'researching', 'proposing', 'experimenting', 'evaluating', 'deployed', 'rejected', 'failed' + error_message TEXT, + + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +-- Track reprocessing jobs spawned by proposals +CREATE TABLE snippet_reprocess_queue ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + snippet_id UUID REFERENCES snippets(id), + proposal_id UUID REFERENCES prompt_rewrite_proposals(id), + reason TEXT, + priority INT DEFAULT 0, + status TEXT DEFAULT 'queued', -- 'queued', 'processing', 'completed', 'failed' + queued_at TIMESTAMPTZ DEFAULT NOW(), + processed_at TIMESTAMPTZ +); + +-- Knowledge base entries (for future RAG) +CREATE TABLE knowledge_facts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + category TEXT NOT NULL, -- 'election', 'covid', 'immigration', etc. + claim TEXT NOT NULL, + fact TEXT NOT NULL, + sources JSONB, + added_by_proposal_id UUID REFERENCES prompt_rewrite_proposals(id), + is_active BOOLEAN DEFAULT true, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); +``` + +--- + +## Implementation Phases + +### Phase 1: Foundation (Week 1-2) +- Database schema extensions +- Basic feedback intake pipeline (webhook → database) +- Manual trigger for rewriter agent (admin API endpoint) + +### Phase 2: Core Agents (Week 3-4) +- Feedback Intake Agent +- Research Agent with retry logic +- Proposal Writer Agent +- Basic experiment runner (single snippet, single run) + +### Phase 3: Experimentation & Evaluation (Week 5-6) +- Full Experiment Runner with statistical testing +- Evaluation Agent with acceptance criteria +- Refinement loop implementation + +### Phase 4: Deployment & Reprocessing (Week 7-8) +- Semantic Search Agent integration +- Deployment Agent with prompt versioning +- Reprocessing queue and workers +- Slack notifications + +### Phase 5: Polish & Monitoring (Week 9-10) +- Dashboard for proposal status tracking +- Metrics and monitoring +- Documentation and runbooks + +--- + +## Extensibility for Future Projects + +The Prompt Rewriter Agent is designed as a **reusable module**: + +```python +class PromptRewriterAgent: + """Base class for prompt rewriting workflows.""" + + def __init__(self, config: RewriterConfig): + self.feedback_intake = FeedbackIntakeAgent(config) + self.research = ResearchAgent(config) + self.proposal_writer = ProposalWriterAgent(config) + self.experiment_runner = ExperimentRunnerAgent(config) + self.evaluator = EvaluationAgent(config) + self.deployer = DeploymentAgent(config) + + async def process_feedback(self, feedback: Feedback) -> ProposalResult: + """Override in subclasses for custom workflows.""" + pass + +class VerdadPromptRewriter(PromptRewriterAgent): + """VERDAD-specific implementation.""" + + async def process_feedback(self, feedback: Feedback) -> ProposalResult: + # VERDAD-specific logic + pass + +class ContextualizerPromptRewriter(PromptRewriterAgent): + """Contextualizer/Docs-AI implementation.""" + + async def process_feedback(self, feedback: Feedback) -> ProposalResult: + # Contextualizer-specific logic + pass +``` + +--- + +## Risk Mitigation + +| Risk | Mitigation | +|------|------------| +| Bad proposals deployed | Require human review for high-impact changes | +| Research returns incorrect info | Multi-source verification, credibility scoring | +| Infinite refinement loops | Max 3 refinement iterations, then human escalation | +| Prompt bloat from too many facts | Monitor prompt token count, implement RAG when >8k tokens | +| Reprocessing overwhelms system | Priority queue, rate limiting, off-peak scheduling | + +--- + +## Success Metrics + +1. **Feedback Resolution Rate**: % of user feedback that results in deployed improvements +2. **Time to Resolution**: Hours from feedback to deployed fix +3. **Accuracy Improvement**: % improvement in correctness for affected claim types +4. **Regression Rate**: % of deployments that cause regressions (target: <5%) +5. **Knowledge Base Growth**: Facts added per week + +--- + +## Conclusion + +The Prompt Rewriter Agent transforms VERDAD from a static analysis system into a **continuously learning platform**. User feedback directly improves the system, creating a virtuous cycle where corrections make future analysis more accurate. + +The architecture is intentionally modular and extensible, serving as the foundation for similar systems in Contextualizer/Docs-AI and other future projects. + +--- + +## Appendix: Example Workflow + +**Scenario**: User clicks thumbs down on a snippet about "vaccine microchips" + +1. **Trigger**: Thumbs down webhook fires +2. **Feedback Intake**: Classifies as "factual_error", extracts claim "vaccines contain microchips" +3. **Research**: Finds CDC, FDA, Reuters fact-checks confirming this is false +4. **Proposal**: Suggests adding to knowledge base: "Claim: Vaccines contain microchips. Fact: Vaccines do not contain microchips. The largest component in vaccines is too small to be a tracking device." +5. **Experiment**: Runs modified prompt 5x against original snippet, achieves 100% correct detection +6. **Evaluation**: Accepts proposal (no regressions, statistically significant improvement) +7. **Deployment**: Updates prompt_versions, queues 23 similar snippets for reprocessing +8. **Notification**: Slack message: "New fact added to knowledge base: vaccine microchips claim" diff --git a/docs/PROMPT_REWRITER_IMPLEMENTATION_ROADMAP.md b/docs/PROMPT_REWRITER_IMPLEMENTATION_ROADMAP.md new file mode 100644 index 0000000..04c55c4 --- /dev/null +++ b/docs/PROMPT_REWRITER_IMPLEMENTATION_ROADMAP.md @@ -0,0 +1,518 @@ +# Prompt Rewriter Agent: Implementation Roadmap + +## Overview + +This document breaks down the Prompt Rewriter Agent into concrete implementation tasks. Each task is sized for ~1-3 days of work and can be assigned as a Linear issue. + +--- + +## Phase 1: Foundation + +### 1.1 Database Schema Deployment +**Priority: Critical | Estimate: 1 day** + +- [ ] Review migration file: `supabase/migrations/20260124000000_prompt_rewriter_agent_schema.sql` +- [ ] Test migration in local Supabase instance +- [ ] Deploy to staging environment +- [ ] Verify all tables, indexes, and functions created correctly +- [ ] Document any schema adjustments needed + +**Files:** +- `supabase/migrations/20260124000000_prompt_rewriter_agent_schema.sql` + +--- + +### 1.2 Feedback Event Capture - Thumbs Down +**Priority: Critical | Estimate: 2 days** + +Add thumbs down button to snippet UI and capture events. + +- [ ] Add `thumbs_down` column to snippets table (or use feedback_events) +- [ ] Create API endpoint: `POST /api/snippets/:id/feedback` +- [ ] Add thumbs down button to snippet card component +- [ ] Wire button to API endpoint +- [ ] Insert record into `user_feedback_events` table +- [ ] Test end-to-end flow + +**Files to modify:** +- `web/src/components/SnippetCard.tsx` (or equivalent) +- `server/src/api/routes.ts` +- `server/src/services/feedbackService.ts` (new) + +--- + +### 1.3 Feedback Event Capture - Comments +**Priority: High | Estimate: 1 day** + +Extend existing Liveblocks webhook to capture comment events for processing. + +- [ ] Modify `handleCommentCreated` to insert into `user_feedback_events` +- [ ] Extract snippet_id from Liveblocks room_id +- [ ] Classify comment sentiment (positive/negative/neutral) +- [ ] Test with sample comments + +**Files to modify:** +- `server/src/api/webhooks.ts` +- `server/src/services/commentService.ts` + +--- + +### 1.4 Basic Orchestrator Prefect Flow +**Priority: Critical | Estimate: 2 days** + +Create the main Prefect flow that orchestrates the agent pipeline. + +```python +@flow(name="Prompt Rewriter Agent") +async def prompt_rewriter_flow(proposal_id: UUID): + # 1. Load proposal from database + # 2. Run feedback intake agent + # 3. Run research agent + # 4. Run proposal writer agent + # 5. Run experiment runner + # 6. Run evaluation agent + # 7. Deploy or refine +``` + +- [ ] Create `src/prompt_rewriter/main.py` with orchestrator flow +- [ ] Create `src/prompt_rewriter/__init__.py` +- [ ] Add Prefect deployment configuration +- [ ] Create manual trigger endpoint for testing +- [ ] Test with mock data + +**Files to create:** +- `src/prompt_rewriter/main.py` +- `src/prompt_rewriter/config.py` +- `src/prompt_rewriter/models.py` (Pydantic models) + +--- + +### 1.5 Database Utilities +**Priority: High | Estimate: 1 day** + +Create database utility functions for the rewriter system. + +- [ ] Add `prompt_rewriter_utils.py` with CRUD operations +- [ ] `create_proposal()` - insert new proposal +- [ ] `update_proposal_status()` - status transitions +- [ ] `get_pending_proposals()` - fetch proposals to process +- [ ] `log_agent_execution()` - insert agent logs +- [ ] Unit tests for all functions + +**Files to create:** +- `src/prompt_rewriter/db_utils.py` +- `tests/prompt_rewriter/test_db_utils.py` + +--- + +## Phase 2: Core Agents + +### 2.1 Feedback Intake Agent +**Priority: Critical | Estimate: 3 days** + +Analyzes user feedback to classify intent and extract actionable information. + +**Input:** Raw feedback event (thumbs down, comment, etc.) +**Output:** Classified intent, extracted claim, affected prompts + +- [ ] Create `src/prompt_rewriter/agents/feedback_intake.py` +- [ ] Design prompt for feedback classification +- [ ] Implement intent classification (factual_error, missing_context, etc.) +- [ ] Implement claim extraction from free-form text +- [ ] Determine which pipeline stages are affected +- [ ] Add confidence scoring +- [ ] Unit tests with sample feedback + +**Prompt design considerations:** +``` +Given this user feedback on a misinformation analysis: +- Snippet title: {title} +- Original analysis: {summary} +- User feedback: {feedback_content} + +Classify the intent and extract any claims mentioned. +``` + +--- + +### 2.2 Research Agent +**Priority: Critical | Estimate: 4 days** + +Conducts web research to verify or debunk claims. + +**Key requirements:** +- Multi-source verification (minimum 3 sources) +- Source credibility scoring +- Retry logic with exponential backoff +- Fail noisily with detailed errors + +- [ ] Create `src/prompt_rewriter/agents/research.py` +- [ ] Implement search query generation (multiple query variants) +- [ ] Integrate with existing Google Search grounding (from Stage 3) +- [ ] Add SearXNG integration as backup search +- [ ] Implement source credibility scoring +- [ ] Implement retry logic (max 5 attempts per search) +- [ ] Aggregate and synthesize results +- [ ] Return structured research output +- [ ] Integration tests with real searches + +**Search strategy:** +```python +search_queries = [ + f"{claim} fact check", + f"{claim} evidence", + f"{claim} site:gov OR site:edu", +] +``` + +--- + +### 2.3 Proposal Writer Agent +**Priority: Critical | Estimate: 3 days** + +Generates structured prompt modification proposals. + +**Input:** Research results, feedback analysis +**Output:** Structured proposal with specific changes + +- [ ] Create `src/prompt_rewriter/agents/proposal_writer.py` +- [ ] Design proposal output schema (JSON) +- [ ] Implement factual addition proposals +- [ ] Implement heuristic update proposals +- [ ] Generate clear rationale and expected impact +- [ ] Select test snippets from database +- [ ] Unit tests with sample research results + +**Proposal schema:** +```json +{ + "type": "factual_addition", + "target_prompts": ["Stage_1_heuristics.md"], + "changes": [{ + "section": "## COVID-19 / Vaccines", + "action": "append", + "content": "- **Claim**: ...\n **Fact**: ..." + }], + "expected_impact": "..." +} +``` + +--- + +### 2.4 Experiment Runner Agent +**Priority: High | Estimate: 4 days** + +Tests proposals against real snippets with statistical rigor. + +- [ ] Create `src/prompt_rewriter/agents/experiment_runner.py` +- [ ] Implement baseline run (current prompt, N times) +- [ ] Implement proposal run (modified prompt, N times) +- [ ] Create temporary prompt modification system +- [ ] Capture and store all LLM outputs +- [ ] Calculate accuracy metrics +- [ ] Implement consistency scoring +- [ ] Integration tests with real LLM calls + +**Methodology:** +1. Run current prompt against test snippet (5 times) +2. Run modified prompt against test snippet (5 times) +3. Compare outputs, calculate improvement metrics + +--- + +### 2.5 Evaluation Agent +**Priority: High | Estimate: 2 days** + +Determines whether to accept, refine, or reject a proposal. + +**Acceptance criteria:** +- Resolves originally reported issue +- No regressions on control snippets +- Improvement is statistically significant +- Consistency > 80% + +- [ ] Create `src/prompt_rewriter/agents/evaluator.py` +- [ ] Implement acceptance criteria checks +- [ ] Implement regression detection +- [ ] Generate refinement suggestions if needed +- [ ] Flag for human review when uncertain +- [ ] Unit tests with sample experiment results + +--- + +## Phase 3: Experimentation & Refinement + +### 3.1 Semantic Search Agent +**Priority: High | Estimate: 2 days** + +Finds similar snippets for broader testing and reprocessing. + +- [ ] Create `src/prompt_rewriter/agents/semantic_search.py` +- [ ] Use existing Stage 5 embeddings +- [ ] Implement cosine similarity search via pgvector +- [ ] Filter by relevant disinformation categories +- [ ] Return ranked list of similar snippets +- [ ] Integration tests + +**Query:** +```sql +SELECT id, title, 1 - (embedding <=> $1) as similarity +FROM snippets +WHERE status = 'Processed' +ORDER BY embedding <=> $1 +LIMIT 50; +``` + +--- + +### 3.2 Refinement Loop +**Priority: Medium | Estimate: 2 days** + +Iteratively improves proposals based on experiment feedback. + +- [ ] Implement refinement iteration in orchestrator +- [ ] Cap at 3 refinement attempts +- [ ] Create refinement prompt for Proposal Writer +- [ ] Track refinement history in proposal record +- [ ] Escalate to human review after max refinements + +--- + +### 3.3 Control Snippet Selection +**Priority: Medium | Estimate: 1 day** + +Select control snippets to detect regressions. + +- [ ] Implement control snippet selection logic +- [ ] Select snippets from different categories than the test snippet +- [ ] Store control snippet IDs in proposal +- [ ] Run experiments against control snippets +- [ ] Flag if control snippet outputs degrade + +--- + +## Phase 4: Deployment & Reprocessing + +### 4.1 Deployment Agent +**Priority: High | Estimate: 3 days** + +Applies approved changes to production prompts. + +- [ ] Create `src/prompt_rewriter/agents/deployer.py` +- [ ] Create new prompt version in `prompt_versions` table +- [ ] Update prompt files on disk (if using file-based storage) +- [ ] Set new version as active +- [ ] Add facts to `knowledge_facts` table +- [ ] Send Slack notification +- [ ] Create rollback capability + +**Slack notification:** +``` +New prompt improvement deployed! +- Type: Factual addition +- Category: COVID-19 / Vaccines +- Claim addressed: "Vaccines cause autism" +- Triggered by: user@example.com +- Snippets to reprocess: 23 +``` + +--- + +### 4.2 Knowledge Base Injection +**Priority: High | Estimate: 2 days** + +Inject knowledge base into prompts at runtime. + +- [ ] Modify `constants.py` to load knowledge base +- [ ] Query active facts from `knowledge_facts` table +- [ ] Format facts for injection into prompts +- [ ] Add to Stage 1 and Stage 3 prompts +- [ ] Monitor prompt token count +- [ ] Add logging for knowledge base size + +**Implementation:** +```python +def get_knowledge_base_section(): + facts = supabase.table('knowledge_facts').select('*').eq('is_active', True).execute() + return format_facts_as_markdown(facts.data) +``` + +--- + +### 4.3 Reprocessing Queue Worker +**Priority: High | Estimate: 3 days** + +Process snippets affected by prompt improvements. + +- [ ] Create `src/prompt_rewriter/reprocess_worker.py` +- [ ] Create Prefect flow for reprocessing +- [ ] Fetch from `snippet_reprocess_queue` +- [ ] Run Stage 3 analysis with new prompts +- [ ] Compare before/after outputs +- [ ] Update snippet record +- [ ] Mark queue item complete +- [ ] Rate limiting to avoid overload + +--- + +### 4.4 Reprocessing Queue Population +**Priority: Medium | Estimate: 1 day** + +Populate the reprocessing queue after deployment. + +- [ ] Query similar snippets via Semantic Search Agent +- [ ] Insert into `snippet_reprocess_queue` with priority +- [ ] Skip already-processed snippets +- [ ] Estimate and log total snippets to reprocess + +--- + +## Phase 5: Monitoring & Polish + +### 5.1 Admin Dashboard - Proposal Status +**Priority: Medium | Estimate: 3 days** + +Dashboard to monitor proposal status and history. + +- [ ] Create `/admin/proposals` page +- [ ] List all proposals with status +- [ ] Show proposal details (research, changes, experiments) +- [ ] Allow manual approval/rejection +- [ ] Show agent execution logs +- [ ] Filter by status, date, user + +--- + +### 5.2 Human Review Interface +**Priority: Medium | Estimate: 2 days** + +Interface for reviewing proposals flagged for human review. + +- [ ] Create `/admin/proposals/:id/review` page +- [ ] Show full proposal context +- [ ] Show experiment results comparison +- [ ] Allow approve/reject with notes +- [ ] Allow editing proposed changes +- [ ] Trigger deployment on approval + +--- + +### 5.3 Metrics & Monitoring +**Priority: Medium | Estimate: 2 days** + +Track system health and effectiveness. + +- [ ] Add Prometheus/Datadog metrics +- [ ] Track: proposals created, accepted, rejected +- [ ] Track: average time to resolution +- [ ] Track: reprocessing queue depth +- [ ] Track: knowledge base size +- [ ] Create alerts for failures + +--- + +### 5.4 Slack Notifications +**Priority: Low | Estimate: 1 day** + +Notify team of key events. + +- [ ] Notification on new proposal created +- [ ] Notification on proposal awaiting human review +- [ ] Notification on proposal deployed +- [ ] Notification on proposal rejected/failed +- [ ] Daily summary of proposals + +--- + +## Task Summary for Linear + +| Issue Title | Priority | Estimate | Phase | +|------------|----------|----------|-------| +| Deploy prompt rewriter database schema | Critical | 1 day | 1 | +| Implement thumbs down feedback capture | Critical | 2 days | 1 | +| Extend comment webhook for feedback events | High | 1 day | 1 | +| Create orchestrator Prefect flow | Critical | 2 days | 1 | +| Create database utility functions | High | 1 day | 1 | +| Implement Feedback Intake Agent | Critical | 3 days | 2 | +| Implement Research Agent | Critical | 4 days | 2 | +| Implement Proposal Writer Agent | Critical | 3 days | 2 | +| Implement Experiment Runner Agent | High | 4 days | 2 | +| Implement Evaluation Agent | High | 2 days | 2 | +| Implement Semantic Search Agent | High | 2 days | 3 | +| Implement refinement loop | Medium | 2 days | 3 | +| Implement control snippet selection | Medium | 1 day | 3 | +| Implement Deployment Agent | High | 3 days | 4 | +| Implement knowledge base injection | High | 2 days | 4 | +| Create reprocessing queue worker | High | 3 days | 4 | +| Implement reprocess queue population | Medium | 1 day | 4 | +| Build admin proposals dashboard | Medium | 3 days | 5 | +| Build human review interface | Medium | 2 days | 5 | +| Add metrics and monitoring | Medium | 2 days | 5 | +| Add Slack notifications | Low | 1 day | 5 | + +**Total estimated effort: ~42 days** + +--- + +## Dependencies + +``` +Phase 1 (Foundation) +├── 1.1 Database Schema ─────────────────────────────────────────┐ +├── 1.2 Thumbs Down Capture ──────────────────────────────────┐ │ +├── 1.3 Comment Capture ──────────────────────────────────────┤ │ +├── 1.4 Orchestrator Flow ◄───────────────────────────────────┤ │ +└── 1.5 DB Utilities ◄────────────────────────────────────────┘ │ + │ +Phase 2 (Core Agents) │ +├── 2.1 Feedback Intake Agent ◄───────────────────────────────────┤ +├── 2.2 Research Agent ◄──────────────────────────────────────────┤ +├── 2.3 Proposal Writer Agent ◄───────────────────────────────────┤ +├── 2.4 Experiment Runner Agent ◄─────────────────────────────────┤ +└── 2.5 Evaluation Agent ◄────────────────────────────────────────┘ + +Phase 3 (Experimentation) +├── 3.1 Semantic Search Agent +├── 3.2 Refinement Loop +└── 3.3 Control Snippet Selection + +Phase 4 (Deployment) +├── 4.1 Deployment Agent +├── 4.2 Knowledge Base Injection +├── 4.3 Reprocessing Worker +└── 4.4 Queue Population + +Phase 5 (Polish) +├── 5.1 Admin Dashboard +├── 5.2 Human Review Interface +├── 5.3 Metrics +└── 5.4 Slack Notifications +``` + +--- + +## Quick Start for Development + +1. **Run migration locally:** + ```bash + supabase db reset + # or + supabase migration up + ``` + +2. **Create agent module structure:** + ```bash + mkdir -p src/prompt_rewriter/agents + touch src/prompt_rewriter/__init__.py + touch src/prompt_rewriter/agents/__init__.py + ``` + +3. **Start with orchestrator and one agent:** + - Implement `main.py` orchestrator + - Implement `feedback_intake.py` agent + - Test end-to-end with mock data + +4. **Iterate:** + - Add agents one at a time + - Test each integration point + - Monitor logs and metrics diff --git a/src/prompt_rewriter/__init__.py b/src/prompt_rewriter/__init__.py new file mode 100644 index 0000000..c4ba23e --- /dev/null +++ b/src/prompt_rewriter/__init__.py @@ -0,0 +1,37 @@ +""" +Prompt Rewriter Agent System + +An autonomous system for improving prompts based on user feedback. +This module provides agents that analyze feedback, research claims, +propose prompt modifications, test changes, and deploy improvements. + +Usage: + from prompt_rewriter import PromptRewriterOrchestrator + + orchestrator = PromptRewriterOrchestrator() + result = await orchestrator.process_feedback(feedback_event) +""" + +from .models import ( + FeedbackEvent, + FeedbackIntent, + ProposalType, + ProposalStatus, + ResearchResult, + PromptProposal, + ExperimentResult, + EvaluationResult, +) + +__all__ = [ + "FeedbackEvent", + "FeedbackIntent", + "ProposalType", + "ProposalStatus", + "ResearchResult", + "PromptProposal", + "ExperimentResult", + "EvaluationResult", +] + +__version__ = "0.1.0" diff --git a/src/prompt_rewriter/agents/__init__.py b/src/prompt_rewriter/agents/__init__.py new file mode 100644 index 0000000..412aa2d --- /dev/null +++ b/src/prompt_rewriter/agents/__init__.py @@ -0,0 +1,17 @@ +""" +Prompt Rewriter Agents + +Each agent is responsible for a specific step in the prompt rewriting pipeline: + +1. FeedbackIntakeAgent - Analyzes user feedback to classify intent +2. ResearchAgent - Conducts web research on disputed claims +3. ProposalWriterAgent - Generates structured prompt modification proposals +4. ExperimentRunnerAgent - Tests proposals against real snippets +5. EvaluationAgent - Decides whether to accept, refine, or reject proposals +6. SemanticSearchAgent - Finds similar snippets for broader testing +7. DeploymentAgent - Applies approved changes to production +""" + +from .base import BaseAgent + +__all__ = ["BaseAgent"] diff --git a/src/prompt_rewriter/agents/base.py b/src/prompt_rewriter/agents/base.py new file mode 100644 index 0000000..f797a37 --- /dev/null +++ b/src/prompt_rewriter/agents/base.py @@ -0,0 +1,250 @@ +""" +Base agent class for the Prompt Rewriter system. + +All agents inherit from BaseAgent, which provides: +- Logging infrastructure +- LLM calling utilities +- Error handling and retry logic +- Database access +""" + +import logging +import time +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Any, Generic, TypeVar +from uuid import UUID + +from ..config import PromptRewriterConfig, default_config +from ..models import AgentLogEntry + +logger = logging.getLogger(__name__) + +InputT = TypeVar("InputT") +OutputT = TypeVar("OutputT") + + +class BaseAgent(ABC, Generic[InputT, OutputT]): + """ + Base class for all prompt rewriter agents. + + Subclasses must implement: + - name: str - The agent's name for logging + - run(input: InputT) -> OutputT - The main execution method + """ + + name: str = "base_agent" + + def __init__(self, config: PromptRewriterConfig | None = None): + self.config = config or default_config + self._supabase_client = None + + @property + def supabase(self): + """Lazy-load Supabase client.""" + if self._supabase_client is None: + from supabase import create_client + + self._supabase_client = create_client( + self.config.supabase_url, + self.config.supabase_key, + ) + return self._supabase_client + + @abstractmethod + async def run(self, input_data: InputT) -> OutputT: + """ + Execute the agent's main logic. + + Args: + input_data: Input data specific to this agent type + + Returns: + Output data specific to this agent type + """ + pass + + async def execute( + self, input_data: InputT, proposal_id: UUID | None = None + ) -> OutputT: + """ + Execute the agent with logging and error handling. + + This is the main entry point for running an agent. It wraps + the run() method with logging, timing, and error handling. + + Args: + input_data: Input data for the agent + proposal_id: Optional proposal ID for logging context + + Returns: + Output from the run() method + """ + log_entry = AgentLogEntry( + agent_name=self.name, + proposal_id=proposal_id or UUID("00000000-0000-0000-0000-000000000000"), + input_summary=self._summarize_input(input_data), + ) + + start_time = time.time() + logger.info(f"[{self.name}] Starting execution") + + try: + result = await self.run(input_data) + + elapsed_ms = int((time.time() - start_time) * 1000) + log_entry.completed_at = datetime.utcnow() + log_entry.duration_ms = elapsed_ms + log_entry.status = "completed" + log_entry.output_summary = self._summarize_output(result) + + logger.info(f"[{self.name}] Completed in {elapsed_ms}ms") + + if proposal_id and self.config.log_all_llm_calls: + await self._save_log_entry(log_entry) + + return result + + except Exception as e: + elapsed_ms = int((time.time() - start_time) * 1000) + log_entry.completed_at = datetime.utcnow() + log_entry.duration_ms = elapsed_ms + log_entry.status = "failed" + log_entry.error_message = str(e) + + logger.error(f"[{self.name}] Failed after {elapsed_ms}ms: {e}") + + if proposal_id and self.config.log_all_llm_calls: + await self._save_log_entry(log_entry) + + raise + + def _summarize_input(self, input_data: Any) -> str: + """Create a brief summary of input for logging.""" + if hasattr(input_data, "model_dump"): + data = input_data.model_dump() + return str(data)[:500] + return str(input_data)[:500] + + def _summarize_output(self, output_data: Any) -> str: + """Create a brief summary of output for logging.""" + if hasattr(output_data, "model_dump"): + data = output_data.model_dump() + return str(data)[:500] + return str(output_data)[:500] + + async def _save_log_entry(self, log_entry: AgentLogEntry) -> None: + """Save a log entry to the database.""" + try: + self.supabase.table("prompt_rewriter_agent_logs").insert( + { + "agent_name": log_entry.agent_name, + "proposal_id": str(log_entry.proposal_id), + "started_at": log_entry.started_at.isoformat(), + "completed_at": ( + log_entry.completed_at.isoformat() + if log_entry.completed_at + else None + ), + "duration_ms": log_entry.duration_ms, + "status": log_entry.status, + "input_data": {"summary": log_entry.input_summary}, + "output_data": {"summary": log_entry.output_summary}, + "error_message": log_entry.error_message, + "llm_total_tokens": log_entry.llm_tokens_used, + } + ).execute() + except Exception as e: + logger.warning(f"[{self.name}] Failed to save log entry: {e}") + + async def call_llm( + self, + prompt: str, + system_instruction: str | None = None, + model: str | None = None, + temperature: float = 0.7, + max_tokens: int = 4096, + ) -> str: + """ + Call an LLM with the given prompt. + + This is a utility method for subclasses to use when they need + to call an LLM. It handles model selection and basic error handling. + + Args: + prompt: The user prompt + system_instruction: Optional system instruction + model: Model to use (defaults to config.default_llm_model) + temperature: Sampling temperature + max_tokens: Maximum tokens to generate + + Returns: + The LLM's response text + """ + import google.generativeai as genai + + model_name = model or self.config.default_llm_model + + generation_config = genai.GenerationConfig( + temperature=temperature, + max_output_tokens=max_tokens, + ) + + model_instance = genai.GenerativeModel( + model_name=model_name, + system_instruction=system_instruction, + generation_config=generation_config, + ) + + response = model_instance.generate_content(prompt) + return response.text + + async def call_llm_with_search( + self, + prompt: str, + system_instruction: str | None = None, + model: str | None = None, + ) -> tuple[str, list[dict]]: + """ + Call an LLM with Google Search grounding enabled. + + Args: + prompt: The user prompt + system_instruction: Optional system instruction + model: Model to use + + Returns: + Tuple of (response_text, grounding_sources) + """ + import google.generativeai as genai + from google.generativeai.types import GenerateContentConfig, Tool + from google.generativeai.types.content_types import GoogleSearch + + model_name = model or self.config.research_model + + tools = [Tool(google_search=GoogleSearch())] + + config = GenerateContentConfig( + tools=tools, + system_instruction=system_instruction, + ) + + model_instance = genai.GenerativeModel(model_name=model_name) + + response = model_instance.generate_content(prompt, generation_config=config) + + # Extract grounding metadata + sources = [] + if hasattr(response.candidates[0], "grounding_metadata"): + metadata = response.candidates[0].grounding_metadata + if hasattr(metadata, "grounding_chunks"): + for chunk in metadata.grounding_chunks: + if hasattr(chunk, "web"): + sources.append( + { + "url": chunk.web.uri, + "title": chunk.web.title, + } + ) + + return response.text, sources diff --git a/src/prompt_rewriter/agents/feedback_intake.py b/src/prompt_rewriter/agents/feedback_intake.py new file mode 100644 index 0000000..33bd784 --- /dev/null +++ b/src/prompt_rewriter/agents/feedback_intake.py @@ -0,0 +1,164 @@ +""" +Feedback Intake Agent + +Analyzes user feedback to classify intent and extract actionable information. +This is the first agent in the pipeline, transforming raw user feedback +into structured data for the Research Agent. +""" + +import json +import logging +from uuid import UUID + +from ..config import PromptRewriterConfig +from ..models import FeedbackAnalysis, FeedbackEvent, FeedbackIntent +from .base import BaseAgent + +logger = logging.getLogger(__name__) + +FEEDBACK_CLASSIFICATION_PROMPT = """You are analyzing user feedback on a misinformation detection system. +The system analyzes audio snippets from radio broadcasts and identifies potential misinformation. + +A user has provided feedback on a snippet analysis. Your job is to: +1. Classify the intent of the feedback +2. Extract any specific claims the user is disputing +3. Determine what the user believes the correct interpretation should be +4. Identify which parts of the analysis pipeline may need adjustment + +## Snippet Context +Title: {title} +Summary: {summary} +Categories detected: {categories} +Original transcription: {transcription} + +## User Feedback +Feedback type: {feedback_type} +Content: {feedback_content} + +## Classification Instructions + +Classify the feedback intent as one of: +- factual_error: The analysis contains incorrect factual claims or misidentifies something as misinformation when it's true (or vice versa) +- missing_context: The analysis lacks important context that would change the interpretation +- wrong_category: The disinformation category assignment is incorrect +- false_positive: This content was incorrectly flagged as misinformation +- false_negative: This content should have been flagged but wasn't +- unclear_explanation: The explanation is confusing or poorly written +- translation_error: The Spanish/English translation is incorrect +- other: Feedback doesn't fit other categories + +## Response Format +Respond with a JSON object: +{{ + "intent": "", + "intent_confidence": <0.0-1.0>, + "extracted_claim": "", + "user_correction": "", + "affected_prompt_stages": [], + "priority": "", + "reasoning": "" +}} + +Priority guidelines: +- critical: Dangerous misinformation being spread as truth, or truth being labeled as dangerous misinformation +- high: Clear factual errors that could mislead users +- medium: Missing context or unclear explanations +- low: Minor issues, translation tweaks, category refinements +""" + + +class FeedbackIntakeAgent(BaseAgent[FeedbackEvent, FeedbackAnalysis]): + """ + Analyzes user feedback to determine intent and extract actionable information. + + Input: FeedbackEvent (raw user feedback) + Output: FeedbackAnalysis (structured classification and extracted claims) + """ + + name = "feedback_intake" + + def __init__(self, config: PromptRewriterConfig | None = None): + super().__init__(config) + + async def run(self, feedback: FeedbackEvent) -> FeedbackAnalysis: + """ + Analyze the feedback and return structured classification. + """ + # Fetch snippet context from database + snippet = await self._get_snippet(feedback.snippet_id) + + if not snippet: + logger.warning(f"Snippet {feedback.snippet_id} not found") + return FeedbackAnalysis( + intent=FeedbackIntent.OTHER, + intent_confidence=0.5, + reasoning="Could not find snippet context", + ) + + # Build the classification prompt + prompt = FEEDBACK_CLASSIFICATION_PROMPT.format( + title=snippet.get("title", "Unknown"), + summary=snippet.get("summary", "No summary available"), + categories=", ".join(snippet.get("disinformation_categories", [])), + transcription=snippet.get("transcription", "")[:1000], # Truncate + feedback_type=feedback.feedback_type, + feedback_content=feedback.content or "No content provided", + ) + + # Call LLM for classification + response = await self.call_llm( + prompt=prompt, + model=self.config.feedback_intake_model, + temperature=0.3, # Lower temperature for classification + ) + + # Parse the response + return self._parse_response(response) + + async def _get_snippet(self, snippet_id: UUID) -> dict | None: + """Fetch snippet details from database.""" + try: + result = ( + self.supabase.table("snippets") + .select("id, title, summary, transcription, disinformation_categories") + .eq("id", str(snippet_id)) + .single() + .execute() + ) + return result.data + except Exception as e: + logger.error(f"Error fetching snippet {snippet_id}: {e}") + return None + + def _parse_response(self, response: str) -> FeedbackAnalysis: + """Parse LLM response into FeedbackAnalysis.""" + try: + # Extract JSON from response (handle markdown code blocks) + json_str = response + if "```json" in response: + json_str = response.split("```json")[1].split("```")[0] + elif "```" in response: + json_str = response.split("```")[1].split("```")[0] + + data = json.loads(json_str.strip()) + + return FeedbackAnalysis( + intent=FeedbackIntent(data.get("intent", "other")), + intent_confidence=float(data.get("intent_confidence", 0.5)), + extracted_claim=data.get("extracted_claim"), + user_correction=data.get("user_correction"), + affected_prompt_stages=data.get("affected_prompt_stages", [1, 3]), + priority=data.get("priority", "medium"), + reasoning=data.get("reasoning"), + ) + + except (json.JSONDecodeError, KeyError, ValueError) as e: + logger.error(f"Failed to parse LLM response: {e}") + logger.debug(f"Raw response: {response}") + + # Return a default analysis on parse failure + return FeedbackAnalysis( + intent=FeedbackIntent.OTHER, + intent_confidence=0.3, + reasoning=f"Failed to parse LLM response: {e}", + ) diff --git a/src/prompt_rewriter/config.py b/src/prompt_rewriter/config.py new file mode 100644 index 0000000..b4f9a85 --- /dev/null +++ b/src/prompt_rewriter/config.py @@ -0,0 +1,119 @@ +""" +Configuration for the Prompt Rewriter Agent system. +""" + +import os +from dataclasses import dataclass, field + + +@dataclass +class ResearchConfig: + """Configuration for the Research Agent.""" + + max_search_attempts: int = 5 + min_sources_required: int = 3 + retry_delay_seconds: float = 2.0 + retry_backoff_multiplier: float = 2.0 + credibility_threshold: float = 0.6 + timeout_seconds: int = 30 + + # Source credibility weights + source_weights: dict = field( + default_factory=lambda: { + "gov": 0.95, + "edu": 0.90, + "factcheck": 0.85, + "peer_reviewed": 0.90, + "major_news": 0.75, + "other": 0.50, + } + ) + + +@dataclass +class ExperimentConfig: + """Configuration for the Experiment Runner Agent.""" + + runs_per_variant: int = 5 + timeout_per_run_seconds: int = 120 + min_consistency_threshold: float = 0.80 + min_improvement_threshold: float = 0.10 + + +@dataclass +class EvaluationConfig: + """Configuration for the Evaluation Agent.""" + + min_accuracy_improvement: float = 0.10 + max_regression_tolerance: float = 0.05 + confidence_threshold_for_auto_deploy: float = 0.90 + max_refinement_iterations: int = 3 + + +@dataclass +class DeploymentConfig: + """Configuration for the Deployment Agent.""" + + max_snippets_to_reprocess: int = 1000 + reprocess_similarity_threshold: float = 0.75 + slack_webhook_url: str = "" + notify_on_deploy: bool = True + require_human_review_for_high_impact: bool = True + high_impact_snippet_threshold: int = 100 + + +@dataclass +class PromptRewriterConfig: + """Main configuration for the Prompt Rewriter system.""" + + # Sub-agent configs + research: ResearchConfig = field(default_factory=ResearchConfig) + experiment: ExperimentConfig = field(default_factory=ExperimentConfig) + evaluation: EvaluationConfig = field(default_factory=EvaluationConfig) + deployment: DeploymentConfig = field(default_factory=DeploymentConfig) + + # LLM settings + default_llm_model: str = "gemini-2.5-pro" + feedback_intake_model: str = "gemini-2.5-flash" + research_model: str = "gemini-2.5-pro" + proposal_writer_model: str = "gemini-2.5-pro" + evaluator_model: str = "gemini-2.5-pro" + + # Prompt file paths (relative to project root) + prompts_dir: str = "prompts" + stage_1_heuristics_file: str = "Stage_1_heuristics.md" + stage_3_heuristics_file: str = "Stage_3_heuristics.md" + + # Database + supabase_url: str = "" + supabase_key: str = "" + + # Feature flags + auto_deploy_enabled: bool = False # Require human review by default + reprocess_on_deploy: bool = True + log_all_llm_calls: bool = True + + @classmethod + def from_env(cls) -> "PromptRewriterConfig": + """Create config from environment variables.""" + config = cls() + + # Override with environment variables if set + if url := os.getenv("SUPABASE_URL"): + config.supabase_url = url + if key := os.getenv("SUPABASE_SERVICE_ROLE_KEY"): + config.supabase_key = key + if slack := os.getenv("SLACK_WEBHOOK_URL"): + config.deployment.slack_webhook_url = slack + if model := os.getenv("PROMPT_REWRITER_LLM_MODEL"): + config.default_llm_model = model + + # Feature flags from env + if os.getenv("PROMPT_REWRITER_AUTO_DEPLOY", "").lower() == "true": + config.auto_deploy_enabled = True + + return config + + +# Default configuration instance +default_config = PromptRewriterConfig.from_env() diff --git a/src/prompt_rewriter/main.py b/src/prompt_rewriter/main.py new file mode 100644 index 0000000..b54aa96 --- /dev/null +++ b/src/prompt_rewriter/main.py @@ -0,0 +1,407 @@ +""" +Prompt Rewriter Agent Orchestrator + +This module contains the main Prefect flow that orchestrates the prompt +rewriting pipeline. It coordinates the execution of all agents in sequence +and handles state transitions. + +Usage: + # Trigger manually + await prompt_rewriter_flow(proposal_id=uuid) + + # Or via the continuous worker + prefect deployment run "Prompt Rewriter Agent/default" +""" + +import logging +from uuid import UUID + +from prefect import flow, task +from prefect.task_runners import ConcurrentTaskRunner + +from .agents.feedback_intake import FeedbackIntakeAgent +from .config import PromptRewriterConfig, default_config +from .models import ( + EvaluationDecision, + FeedbackAnalysis, + FeedbackEvent, + ProposalStatus, + PromptProposal, + ResearchResult, +) + +logger = logging.getLogger(__name__) + + +@task(name="Load Proposal", retries=2, retry_delay_seconds=5) +async def load_proposal(proposal_id: UUID, config: PromptRewriterConfig) -> dict: + """Load a proposal from the database.""" + from supabase import create_client + + client = create_client(config.supabase_url, config.supabase_key) + + result = ( + client.table("prompt_rewrite_proposals") + .select("*") + .eq("id", str(proposal_id)) + .single() + .execute() + ) + + return result.data + + +@task(name="Update Proposal Status") +async def update_proposal_status( + proposal_id: UUID, + status: ProposalStatus, + config: PromptRewriterConfig, + additional_data: dict | None = None, +) -> None: + """Update the status of a proposal in the database.""" + from supabase import create_client + + client = create_client(config.supabase_url, config.supabase_key) + + update_data = {"status": status.value} + if additional_data: + update_data.update(additional_data) + + client.table("prompt_rewrite_proposals").update(update_data).eq( + "id", str(proposal_id) + ).execute() + + +@task(name="Analyze Feedback") +async def analyze_feedback( + proposal_data: dict, + config: PromptRewriterConfig, +) -> FeedbackAnalysis: + """Run the Feedback Intake Agent.""" + agent = FeedbackIntakeAgent(config) + + feedback = FeedbackEvent( + id=UUID(proposal_data["id"]), + feedback_type=proposal_data["triggered_by_feedback_type"], + snippet_id=UUID(proposal_data["triggered_by_snippet_id"]), + user_id=( + UUID(proposal_data["triggered_by_user_id"]) + if proposal_data.get("triggered_by_user_id") + else None + ), + content=proposal_data.get("trigger_content"), + comment_id=proposal_data.get("triggered_by_comment_id"), + ) + + return await agent.execute(feedback, proposal_id=UUID(proposal_data["id"])) + + +@task(name="Research Claim") +async def research_claim( + analysis: FeedbackAnalysis, + proposal_id: UUID, + config: PromptRewriterConfig, +) -> ResearchResult: + """Run the Research Agent.""" + # TODO: Implement ResearchAgent + # For now, return a placeholder + logger.info(f"Research agent would research: {analysis.extracted_claim}") + + return ResearchResult( + claim=analysis.extracted_claim or "Unknown claim", + verdict="inconclusive", + confidence=0.5, + sources=[], + summary="Research agent not yet implemented", + research_complete=False, + ) + + +@task(name="Write Proposal") +async def write_proposal( + analysis: FeedbackAnalysis, + research: ResearchResult, + proposal_id: UUID, + config: PromptRewriterConfig, +) -> PromptProposal: + """Run the Proposal Writer Agent.""" + # TODO: Implement ProposalWriterAgent + # For now, return a placeholder + logger.info("Proposal writer would generate prompt modifications") + + from .models import ProposalChange, ProposalType + + return PromptProposal( + proposal_id=proposal_id, + proposal_type=ProposalType.FACTUAL_ADDITION, + target_prompts=["Stage_1_heuristics.md"], + changes=[ + ProposalChange( + file="Stage_1_heuristics.md", + section="## Verified Facts", + action="append", + content=f"- Claim: {research.claim}\n Fact: {research.summary}", + rationale="Based on user feedback and research", + ) + ], + expected_impact="Improve detection accuracy for this claim type", + ) + + +@task(name="Run Experiments") +async def run_experiments( + proposal: PromptProposal, + config: PromptRewriterConfig, +) -> dict: + """Run the Experiment Runner Agent.""" + # TODO: Implement ExperimentRunnerAgent + logger.info("Experiment runner would test the proposal") + + return { + "baseline_accuracy": 0.0, + "proposal_accuracy": 0.8, + "improvement": 0.8, + "consistency_score": 0.9, + } + + +@task(name="Evaluate Results") +async def evaluate_results( + experiment_results: dict, + config: PromptRewriterConfig, +) -> dict: + """Run the Evaluation Agent.""" + # TODO: Implement EvaluationAgent + improvement = experiment_results.get("improvement", 0) + consistency = experiment_results.get("consistency_score", 0) + + if improvement >= config.evaluation.min_accuracy_improvement and consistency >= 0.8: + decision = EvaluationDecision.ACCEPT + elif improvement > 0: + decision = EvaluationDecision.REFINE + else: + decision = EvaluationDecision.REJECT + + return { + "decision": decision.value, + "confidence": 0.8, + "issues_resolved": improvement > 0, + "regressions_detected": False, + "human_review_required": not config.auto_deploy_enabled, + } + + +@task(name="Deploy Changes") +async def deploy_changes( + proposal: PromptProposal, + config: PromptRewriterConfig, +) -> dict: + """Run the Deployment Agent.""" + # TODO: Implement DeploymentAgent + logger.info("Deployment agent would apply changes") + + return { + "success": True, + "prompt_version_id": None, + "snippets_queued_for_reprocess": 0, + } + + +@flow( + name="Prompt Rewriter Agent", + task_runner=ConcurrentTaskRunner(), + description="Orchestrates the prompt rewriting pipeline from feedback to deployment", +) +async def prompt_rewriter_flow( + proposal_id: UUID, + config: PromptRewriterConfig | None = None, +) -> dict: + """ + Main orchestrator flow for the Prompt Rewriter Agent system. + + This flow coordinates the execution of all agents: + 1. Load proposal from database + 2. Analyze feedback (Feedback Intake Agent) + 3. Research the disputed claim (Research Agent) + 4. Generate proposal (Proposal Writer Agent) + 5. Run experiments (Experiment Runner Agent) + 6. Evaluate results (Evaluation Agent) + 7. Deploy if approved (Deployment Agent) + + Args: + proposal_id: UUID of the proposal to process + config: Optional configuration override + + Returns: + Dictionary with final status and results + """ + config = config or default_config + + logger.info(f"Starting prompt rewriter flow for proposal {proposal_id}") + + # 1. Load proposal + proposal_data = await load_proposal(proposal_id, config) + + if not proposal_data: + logger.error(f"Proposal {proposal_id} not found") + return {"status": "failed", "error": "Proposal not found"} + + # 2. Analyze feedback + await update_proposal_status( + proposal_id, ProposalStatus.ANALYZING_FEEDBACK, config + ) + analysis = await analyze_feedback(proposal_data, config) + + await update_proposal_status( + proposal_id, + ProposalStatus.RESEARCHING, + config, + { + "intent_classification": analysis.intent.value, + "intent_confidence": analysis.intent_confidence, + "extracted_claim": analysis.extracted_claim, + "user_correction": analysis.user_correction, + "priority": analysis.priority, + }, + ) + + # 3. Research the claim + research = await research_claim(analysis, proposal_id, config) + + await update_proposal_status( + proposal_id, + ProposalStatus.WRITING_PROPOSAL, + config, + { + "research_summary": research.summary, + "research_sources": [s.model_dump() for s in research.sources], + }, + ) + + # 4. Generate proposal + proposal = await write_proposal(analysis, research, proposal_id, config) + + await update_proposal_status( + proposal_id, + ProposalStatus.EXPERIMENTING, + config, + { + "proposal_type": proposal.proposal_type.value, + "proposal_changes": [c.model_dump() for c in proposal.changes], + "expected_impact": proposal.expected_impact, + }, + ) + + # 5. Run experiments + experiment_results = await run_experiments(proposal, config) + + await update_proposal_status( + proposal_id, + ProposalStatus.EVALUATING, + config, + { + "experiment_results": experiment_results, + "baseline_accuracy": experiment_results.get("baseline_accuracy"), + "proposal_accuracy": experiment_results.get("proposal_accuracy"), + }, + ) + + # 6. Evaluate results + evaluation = await evaluate_results(experiment_results, config) + + # 7. Handle evaluation decision + if evaluation["decision"] == EvaluationDecision.ACCEPT.value: + if evaluation["human_review_required"]: + await update_proposal_status( + proposal_id, + ProposalStatus.AWAITING_REVIEW, + config, + {"evaluation_decision": evaluation["decision"]}, + ) + logger.info(f"Proposal {proposal_id} awaiting human review") + return {"status": "awaiting_review", "evaluation": evaluation} + else: + # Auto-deploy + await update_proposal_status( + proposal_id, ProposalStatus.DEPLOYING, config + ) + deployment = await deploy_changes(proposal, config) + + if deployment["success"]: + await update_proposal_status( + proposal_id, ProposalStatus.DEPLOYED, config + ) + logger.info(f"Proposal {proposal_id} deployed successfully") + return {"status": "deployed", "deployment": deployment} + else: + await update_proposal_status( + proposal_id, + ProposalStatus.FAILED, + config, + {"error_message": "Deployment failed"}, + ) + return {"status": "failed", "error": "Deployment failed"} + + elif evaluation["decision"] == EvaluationDecision.REFINE.value: + # TODO: Implement refinement loop + await update_proposal_status( + proposal_id, + ProposalStatus.AWAITING_REVIEW, + config, + { + "evaluation_decision": evaluation["decision"], + "human_review_required": True, + }, + ) + logger.info(f"Proposal {proposal_id} needs refinement, escalating to review") + return {"status": "awaiting_review", "evaluation": evaluation} + + else: # REJECT + await update_proposal_status( + proposal_id, + ProposalStatus.REJECTED, + config, + {"evaluation_decision": evaluation["decision"]}, + ) + logger.info(f"Proposal {proposal_id} rejected") + return {"status": "rejected", "evaluation": evaluation} + + +@flow(name="Process Pending Proposals") +async def process_pending_proposals(config: PromptRewriterConfig | None = None): + """ + Continuous worker flow that processes pending proposals. + + This flow fetches the next pending proposal and processes it. + It should be deployed as a continuous worker. + """ + config = config or default_config + + from supabase import create_client + + client = create_client(config.supabase_url, config.supabase_key) + + # Get next pending proposal using the database function + result = client.rpc("get_next_pending_proposal").execute() + + if not result.data: + logger.info("No pending proposals to process") + return {"status": "no_work"} + + proposal_id = UUID(result.data) + logger.info(f"Processing proposal {proposal_id}") + + return await prompt_rewriter_flow(proposal_id, config) + + +# Entry point for manual testing +if __name__ == "__main__": + import asyncio + import sys + + if len(sys.argv) > 1: + proposal_id = UUID(sys.argv[1]) + result = asyncio.run(prompt_rewriter_flow(proposal_id)) + print(f"Result: {result}") + else: + print("Usage: python -m prompt_rewriter.main ") diff --git a/src/prompt_rewriter/models.py b/src/prompt_rewriter/models.py new file mode 100644 index 0000000..b3e2abc --- /dev/null +++ b/src/prompt_rewriter/models.py @@ -0,0 +1,205 @@ +""" +Pydantic models for the Prompt Rewriter Agent system. + +These models define the data structures passed between agents +and stored in the database. +""" + +from datetime import datetime +from enum import Enum +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, Field + + +class FeedbackIntent(str, Enum): + """Classification of user feedback intent.""" + + FACTUAL_ERROR = "factual_error" + MISSING_CONTEXT = "missing_context" + WRONG_CATEGORY = "wrong_category" + FALSE_POSITIVE = "false_positive" + FALSE_NEGATIVE = "false_negative" + UNCLEAR_EXPLANATION = "unclear_explanation" + TRANSLATION_ERROR = "translation_error" + OTHER = "other" + + +class ProposalType(str, Enum): + """Type of prompt modification proposal.""" + + FACTUAL_ADDITION = "factual_addition" + HEURISTIC_UPDATE = "heuristic_update" + INSTRUCTION_CLARIFICATION = "instruction_clarification" + CATEGORY_ADDITION = "category_addition" + CATEGORY_REMOVAL = "category_removal" + PROMPT_REWRITE = "prompt_rewrite" + + +class ProposalStatus(str, Enum): + """Status of a prompt rewrite proposal.""" + + PENDING = "pending" + ANALYZING_FEEDBACK = "analyzing_feedback" + RESEARCHING = "researching" + WRITING_PROPOSAL = "writing_proposal" + EXPERIMENTING = "experimenting" + EVALUATING = "evaluating" + REFINING = "refining" + AWAITING_REVIEW = "awaiting_review" + DEPLOYING = "deploying" + DEPLOYED = "deployed" + REJECTED = "rejected" + FAILED = "failed" + + +class EvaluationDecision(str, Enum): + """Decision from the evaluation agent.""" + + ACCEPT = "accept" + REFINE = "refine" + REJECT = "reject" + + +class FeedbackEvent(BaseModel): + """A user feedback event that may trigger the rewriter.""" + + id: Optional[UUID] = None + feedback_type: str # 'thumbs_down', 'comment', 'label_dispute', 'manual' + snippet_id: UUID + user_id: Optional[UUID] = None + content: Optional[str] = None # Comment text or feedback details + comment_id: Optional[str] = None # Liveblocks comment ID + label_id: Optional[UUID] = None + created_at: datetime = Field(default_factory=datetime.utcnow) + + +class FeedbackAnalysis(BaseModel): + """Output from the Feedback Intake Agent.""" + + intent: FeedbackIntent + intent_confidence: float = Field(ge=0, le=1) + extracted_claim: Optional[str] = None + user_correction: Optional[str] = None + affected_prompt_stages: list[int] = Field(default_factory=lambda: [1, 3]) + priority: str = "medium" # 'low', 'medium', 'high', 'critical' + reasoning: Optional[str] = None + + +class ResearchSource(BaseModel): + """A source found during research.""" + + url: str + title: str + credibility_score: float = Field(ge=0, le=1) + excerpt: Optional[str] = None + publication_date: Optional[str] = None + source_type: Optional[str] = None # 'gov', 'edu', 'news', 'factcheck', etc. + + +class ResearchResult(BaseModel): + """Output from the Research Agent.""" + + claim: str + verdict: str # 'confirmed', 'debunked', 'inconclusive' + confidence: float = Field(ge=0, le=1) + sources: list[ResearchSource] = Field(default_factory=list) + summary: str + research_complete: bool = True + research_attempts: int = 1 + error_message: Optional[str] = None + + +class ProposalChange(BaseModel): + """A single change to apply to a prompt file.""" + + file: str # e.g., 'Stage_1_heuristics.md' + section: Optional[str] = None # e.g., '## COVID-19 / Vaccines' + action: str # 'append', 'replace', 'insert_before', 'insert_after' + content: str + rationale: str + + +class PromptProposal(BaseModel): + """Output from the Proposal Writer Agent.""" + + proposal_id: Optional[UUID] = None + proposal_type: ProposalType + target_prompts: list[str] # List of prompt files to modify + changes: list[ProposalChange] + expected_impact: str + test_snippet_ids: list[UUID] = Field(default_factory=list) + control_snippet_ids: list[UUID] = Field(default_factory=list) + + +class ExperimentRun(BaseModel): + """A single experiment run result.""" + + run_id: int + run_type: str # 'baseline' or 'proposal' + snippet_id: UUID + llm_output: dict + is_correct: bool + correctness_confidence: float = Field(ge=0, le=1) + duration_ms: int + + +class ExperimentResult(BaseModel): + """Output from the Experiment Runner Agent.""" + + experiment_id: Optional[UUID] = None + proposal_id: UUID + snippet_id: UUID + baseline_runs: list[ExperimentRun] = Field(default_factory=list) + proposal_runs: list[ExperimentRun] = Field(default_factory=list) + baseline_accuracy: float = Field(ge=0, le=1) + proposal_accuracy: float = Field(ge=0, le=1) + improvement: float # Can be negative if regression + consistency_score: float = Field(ge=0, le=1) + + +class EvaluationResult(BaseModel): + """Output from the Evaluation Agent.""" + + decision: EvaluationDecision + confidence: float = Field(ge=0, le=1) + issues_resolved: bool + regressions_detected: bool = False + refinement_suggestions: Optional[str] = None + human_review_required: bool = False + reasoning: str + + +class SemanticSearchResult(BaseModel): + """Output from the Semantic Search Agent.""" + + query_snippet_id: UUID + similar_snippets: list[dict] # [{id, similarity, title}, ...] + reprocess_candidates: list[UUID] + estimated_impact: int + + +class DeploymentResult(BaseModel): + """Output from the Deployment Agent.""" + + success: bool + prompt_version_id: Optional[UUID] = None + knowledge_fact_ids: list[UUID] = Field(default_factory=list) + snippets_queued_for_reprocess: int = 0 + error_message: Optional[str] = None + + +class AgentLogEntry(BaseModel): + """Log entry for agent execution.""" + + agent_name: str + proposal_id: UUID + started_at: datetime = Field(default_factory=datetime.utcnow) + completed_at: Optional[datetime] = None + duration_ms: Optional[int] = None + status: str = "running" # 'running', 'completed', 'failed', 'retrying' + input_summary: Optional[str] = None + output_summary: Optional[str] = None + llm_tokens_used: Optional[int] = None + error_message: Optional[str] = None diff --git a/supabase/migrations/20260124000000_prompt_rewriter_agent_schema.sql b/supabase/migrations/20260124000000_prompt_rewriter_agent_schema.sql new file mode 100644 index 0000000..06b9c43 --- /dev/null +++ b/supabase/migrations/20260124000000_prompt_rewriter_agent_schema.sql @@ -0,0 +1,429 @@ +-- Prompt Rewriter Agent Schema +-- Migration for the autonomous prompt improvement system + +-- Create enum for proposal status tracking +CREATE TYPE prompt_rewrite_status AS ENUM ( + 'pending', + 'analyzing_feedback', + 'researching', + 'writing_proposal', + 'experimenting', + 'evaluating', + 'refining', + 'awaiting_review', + 'deploying', + 'deployed', + 'rejected', + 'failed' +); + +-- Create enum for feedback intent classification +CREATE TYPE feedback_intent AS ENUM ( + 'factual_error', + 'missing_context', + 'wrong_category', + 'false_positive', + 'false_negative', + 'unclear_explanation', + 'translation_error', + 'other' +); + +-- Create enum for proposal types +CREATE TYPE proposal_type AS ENUM ( + 'factual_addition', + 'heuristic_update', + 'instruction_clarification', + 'category_addition', + 'category_removal', + 'prompt_rewrite' +); + +-- Main table to track prompt rewrite proposals +CREATE TABLE prompt_rewrite_proposals ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Trigger information + triggered_by_feedback_type TEXT NOT NULL, -- 'thumbs_down', 'comment', 'label_dispute', 'manual' + triggered_by_snippet_id UUID REFERENCES snippets(id), + triggered_by_user_id UUID REFERENCES auth.users(id), + triggered_by_comment_id TEXT, -- Liveblocks comment ID if applicable + trigger_content TEXT, -- The actual comment/feedback text + + -- Feedback analysis results + intent_classification feedback_intent, + intent_confidence FLOAT, + extracted_claim TEXT, + user_correction TEXT, + affected_prompt_stages INTEGER[], -- e.g., [1, 3] for Stage 1 and Stage 3 + priority TEXT DEFAULT 'medium', -- 'low', 'medium', 'high', 'critical' + + -- Research results + research_started_at TIMESTAMPTZ, + research_completed_at TIMESTAMPTZ, + research_attempts INTEGER DEFAULT 0, + research_summary TEXT, + research_verdict TEXT, -- 'confirmed', 'debunked', 'inconclusive' + research_confidence FLOAT, + research_sources JSONB, -- Array of {url, title, credibility_score, excerpt, date} + + -- Proposal details + proposal_type proposal_type, + proposal_changes JSONB, -- Structured changes to apply + expected_impact TEXT, + + -- Experiment configuration and results + test_snippet_ids UUID[], -- Snippets used for testing + control_snippet_ids UUID[], -- Control snippets for regression testing + experiment_runs INTEGER DEFAULT 5, + experiment_results JSONB, + baseline_accuracy FLOAT, + proposal_accuracy FLOAT, + improvement_score FLOAT, + consistency_score FLOAT, + + -- Evaluation + evaluation_decision TEXT, -- 'accept', 'refine', 'reject' + evaluation_confidence FLOAT, + refinement_count INTEGER DEFAULT 0, + max_refinements INTEGER DEFAULT 3, + regression_detected BOOLEAN DEFAULT false, + human_review_required BOOLEAN DEFAULT false, + human_review_notes TEXT, + reviewed_by UUID REFERENCES auth.users(id), + reviewed_at TIMESTAMPTZ, + + -- Deployment + deployed_at TIMESTAMPTZ, + deployed_prompt_version_id UUID, -- Reference to prompt_versions if it exists + rollback_at TIMESTAMPTZ, + rollback_reason TEXT, + + -- Reprocessing tracking + reprocess_snippet_count INTEGER DEFAULT 0, + reprocess_completed_count INTEGER DEFAULT 0, + + -- Status and errors + status prompt_rewrite_status DEFAULT 'pending', + current_agent TEXT, -- Which agent is currently processing + error_message TEXT, + error_details JSONB, + + -- Timestamps + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +-- Index for finding proposals by status +CREATE INDEX idx_prompt_rewrite_proposals_status ON prompt_rewrite_proposals(status); + +-- Index for finding proposals by snippet +CREATE INDEX idx_prompt_rewrite_proposals_snippet ON prompt_rewrite_proposals(triggered_by_snippet_id); + +-- Index for finding proposals by user +CREATE INDEX idx_prompt_rewrite_proposals_user ON prompt_rewrite_proposals(triggered_by_user_id); + +-- Track individual experiment runs +CREATE TABLE prompt_experiment_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + proposal_id UUID NOT NULL REFERENCES prompt_rewrite_proposals(id) ON DELETE CASCADE, + snippet_id UUID NOT NULL REFERENCES snippets(id), + run_type TEXT NOT NULL, -- 'baseline' or 'proposal' + run_number INTEGER NOT NULL, + + -- Input + prompt_version TEXT, -- 'current' or 'proposed' + prompt_content_hash TEXT, -- Hash of prompt used + + -- Output + llm_output JSONB, + llm_model TEXT, + llm_tokens_used INTEGER, + + -- Evaluation + is_correct BOOLEAN, + correctness_confidence FLOAT, + evaluation_notes TEXT, + + -- Timing + started_at TIMESTAMPTZ DEFAULT NOW(), + completed_at TIMESTAMPTZ, + duration_ms INTEGER +); + +CREATE INDEX idx_experiment_runs_proposal ON prompt_experiment_runs(proposal_id); + +-- Track snippets queued for reprocessing +CREATE TABLE snippet_reprocess_queue ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + snippet_id UUID NOT NULL REFERENCES snippets(id), + proposal_id UUID NOT NULL REFERENCES prompt_rewrite_proposals(id) ON DELETE CASCADE, + + -- Why this snippet was queued + reason TEXT NOT NULL, -- 'semantic_similarity', 'same_category', 'manual' + similarity_score FLOAT, -- If queued due to semantic similarity + + -- Processing + priority INTEGER DEFAULT 0, -- Higher = process first + status TEXT DEFAULT 'queued', -- 'queued', 'processing', 'completed', 'failed', 'skipped' + + -- Results + original_output JSONB, -- Store original analysis for comparison + reprocessed_output JSONB, + output_changed BOOLEAN, + improvement_detected BOOLEAN, + + -- Timestamps + queued_at TIMESTAMPTZ DEFAULT NOW(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + + -- Errors + error_message TEXT, + retry_count INTEGER DEFAULT 0 +); + +CREATE INDEX idx_reprocess_queue_status ON snippet_reprocess_queue(status); +CREATE INDEX idx_reprocess_queue_proposal ON snippet_reprocess_queue(proposal_id); +CREATE INDEX idx_reprocess_queue_priority ON snippet_reprocess_queue(priority DESC, queued_at ASC); + +-- Knowledge base for verified facts +-- This enables future RAG-based retrieval when the knowledge base grows large +CREATE TABLE knowledge_facts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Classification + category TEXT NOT NULL, -- 'election', 'covid', 'immigration', 'climate', etc. + subcategory TEXT, + + -- The fact itself + claim TEXT NOT NULL, -- The misinformation claim + fact TEXT NOT NULL, -- The verified truth + fact_summary TEXT, -- One-line summary for quick injection + + -- Supporting evidence + sources JSONB, -- Array of source objects + confidence_score FLOAT DEFAULT 0.9, + + -- Provenance + added_by_proposal_id UUID REFERENCES prompt_rewrite_proposals(id), + added_by_user_id UUID REFERENCES auth.users(id), + + -- Version control + version INTEGER DEFAULT 1, + previous_version_id UUID REFERENCES knowledge_facts(id), + + -- Usage tracking + times_used INTEGER DEFAULT 0, + last_used_at TIMESTAMPTZ, + effectiveness_score FLOAT, -- How often this fact leads to correct analysis + + -- Status + is_active BOOLEAN DEFAULT true, + reviewed_by UUID REFERENCES auth.users(id), + reviewed_at TIMESTAMPTZ, + + -- Timestamps + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX idx_knowledge_facts_category ON knowledge_facts(category); +CREATE INDEX idx_knowledge_facts_active ON knowledge_facts(is_active) WHERE is_active = true; + +-- Full-text search on claims and facts +CREATE INDEX idx_knowledge_facts_claim_fts ON knowledge_facts USING GIN (to_tsvector('english', claim)); +CREATE INDEX idx_knowledge_facts_fact_fts ON knowledge_facts USING GIN (to_tsvector('english', fact)); + +-- Track feedback that triggers the rewriter +-- This provides a unified view of all feedback types +CREATE TABLE user_feedback_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- What type of feedback + feedback_type TEXT NOT NULL, -- 'thumbs_down', 'thumbs_up', 'comment', 'label_applied', 'label_removed', 'label_upvote' + + -- Context + snippet_id UUID REFERENCES snippets(id), + user_id UUID REFERENCES auth.users(id), + + -- Related entities + comment_id TEXT, -- Liveblocks comment ID + label_id UUID REFERENCES labels(id), + snippet_label_id UUID REFERENCES snippet_labels(id), + + -- Content + content TEXT, -- Comment text, label text, etc. + sentiment TEXT, -- 'positive', 'negative', 'neutral' + + -- Processing + processed BOOLEAN DEFAULT false, + processed_at TIMESTAMPTZ, + proposal_id UUID REFERENCES prompt_rewrite_proposals(id), + + -- Timestamps + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX idx_feedback_events_snippet ON user_feedback_events(snippet_id); +CREATE INDEX idx_feedback_events_unprocessed ON user_feedback_events(processed) WHERE processed = false; +CREATE INDEX idx_feedback_events_type ON user_feedback_events(feedback_type); + +-- Agent execution logs for debugging and monitoring +CREATE TABLE prompt_rewriter_agent_logs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + proposal_id UUID REFERENCES prompt_rewrite_proposals(id) ON DELETE CASCADE, + + -- Which agent + agent_name TEXT NOT NULL, -- 'feedback_intake', 'research', 'proposal_writer', etc. + + -- Execution details + started_at TIMESTAMPTZ DEFAULT NOW(), + completed_at TIMESTAMPTZ, + duration_ms INTEGER, + + -- Input/Output + input_data JSONB, + output_data JSONB, + + -- LLM details if applicable + llm_model TEXT, + llm_prompt_tokens INTEGER, + llm_completion_tokens INTEGER, + llm_total_tokens INTEGER, + llm_cost_usd FLOAT, + + -- Status + status TEXT DEFAULT 'running', -- 'running', 'completed', 'failed', 'retrying' + error_message TEXT, + retry_count INTEGER DEFAULT 0 +); + +CREATE INDEX idx_agent_logs_proposal ON prompt_rewriter_agent_logs(proposal_id); +CREATE INDEX idx_agent_logs_agent ON prompt_rewriter_agent_logs(agent_name); + +-- Function to update updated_at timestamp +CREATE OR REPLACE FUNCTION update_prompt_rewriter_timestamp() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Apply timestamp triggers +CREATE TRIGGER prompt_rewrite_proposals_update_timestamp + BEFORE UPDATE ON prompt_rewrite_proposals + FOR EACH ROW EXECUTE FUNCTION update_prompt_rewriter_timestamp(); + +CREATE TRIGGER knowledge_facts_update_timestamp + BEFORE UPDATE ON knowledge_facts + FOR EACH ROW EXECUTE FUNCTION update_prompt_rewriter_timestamp(); + +-- Function to get next proposal to process +CREATE OR REPLACE FUNCTION get_next_pending_proposal() +RETURNS UUID AS $$ +DECLARE + proposal_id UUID; +BEGIN + SELECT id INTO proposal_id + FROM prompt_rewrite_proposals + WHERE status = 'pending' + ORDER BY + CASE priority + WHEN 'critical' THEN 1 + WHEN 'high' THEN 2 + WHEN 'medium' THEN 3 + WHEN 'low' THEN 4 + END, + created_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED; + + IF proposal_id IS NOT NULL THEN + UPDATE prompt_rewrite_proposals + SET status = 'analyzing_feedback', updated_at = NOW() + WHERE id = proposal_id; + END IF; + + RETURN proposal_id; +END; +$$ LANGUAGE plpgsql; + +-- Function to get next snippet to reprocess +CREATE OR REPLACE FUNCTION get_next_reprocess_snippet() +RETURNS TABLE(queue_id UUID, snippet_id UUID, proposal_id UUID) AS $$ +BEGIN + RETURN QUERY + WITH selected AS ( + SELECT q.id, q.snippet_id, q.proposal_id + FROM snippet_reprocess_queue q + WHERE q.status = 'queued' + ORDER BY q.priority DESC, q.queued_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + UPDATE snippet_reprocess_queue q + SET status = 'processing', started_at = NOW() + FROM selected s + WHERE q.id = s.id + RETURNING q.id, q.snippet_id, q.proposal_id; +END; +$$ LANGUAGE plpgsql; + +-- Grant permissions +GRANT ALL ON prompt_rewrite_proposals TO authenticated; +GRANT ALL ON prompt_rewrite_proposals TO service_role; +GRANT ALL ON prompt_experiment_runs TO authenticated; +GRANT ALL ON prompt_experiment_runs TO service_role; +GRANT ALL ON snippet_reprocess_queue TO authenticated; +GRANT ALL ON snippet_reprocess_queue TO service_role; +GRANT ALL ON knowledge_facts TO authenticated; +GRANT ALL ON knowledge_facts TO service_role; +GRANT ALL ON user_feedback_events TO authenticated; +GRANT ALL ON user_feedback_events TO service_role; +GRANT ALL ON prompt_rewriter_agent_logs TO authenticated; +GRANT ALL ON prompt_rewriter_agent_logs TO service_role; + +-- Enable RLS +ALTER TABLE prompt_rewrite_proposals ENABLE ROW LEVEL SECURITY; +ALTER TABLE prompt_experiment_runs ENABLE ROW LEVEL SECURITY; +ALTER TABLE snippet_reprocess_queue ENABLE ROW LEVEL SECURITY; +ALTER TABLE knowledge_facts ENABLE ROW LEVEL SECURITY; +ALTER TABLE user_feedback_events ENABLE ROW LEVEL SECURITY; +ALTER TABLE prompt_rewriter_agent_logs ENABLE ROW LEVEL SECURITY; + +-- RLS Policies - allow authenticated users to read, service_role for write +CREATE POLICY "Allow authenticated read on proposals" ON prompt_rewrite_proposals + FOR SELECT TO authenticated USING (true); +CREATE POLICY "Allow service_role all on proposals" ON prompt_rewrite_proposals + FOR ALL TO service_role USING (true); + +CREATE POLICY "Allow authenticated read on experiment_runs" ON prompt_experiment_runs + FOR SELECT TO authenticated USING (true); +CREATE POLICY "Allow service_role all on experiment_runs" ON prompt_experiment_runs + FOR ALL TO service_role USING (true); + +CREATE POLICY "Allow authenticated read on reprocess_queue" ON snippet_reprocess_queue + FOR SELECT TO authenticated USING (true); +CREATE POLICY "Allow service_role all on reprocess_queue" ON snippet_reprocess_queue + FOR ALL TO service_role USING (true); + +CREATE POLICY "Allow authenticated read on knowledge_facts" ON knowledge_facts + FOR SELECT TO authenticated USING (true); +CREATE POLICY "Allow service_role all on knowledge_facts" ON knowledge_facts + FOR ALL TO service_role USING (true); + +CREATE POLICY "Allow authenticated read on feedback_events" ON user_feedback_events + FOR SELECT TO authenticated USING (true); +CREATE POLICY "Allow service_role all on feedback_events" ON user_feedback_events + FOR ALL TO service_role USING (true); + +CREATE POLICY "Allow authenticated read on agent_logs" ON prompt_rewriter_agent_logs + FOR SELECT TO authenticated USING (true); +CREATE POLICY "Allow service_role all on agent_logs" ON prompt_rewriter_agent_logs + FOR ALL TO service_role USING (true); + +COMMENT ON TABLE prompt_rewrite_proposals IS 'Tracks autonomous prompt improvement proposals from user feedback'; +COMMENT ON TABLE knowledge_facts IS 'Verified facts knowledge base for prompt augmentation'; +COMMENT ON TABLE user_feedback_events IS 'Unified tracking of all user feedback that may trigger prompt rewrites';