From 31a8ab4bb79bac1407ffaf50b6fae650920ec138 Mon Sep 17 00:00:00 2001 From: MatteoGabriele Date: Sat, 28 Mar 2026 23:03:51 +0100 Subject: [PATCH 1/5] feat: add ai analysis --- app/components/Analysis/Card.vue | 3 +- nuxt.config.ts | 1 + package-lock.json | 20 ++- package.json | 4 +- .../api/identify-replicant/[username].get.ts | 165 +++++++++++++++++- 5 files changed, 189 insertions(+), 4 deletions(-) diff --git a/app/components/Analysis/Card.vue b/app/components/Analysis/Card.vue index 54f06fa..5d522c9 100644 --- a/app/components/Analysis/Card.vue +++ b/app/components/Analysis/Card.vue @@ -14,7 +14,8 @@ const { data, status, error } = useFetch( query: { created_at: props.user.created_at, repos_count: props.user.public_repos, - pages: 2, + pages: 1, + ai: true, }, key: analysisKey, watch: [username], diff --git a/nuxt.config.ts b/nuxt.config.ts index c99ec11..c74a050 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -40,6 +40,7 @@ export default defineNuxtConfig({ runtimeConfig: { githubToken: "", + geminiApiKey: "", }, css: ["~/assets/main.css"], diff --git a/package-lock.json b/package-lock.json index 868612f..914a9cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "name": "agentscan", "hasInstallScript": true, "dependencies": { + "@google/generative-ai": "^0.24.1", "@nuxt/fonts": "^0.14.0", "@nuxtjs/color-mode": "^4.0.0", "@unocss/preset-icons": "^66.6.0", @@ -23,7 +24,8 @@ "@iconify-json/carbon": "^1.2.18", "@octokit/types": "^16.0.0", "@unocss/nuxt": "^66.6.0", - "unocss": "^66.6.0" + "unocss": "^66.6.0", + "voight-kampff-compactor": "^1.0.0" } }, "node_modules/@antfu/install-pkg": { @@ -970,6 +972,15 @@ "node": ">=18" } }, + "node_modules/@google/generative-ai": { + "version": "0.24.1", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz", + "integrity": "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@iconify-json/carbon": { "version": "1.2.18", "resolved": "https://registry.npmjs.org/@iconify-json/carbon/-/carbon-1.2.18.tgz", @@ -11684,6 +11695,13 @@ "@types/estree": "^1.0.0" } }, + "node_modules/voight-kampff-compactor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/voight-kampff-compactor/-/voight-kampff-compactor-1.0.0.tgz", + "integrity": "sha512-2YRQUuD4XoGTVGbjc2NUbDVJKWt3x4ZhTW5MCvUhdDxQSMdulXY6bfK+yiB0WtZod1fYiImwHntOjxheQuEsEA==", + "dev": true, + "license": "MIT" + }, "node_modules/voight-kampff-test": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/voight-kampff-test/-/voight-kampff-test-2.5.0.tgz", diff --git a/package.json b/package.json index 8847b8a..b0a1d0b 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "test": "vitest" }, "dependencies": { + "@google/generative-ai": "^0.24.1", "@nuxt/fonts": "^0.14.0", "@nuxtjs/color-mode": "^4.0.0", "@unocss/preset-icons": "^66.6.0", @@ -28,6 +29,7 @@ "@iconify-json/carbon": "^1.2.18", "@octokit/types": "^16.0.0", "@unocss/nuxt": "^66.6.0", - "unocss": "^66.6.0" + "unocss": "^66.6.0", + "voight-kampff-compactor": "^1.0.0" } } diff --git a/server/api/identify-replicant/[username].get.ts b/server/api/identify-replicant/[username].get.ts index efcaaf9..1028741 100644 --- a/server/api/identify-replicant/[username].get.ts +++ b/server/api/identify-replicant/[username].get.ts @@ -1,12 +1,15 @@ -import { identifyReplicant } from "voight-kampff-test"; +import { identifyReplicant, IdentifyReplicantResult } from "voight-kampff-test"; import { Octokit } from "octokit"; import * as v from "valibot"; import { formatUsername } from "~~/server/utils/format-username"; +import { GoogleGenerativeAI } from "@google/generative-ai"; +import { compactor } from "voight-kampff-compactor"; const MIN_PAGES = 1; const MAX_PAGES = 2; const QuerySchema = v.object({ + ai: v.optional(v.boolean(), false), created_at: v.pipe( v.string("created_at is required"), v.check( @@ -40,6 +43,7 @@ export default defineEventHandler(async (event) => { const query = getQuery(event); const parsedQuery = v.safeParse(QuerySchema, { + ai: Boolean(query.ai), created_at: query.created_at, pages: query.pages ? parseInt(String(query.pages), 10) : 1, repos_count: query.repos_count @@ -70,6 +74,165 @@ export default defineEventHandler(async (event) => { const responses = await Promise.all(pageRequests); const events = responses.flatMap((response) => response.data); + if (parsedQuery.output.ai) { + const compactedData = compactor( + JSON.stringify({ + user: { + login: formatUsername, + created_at: parsedQuery.output.created_at, + public_repos: parsedQuery.output.repos_count, + }, + events, + }), + ); + + const systemPrompt = `You are an expert AI system designed to analyze GitHub user accounts and classify them as human-operated ("organic"), bot/automated ("automation"), or mixed behavior patterns. + + ## Your Task + Analyze a GitHub user's activity data (account metadata and event history) and return a classification with supporting evidence and a humanness score. + + ## Input Data Structure + - user.login: GitHub username + - user.created_at: ISO 8601 date string (account creation time) + - user.public_repos: number of public repositories owned + - events: array of GitHub events with type, created_at, repo.name, payload + + ## Classification Categories + - **organic**: Human-operated account (low bot-like signals, score ≥ 70) + - **mixed**: Uncertain patterns (moderate bot-like signals, score 50-69) + - **automation**: Likely bot-operated (strong bot-like signals, score < 50) + + ## Analysis Framework + + Evaluate each pattern independently. Assign a score per flag reflecting severity of the detected behavior (low, medium, high). + + ### 1. Account Age Context + - New account (< 30 days): Apply stricter scrutiny for bot patterns + - Young account (30-89 days): Moderate scrutiny, evaluate patterns carefully + - Established account (≥ 90 days): Higher tolerance for activity volume + + ### 2. Repository Activity Baseline + - Account has no personal repos but 20+ events: Suspicious pattern + - 95%+ external activity with < 5 personal repos: No personal investment + + ### 3. Bot-Like Pattern Detection (11 patterns to evaluate) + + #### A. Rapid Repository Creation + Detect CreateEvent (ref_type="repository") clustering in 24 hours. + Pattern: Rapid-fire repo creation suggests automation. + + #### B. Fork Surge + Detect ForkEvent clustering in 24-hour window. + Pattern: Concentrated forking activity suggests bot behavior. + + #### C. Commit Burst + Detect PushEvent clustering in 1-hour window. + Pattern: Inhuman commit velocity or ultra-tight clustering (seconds apart). + + #### D. 24/7 Activity Pattern + Analyze each calendar day: activity spanning 21+ unique hours with minimal rest suggests no sleep. + Pattern: Sustained multi-day coding without realistic sleep windows. + + #### E. Event Type Diversity (Shannon's Entropy) + Calculate normalized Shannon entropy of event types: + - Entropy = Σ(p * log₂(p)) for each type's probability p + - Normalized entropy = Entropy / log₂(number_of_types) + - Low entropy (< 0.5): Bot-like concentrated profile + - High entropy (> 0.8): Suspicious uniform distribution across types + + Pattern: Either narrow rigid focus (few types) OR artificial cycling through all types, combined with no human interactions (comments, reviews, watches). + + #### F. Issue Comment Spam + Detect IssueCommentEvent clustering across many repos within 30-minute window. + Pattern: Rapid-fire commenting across unrelated repos suggests automation. + + #### G. Branch → Pull Request Correlation + Detect pattern: branch created → PR opened within window, repeated consistently. + Pattern: Mechanical CI/CD automation cycling (not typical human workflow). + + #### H. PR Volume + Detect PR bursts to external repos (young accounts only, < 90 days). + Pattern: High external PR volume without personal repo activity. + + #### I. Consecutive Days Activity + Count calendar days with any activity. + Pattern: 21+ consecutive days suggests either dedication or tireless bot. + + #### J. External Repo Spread + Count unique external repos (young accounts only, < 90 days). + Pattern: Contributing to many different external repos broadly suggests spray-and-pray behavior. + + #### K. Daily Coding Hour Distribution + Analyze hour spread within each calendar day separately. + Pattern: High entropy (>0.8) across 16+ hours in a day suggests automated activity cycling. + + ## Scoring Methodology + Evaluate all detected patterns independently. For each flag present, assign a severity-based score (0-100 scale per flag). + Calculate final humanness score as: average of severity assessments across all flags, weighted by pattern significance. + + - Extreme automated signals: 0-20 (strong bot indicators) + - High bot-like behavior: 20-40 (multiple suspicious patterns) + - Moderate concerns: 40-60 (mixed or ambiguous signals) + - Low concerns: 60-80 (mostly human-like with isolated flags) + - Confident human: 80-100 (organic patterns throughout) + + ## Time Window Analysis Rules + - 24-hour rolling windows: sliding analysis for clustering patterns + - Per-day analysis: evaluate each calendar day independently (not globally) + - All times treated as UTC + + ## Return JSON Format (MUST be valid JSON only) + \`\`\`json + { + "score": number (0-100, overall humanness assessment), + "classification": "organic" | "mixed" | "automation", + "flags": [ + { + "label": "string (concise pattern name)", + "points": number (severity score for this flag: 0-100), + "detail": "string (specific evidence found: counts, timeframes, specifics)" + } + ], + "profile": { + "age": number (days since account creation), + "repos": number (public repositories count) + } + } + \`\`\` + + ## Output Requirements + - Evaluate patterns independently without predetermined point mappings + - Assign severity per flag based on strength of evidence + - Provide specific evidence in details (actual counts, timeframes, observed behaviors) + - Return ONLY valid JSON - no markdown, no extra text + - Include at least one flag per classification (if suspicious/mixed/automation) + + Be precise. Focus on evidence-based assessment, not fixed rubrics.`; + const userPrompt = `Here is the data to analyze: ${compactedData}`; + + try { + const genAI = new GoogleGenerativeAI(config.geminiApiKey); + const model = genAI.getGenerativeModel({ + model: "gemini-3.1-flash-lite-preview", + generationConfig: { + responseMimeType: "application/json", + }, + systemInstruction: systemPrompt, + }); + + const result = await model.generateContent(userPrompt); + const textContent = result.response.text(); + + return { + analysis: JSON.parse(textContent) as IdentifyReplicantResult, + eventsCount: events.length, + }; + } catch (aiError: unknown) { + console.error("Error during AI analysis:", aiError); + throw aiError; + } + } + return { analysis: identifyReplicant({ accountName: formattedUsername, From 0c4af60eca264c031c2561e3f62e87ff4a730344 Mon Sep 17 00:00:00 2001 From: MatteoGabriele Date: Sun, 29 Mar 2026 00:08:20 +0100 Subject: [PATCH 2/5] fix: label naming guidance --- server/api/identify-replicant/[username].get.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/api/identify-replicant/[username].get.ts b/server/api/identify-replicant/[username].get.ts index 1028741..4119aad 100644 --- a/server/api/identify-replicant/[username].get.ts +++ b/server/api/identify-replicant/[username].get.ts @@ -184,7 +184,7 @@ export default defineEventHandler(async (event) => { ## Return JSON Format (MUST be valid JSON only) \`\`\`json { - "score": number (0-100, overall humanness assessment), + "score": number (0-100), "classification": "organic" | "mixed" | "automation", "flags": [ { @@ -206,6 +206,7 @@ export default defineEventHandler(async (event) => { - Provide specific evidence in details (actual counts, timeframes, observed behaviors) - Return ONLY valid JSON - no markdown, no extra text - Include at least one flag per classification (if suspicious/mixed/automation) + - If an unexpected pattern emerges requiring a new label, use human-readable format (e.g., "Unusual coordination pattern") instead of snake_case (e.g., "unusual_coordination_pattern") Be precise. Focus on evidence-based assessment, not fixed rubrics.`; const userPrompt = `Here is the data to analyze: ${compactedData}`; From fd809cde7900ee7dd8b9cf74361bcdb1a9c7b0c7 Mon Sep 17 00:00:00 2001 From: MatteoGabriele Date: Sun, 29 Mar 2026 11:52:20 +0200 Subject: [PATCH 3/5] feat: update prompt --- .../api/identify-replicant/[username].get.ts | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/server/api/identify-replicant/[username].get.ts b/server/api/identify-replicant/[username].get.ts index 4119aad..a717336 100644 --- a/server/api/identify-replicant/[username].get.ts +++ b/server/api/identify-replicant/[username].get.ts @@ -6,7 +6,7 @@ import { GoogleGenerativeAI } from "@google/generative-ai"; import { compactor } from "voight-kampff-compactor"; const MIN_PAGES = 1; -const MAX_PAGES = 2; +const MAX_PAGES = 1; const QuerySchema = v.object({ ai: v.optional(v.boolean(), false), @@ -88,14 +88,17 @@ export default defineEventHandler(async (event) => { const systemPrompt = `You are an expert AI system designed to analyze GitHub user accounts and classify them as human-operated ("organic"), bot/automated ("automation"), or mixed behavior patterns. + ## Important Note + This analysis identifies AUTOMATION PATTERNS, not intent or legitimacy. We do not judge whether automation is "good" or "bad". We detect bot-like behavioral signatures to identify automated account activity. This includes spam bots, CI/CD automation left unfiltered, automated contribution patterns, and any coordinated bot behavior - regardless of purpose. A well-intentioned legitimate bot would still be flagged as "automation" if it displays these patterns. + ## Your Task - Analyze a GitHub user's activity data (account metadata and event history) and return a classification with supporting evidence and a humanness score. + Analyze a GitHub user's activity data (account metadata and event history) and return a classification indicating whether the account shows AUTOMATION SIGNATURES. Return a humanness score reflecting how automated vs organic the activity patterns appear. ## Input Data Structure - user.login: GitHub username - user.created_at: ISO 8601 date string (account creation time) - user.public_repos: number of public repositories owned - - events: array of GitHub events with type, created_at, repo.name, payload + - events: array of GitHub events with type, created_at, repo.name, payload (payload may contain text content: comments, PR descriptions, review text) ## Classification Categories - **organic**: Human-operated account (low bot-like signals, score ≥ 70) @@ -115,7 +118,9 @@ export default defineEventHandler(async (event) => { - Account has no personal repos but 20+ events: Suspicious pattern - 95%+ external activity with < 5 personal repos: No personal investment - ### 3. Bot-Like Pattern Detection (11 patterns to evaluate) + ### 3. Bot-Like Pattern Detection (12 patterns to evaluate) + + **NOTE: Text Content Analysis is Critical** - When event payloads contain text (comments, PR descriptions, reviews, commit messages), analyze for exact repetition, templates, or automated signatures. Repetition across multiple unrelated activities is a strong automation indicator. #### A. Rapid Repository Creation Detect CreateEvent (ref_type="repository") clustering in 24 hours. @@ -126,8 +131,8 @@ export default defineEventHandler(async (event) => { Pattern: Concentrated forking activity suggests bot behavior. #### C. Commit Burst - Detect PushEvent clustering in 1-hour window. - Pattern: Inhuman commit velocity or ultra-tight clustering (seconds apart). + Detect PushEvent clustering: 50+ commits in 1-hour window or 100+ commits in 1-hour window. + Pattern: 50+ commits/hour is impossible for human developers. Do NOT assume busy developer - this represents technical limits. #### D. 24/7 Activity Pattern Analyze each calendar day: activity spanning 21+ unique hours with minimal rest suggests no sleep. @@ -143,8 +148,8 @@ export default defineEventHandler(async (event) => { Pattern: Either narrow rigid focus (few types) OR artificial cycling through all types, combined with no human interactions (comments, reviews, watches). #### F. Issue Comment Spam - Detect IssueCommentEvent clustering across many repos within 30-minute window. - Pattern: Rapid-fire commenting across unrelated repos suggests automation. + Detect IssueCommentEvent clustering in 2-minute window: 10+ comments across 10+ different repos = high spam, 15+ repos = extreme spam. + Pattern: Commenting across 10+ unrelated repos in 2 minutes is impossible for humans. Do NOT tolerate this as "active developer". #### G. Branch → Pull Request Correlation Detect pattern: branch created → PR opened within window, repeated consistently. @@ -166,6 +171,14 @@ export default defineEventHandler(async (event) => { Analyze hour spread within each calendar day separately. Pattern: High entropy (>0.8) across 16+ hours in a day suggests automated activity cycling. + #### L. Repetitive or Automated Text Content + Analyze text from comments, PR descriptions, review comments, and commit messages for: + - Identical or near-identical text repeated across multiple unrelated issues/PRs/repos + - Automated comment signatures or templates (e.g., "Automated PR by bot", repeated footers, version strings) + - Generic placeholder text (e.g., form-filled descriptions with minimal variation) + - Templated responses with only variable substitution (same structure, different params) + Pattern: Exact text repetition across many activities or templated/automated language signatures indicate bot behavior. + ## Scoring Methodology Evaluate all detected patterns independently. For each flag present, assign a severity-based score (0-100 scale per flag). Calculate final humanness score as: average of severity assessments across all flags, weighted by pattern significance. @@ -176,6 +189,14 @@ export default defineEventHandler(async (event) => { - Low concerns: 60-80 (mostly human-like with isolated flags) - Confident human: 80-100 (organic patterns throughout) + ## Behavioral Context: Activity Bursts + Short, intense bursts of activity within very small time frames are NOT typical human behavior. Technical limits to consider: + - 50+ commits in 1 hour = highly suspicious (human commit/push cycle is much slower) + - 100+ commits in 1 hour = virtually impossible for human coding + - 10+ comments across 10+ different repos in 2 minutes = impossible for humans + - 15+ repos commented on in 2 minutes = automated commenting bot + These represent realistic physical/cognitive limits. Not tolerant of "busy developer" excuses. Short bursts are strong automation indicators. + ## Time Window Analysis Rules - 24-hour rolling windows: sliding analysis for clustering patterns - Per-day analysis: evaluate each calendar day independently (not globally) @@ -203,12 +224,15 @@ export default defineEventHandler(async (event) => { ## Output Requirements - Evaluate patterns independently without predetermined point mappings - Assign severity per flag based on strength of evidence - - Provide specific evidence in details (actual counts, timeframes, observed behaviors) + - Recognize that short, intense bursts of activity (minutes to hours) are NOT typical human behavior - be realistic about technical limits + - Do NOT excuse activity bursts as "very busy developer" - humans have cognitive and physical limits + - Analyze text content (comments, PR descriptions, reviews) for repetition and automated language - ALWAYS flag exact text repetition across multiple activities + - Provide specific evidence in details (actual counts, timeframes, observed behaviors, text samples if repetition found) - Return ONLY valid JSON - no markdown, no extra text - Include at least one flag per classification (if suspicious/mixed/automation) - If an unexpected pattern emerges requiring a new label, use human-readable format (e.g., "Unusual coordination pattern") instead of snake_case (e.g., "unusual_coordination_pattern") - Be precise. Focus on evidence-based assessment, not fixed rubrics.`; + Be precise, realistic, and evidence-based. Short bursts = automation. Do not be lenient.`; const userPrompt = `Here is the data to analyze: ${compactedData}`; try { From 6d6dc8c43bdc1f7583cfc73b3bc68bb02999e1d8 Mon Sep 17 00:00:00 2001 From: MatteoGabriele Date: Sun, 29 Mar 2026 12:24:05 +0200 Subject: [PATCH 4/5] feat: add events note --- server/api/identify-replicant/[username].get.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/api/identify-replicant/[username].get.ts b/server/api/identify-replicant/[username].get.ts index a717336..94a9a97 100644 --- a/server/api/identify-replicant/[username].get.ts +++ b/server/api/identify-replicant/[username].get.ts @@ -7,6 +7,7 @@ import { compactor } from "voight-kampff-compactor"; const MIN_PAGES = 1; const MAX_PAGES = 1; +const ITEMS_PER_PAGE = 100; const QuerySchema = v.object({ ai: v.optional(v.boolean(), false), @@ -66,7 +67,7 @@ export default defineEventHandler(async (event) => { const pageRequests = Array.from({ length: validatedPages }, (_, index) => { return octokit.rest.activity.listPublicEventsForUser({ username: formattedUsername, - per_page: 100, + per_page: ITEMS_PER_PAGE, page: index + 1, }); }); @@ -231,6 +232,7 @@ export default defineEventHandler(async (event) => { - Return ONLY valid JSON - no markdown, no extra text - Include at least one flag per classification (if suspicious/mixed/automation) - If an unexpected pattern emerges requiring a new label, use human-readable format (e.g., "Unusual coordination pattern") instead of snake_case (e.g., "unusual_coordination_pattern") + - NOTE: Events are limited to the most recent ${ITEMS_PER_PAGE * MAX_PAGES} public events from the GitHub API. This is NOT the user's complete activity history — draw conclusions accordingly and avoid absolute statements about total activity. Be precise, realistic, and evidence-based. Short bursts = automation. Do not be lenient.`; const userPrompt = `Here is the data to analyze: ${compactedData}`; From 9293c1d512c52a54ced13be641e6e1fdeee09a21 Mon Sep 17 00:00:00 2001 From: MatteoGabriele Date: Mon, 30 Mar 2026 01:30:37 +0200 Subject: [PATCH 5/5] refactor: use unveil-project --- app/components/Main/Footer.vue | 10 +- app/composables/useClassificationDetails.ts | 2 +- app/composables/useSeo.ts | 2 +- nuxt.config.ts | 2 +- package-lock.json | 98 +++++++++---------- package.json | 6 +- .../api/identify-replicant/[username].get.ts | 8 +- 7 files changed, 63 insertions(+), 65 deletions(-) diff --git a/app/components/Main/Footer.vue b/app/components/Main/Footer.vue index 235e46d..ba6249e 100644 --- a/app/components/Main/Footer.vue +++ b/app/components/Main/Footer.vue @@ -1,7 +1,7 @@