feat: add ai to detect clankers

README.md

@@ -47,6 +47,42 @@ console.log(analysis);
 // }
 ```
 
+### AI-Enhanced Analysis
+
+For deeper analysis, you can use the `getAIAnalysis` function to run the heuristic results through an LLM via [GitHub Models](https://github.com/marketplace/models). This provides a confidence score and natural language reasoning on top of the rule-based classification.
+
+```js
+import { identify } from "@unveil/identity";
+import { getAIAnalysis } from "@unveil/identity/ai";
+
+const analysis = identify({
+  createdAt: user.created_at,
+  reposCount: user.public_repos,
+  accountName: user.login,
+  events,
+});
+
+const aiResult = await getAIAnalysis({
+  token: process.env.GITHUB_TOKEN,
+  model: "openai/gpt-4o",
+  username: user.login,
+  analysis,
+  accountCreatedAt: user.created_at,
+  publicRepos: user.public_repos,
+  events,
+});
+
+console.log(aiResult);
+```
+
+`getAIAnalysis` accepts any model available on GitHub Models (e.g. `openai/gpt-4o`, `deepseek/DeepSeek-R1`). It returns `null` if the model produces no usable response.
+
+**tested models**:
+- openai/gpt-4o-mini
+- deepseek/DeepSeek-R1
+- openai/gpt-4o **(unreliable)**
+
+
 ### Issues and feature requests
 
 Please drop an issue if you find something that doesn't work, or have an idea for something that works better.

package.json

@@ -18,6 +18,8 @@
   ],
   "exports": {
     ".": "./dist/index.mjs",
+    "./ai": "./dist/ai/index.mjs",
+    "./ai-interpret": "./dist/ai-interpret/index.mjs",
     "./package.json": "./package.json"
   },
   "main": "./dist/index.mjs",
@@ -32,6 +34,7 @@
     "test": "vitest",
     "typecheck": "tsc --noEmit",
     "add:fixture": "node scripts/fetch-github-events.js",
+    "ai:analyse": "npx jiti scripts/ai-analyser-user.ts",
     "prepublishOnly": "vitest --run && pnpm run build"
   },
   "devDependencies": {
@@ -44,5 +47,9 @@
     "tsdown": "^0.18.1",
     "typescript": "^5.9.3",
     "vitest": "^4.0.16"
+  },
+  "dependencies": {
+    "voight-kampff-compactor": "^1.0.0",
+    "zod": "^4.3.6"
   }
 }

scripts/ai-analyser-user.ts

@@ -0,0 +1,94 @@
+import { identify } from "../src/index";
+import { getAIAnalysis } from "../src/ai/index";
+
+const username = process.argv[2];
+const model = process.argv[3] || "openai/gpt-4o-mini";
+const token = process.env.GITHUB_TOKEN;
+
+if (!username) {
+    console.error("Usage: GITHUB_TOKEN=<token> npx jiti scripts/ai-analyser-user.ts <username> [model]");
+    console.error("Example: npx jiti scripts/ai-analyser-user.ts octocat openai/gpt-4o");
+    process.exit(1);
+}
+
+if (!token) {
+    console.error("Error: GITHUB_TOKEN environment variable is required");
+    process.exit(1);
+}
+
+async function run() {
+    console.log(`Fetching data for: ${username}`);
+
+    const userRes = await fetch(`https://api.github.com/users/${username}`, {
+        headers: { Authorization: `Bearer ${token}` },
+    });
+    if (!userRes.ok) throw new Error(`GitHub API error: ${userRes.status} ${userRes.statusText}`);
+    const user = await userRes.json();
+
+    console.log(`Fetching orgs...`);
+    const orgsRes = await fetch(`https://api.github.com/users/${username}/orgs`, {
+        headers: { Authorization: `Bearer ${token}` },
+    });
+    const orgs = orgsRes.ok
+        ? (await orgsRes.json()).map((o: { login: string }) => o.login)
+        : []; 
+    console.log(`Fetching events...`);
+    const events = [];
+    for (let page = 1; page <= 3; page++) {
+        const res = await fetch(
+            `https://api.github.com/users/${username}/events?per_page=100&page=${page}`,
+            { headers: { Authorization: `Bearer ${token}` } },
+        );
+        if (!res.ok) throw new Error(`GitHub API error: ${res.status} ${res.statusText}`);
+        const page_events = await res.json();
+        if (page_events.length === 0) break;
+        events.push(...page_events);
+        if (page_events.length < 100) break;
+    }
+
+    console.log(`Running heuristic analysis...`);
+    const analysis = identify({
+        createdAt: user.created_at,
+        reposCount: user.public_repos,
+        accountName: user.login,
+        events,
+    });
+
+    console.log("\n--- Heuristic Result ---");
+    console.log(`Classification: ${analysis.classification}`);
+    console.log(`Score: ${analysis.score}`);
+    if (analysis.flags.length) {
+        console.log("Flags:");
+        for (const flag of analysis.flags) {
+            console.log(`  - ${flag.label} (${flag.points} pts): ${flag.detail}`);
+        }
+    }
+
+    console.log(`\nRunning AI analysis with model: ${model}...`);
+    const aiResult = await getAIAnalysis({
+        token: token!,
+        model,
+        username: user.login,
+        // let's test without. Feels like this influence too much the LLM
+        // analysis,
+        accountCreatedAt: user.created_at,
+        publicRepos: user.public_repos,
+        events,
+        orgs,
+    });
+
+    if (!aiResult) {
+        console.error("AI analysis returned no result.");
+        process.exit(1);
+    }
+
+    console.log("\n--- AI Result ---");
+    console.log(`Classification: ${aiResult.classification}`);
+    console.log(`Confidence: ${aiResult.confidence}`);
+    console.log(`Reasoning: ${aiResult.reasoning}`);
+}
+
+run().catch((err) => {
+    console.error(`Error: ${err.message}`);
+    process.exit(1);
+});

src/ai/analysis.ts

@@ -0,0 +1,57 @@
+import { z } from "zod/mini";
+import { buildUserPrompt, SYSTEM_PROMPT } from "./prompt";
+import type { AIAnalysisInput, AIAnalysisResult } from "./types";
+
+const aiAnalysisResultSchema = z.object({
+  classification: z.enum(["organic", "mixed", "automation"]),
+  confidence: z.number(),
+  reasoning: z.string(),
+});
+
+export async function getAIAnalysis(input: AIAnalysisInput): Promise<AIAnalysisResult | null> {
+  const { token = process.env.GITHUB_TOKEN, model = 'openai/gpt-4o-mini', username, analysis, accountCreatedAt, publicRepos, events, orgs } = input;
+
+  if (!token) {
+    throw new Error("GitHub token is required for AI analysis. Please provide it in the input or set it as an environment variable GITHUB_TOKEN.");
+  }
+
+  const prompt = buildUserPrompt({ token, model, username, analysis, accountCreatedAt, publicRepos, events, orgs });
+
+  // todo: extract into separate module for calling different AI providers and handling their specific quirks
+  const response = await fetch(
+    "https://models.github.ai/inference/chat/completions",
+    {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model,
+        messages: [
+          { role: "system", content: SYSTEM_PROMPT },
+          { role: "user", content: prompt },
+        ],
+        temperature: 0.3,
+      }),
+    },
+  );
+
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(`${response.status} ${response.statusText}: ${body}`);
+  }
+
+  const data = (await response.json()) as {
+    choices?: { message?: { content?: string } }[];
+  };
+  let content = data.choices?.[0]?.message?.content?.trim() ?? null;
+  if (!content) return null;
+
+  if (model.includes("deepseek")) {
+    // remove DeepSeek-R1 markers if present
+    content = content.replace(/<think>[\s\S]*?<\/think>/g, "").trim();
+  }
+
+  return aiAnalysisResultSchema.parse(JSON.parse(content));
+}

src/ai/index.ts

@@ -0,0 +1,3 @@
+export * from './analysis'
+export * from './types'
+export * from './prompt'