Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,10 @@ export default {

// Option B: Use an existing deploy preview
// previewUrl: 'VERCEL_URL', // env var name or literal URL

// Hint: extra context injected into every LLM prompt.
// Use this to tell the LLM about authentication, test accounts, app quirks, etc.
hint: 'Use email demo@example.com and password "password" to log in.',
},

// ── Trigger ───────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -437,6 +441,30 @@ For files outside these conventions, use `routeMap` in the config to explicitly

---

## Passing hints to the LLM

The `app.hint` field lets you inject free-form context into every LLM prompt. The text is included verbatim under an **"App-specific notes"** heading, so the model sees it before generating any Playwright script.

Common uses:

- **Authentication** — tell the LLM which test account to use so it can log in before navigating to the changed page.
- **Seed data** — describe what records exist in the database so the script can interact with realistic content.
- **App quirks** — explain non-obvious UI patterns (e.g. a custom modal, a multi-step wizard) so the model handles them correctly.

```typescript
export default {
app: {
startCommand: 'npm run dev',
// Injected into every prompt sent to the LLM
hint: 'Use email demo@example.com and password "password" to log in before navigating anywhere.',
},
} satisfies GitGlimpseConfig;
```

The hint is applied to both diff-driven scripts and general overview demos, so it is always available regardless of trigger mode.

---

## Retry and fallback behavior

If the generated Playwright script fails (element not found, timeout, etc.), GitGlimpse:
Expand Down
4 changes: 3 additions & 1 deletion packages/action/dist/check.js
Original file line number Diff line number Diff line change
Expand Up @@ -53433,7 +53433,9 @@ var AppConfigSchema = external_exports.object({
timeout: external_exports.number().default(3e4)
}).optional(),
previewUrl: external_exports.string().optional(),
env: external_exports.record(external_exports.string()).optional()
env: external_exports.record(external_exports.string()).optional(),
/** Extra context appended to every LLM prompt (e.g. auth instructions, app-specific notes). */
hint: external_exports.string().optional()
});
var RecordingConfigSchema = external_exports.object({
viewport: external_exports.object({
Expand Down
4 changes: 2 additions & 2 deletions packages/action/dist/check.js.map

Large diffs are not rendered by default.

19 changes: 14 additions & 5 deletions packages/action/dist/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -65381,7 +65381,10 @@ function buildScriptGenerationPrompt(options) {
- Affected routes:
${routeList}
- Suggested demo flow: ${options.demoFlow}

${options.hint ? `
## App-specific notes
${options.hint}
` : ""}
## Diff
\`\`\`
${truncatedDiff}
Expand Down Expand Up @@ -65422,7 +65425,10 @@ Produce a short, visually engaging tour that demonstrates the app is running and
## Context
- Base URL: ${options.baseUrl}
- Viewport: ${options.viewport.width}x${options.viewport.height}

${options.hint ? `
## App-specific notes
${options.hint}
` : ""}
## Output format
Respond with ONLY the TypeScript script, no markdown fences, no explanation:

Expand Down Expand Up @@ -65486,12 +65492,13 @@ async function generateDemoScript(client, analysis, rawDiff, baseUrl, config, ge
routes: analysis.affectedRoutes,
demoFlow: analysis.suggestedDemoFlow,
maxDuration: recording.maxDuration,
viewport: recording.viewport
viewport: recording.viewport,
hint: config.app.hint
};
const errors = [];
let lastScript = "";
for (let attempt = 1; attempt <= MAX_RETRIES + 1; attempt++) {
const prompt = attempt === 1 ? generalDemo ? buildGeneralDemoPrompt({ baseUrl, maxDuration: recording.maxDuration, viewport: recording.viewport }) : buildScriptGenerationPrompt(promptOptions) : buildRetryPrompt(lastScript, errors[errors.length - 1] ?? "", "", promptOptions);
const prompt = attempt === 1 ? generalDemo ? buildGeneralDemoPrompt({ baseUrl, maxDuration: recording.maxDuration, viewport: recording.viewport, hint: config.app.hint }) : buildScriptGenerationPrompt(promptOptions) : buildRetryPrompt(lastScript, errors[errors.length - 1] ?? "", "", promptOptions);
const response = await client.messages.create({
model: llm.model,
max_tokens: 4096,
Expand Down Expand Up @@ -69948,7 +69955,9 @@ var AppConfigSchema = external_exports.object({
timeout: external_exports.number().default(3e4)
}).optional(),
previewUrl: external_exports.string().optional(),
env: external_exports.record(external_exports.string()).optional()
env: external_exports.record(external_exports.string()).optional(),
/** Extra context appended to every LLM prompt (e.g. auth instructions, app-specific notes). */
hint: external_exports.string().optional()
});
var RecordingConfigSchema = external_exports.object({
viewport: external_exports.object({
Expand Down
4 changes: 2 additions & 2 deletions packages/action/dist/index.js.map

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions packages/core/src/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ export const AppConfigSchema = z.object({
.optional(),
previewUrl: z.string().optional(),
env: z.record(z.string()).optional(),
/** Extra context appended to every LLM prompt (e.g. auth instructions, app-specific notes). */
hint: z.string().optional(),
});

export const RecordingConfigSchema = z.object({
Expand Down
7 changes: 4 additions & 3 deletions packages/core/src/generator/prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ export interface ScriptPromptOptions {
demoFlow: string;
maxDuration: number;
viewport: { width: number; height: number };
hint?: string;
}

export function buildScriptGenerationPrompt(options: ScriptPromptOptions): string {
Expand Down Expand Up @@ -48,7 +49,7 @@ export function buildScriptGenerationPrompt(options: ScriptPromptOptions): strin
- Affected routes:
${routeList}
- Suggested demo flow: ${options.demoFlow}

${options.hint ? `\n## App-specific notes\n${options.hint}\n` : ''}
## Diff
\`\`\`
${truncatedDiff}
Expand All @@ -64,7 +65,7 @@ export async function demo(page: Page): Promise<void> {
}`;
}

export function buildGeneralDemoPrompt(options: Pick<ScriptPromptOptions, 'baseUrl' | 'maxDuration' | 'viewport'>): string {
export function buildGeneralDemoPrompt(options: Pick<ScriptPromptOptions, 'baseUrl' | 'maxDuration' | 'viewport' | 'hint'>): string {
return `You are a Playwright script generator. Generate a TypeScript Playwright script that records a general overview demo of a web app — as if showing it to someone for the first time.

## Goal
Expand All @@ -90,7 +91,7 @@ Produce a short, visually engaging tour that demonstrates the app is running and
## Context
- Base URL: ${options.baseUrl}
- Viewport: ${options.viewport.width}x${options.viewport.height}

${options.hint ? `\n## App-specific notes\n${options.hint}\n` : ''}
## Output format
Respond with ONLY the TypeScript script, no markdown fences, no explanation:

Expand Down
3 changes: 2 additions & 1 deletion packages/core/src/generator/script-generator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export async function generateDemoScript(
demoFlow: analysis.suggestedDemoFlow,
maxDuration: recording.maxDuration,
viewport: recording.viewport,
hint: config.app.hint,
};

const errors: string[] = [];
Expand All @@ -39,7 +40,7 @@ export async function generateDemoScript(
const prompt =
attempt === 1
? (generalDemo
? buildGeneralDemoPrompt({ baseUrl, maxDuration: recording.maxDuration, viewport: recording.viewport })
? buildGeneralDemoPrompt({ baseUrl, maxDuration: recording.maxDuration, viewport: recording.viewport, hint: config.app.hint })
: buildScriptGenerationPrompt(promptOptions))
: buildRetryPrompt(lastScript, errors[errors.length - 1] ?? '', '', promptOptions);

Expand Down
74 changes: 74 additions & 0 deletions tests/unit/prompts.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { describe, it, expect } from 'vitest';
import {
buildScriptGenerationPrompt,
buildGeneralDemoPrompt,
} from '../../packages/core/src/generator/prompts.js';
import type { ScriptPromptOptions } from '../../packages/core/src/generator/prompts.js';

const BASE_OPTIONS: ScriptPromptOptions = {
baseUrl: 'http://localhost:3000',
diff: 'diff --git a/src/Button.tsx b/src/Button.tsx\n+export function Button() {}',
routes: [{ route: '/', file: 'src/Button.tsx' }],
demoFlow: 'Navigate to home and click the button',
maxDuration: 30,
viewport: { width: 1280, height: 720 },
};

describe('buildScriptGenerationPrompt', () => {
it('includes hint under App-specific notes when provided', () => {
const prompt = buildScriptGenerationPrompt({
...BASE_OPTIONS,
hint: 'Log in with demo@example.com / password',
});
expect(prompt).toContain('## App-specific notes');
expect(prompt).toContain('Log in with demo@example.com / password');
});

it('omits App-specific notes section when hint is not provided', () => {
const prompt = buildScriptGenerationPrompt(BASE_OPTIONS);
expect(prompt).not.toContain('## App-specific notes');
});

it('omits App-specific notes section when hint is empty string', () => {
const prompt = buildScriptGenerationPrompt({ ...BASE_OPTIONS, hint: '' });
expect(prompt).not.toContain('## App-specific notes');
});

it('places hint before the Diff section', () => {
const prompt = buildScriptGenerationPrompt({
...BASE_OPTIONS,
hint: 'my hint',
});
const hintPos = prompt.indexOf('## App-specific notes');
const diffPos = prompt.indexOf('## Diff');
expect(hintPos).toBeGreaterThan(-1);
expect(hintPos).toBeLessThan(diffPos);
});
});

describe('buildGeneralDemoPrompt', () => {
const BASE_GENERAL = {
baseUrl: 'http://localhost:3000',
maxDuration: 30,
viewport: { width: 1280, height: 720 },
};

it('includes hint under App-specific notes when provided', () => {
const prompt = buildGeneralDemoPrompt({
...BASE_GENERAL,
hint: 'Use test account admin / secret',
});
expect(prompt).toContain('## App-specific notes');
expect(prompt).toContain('Use test account admin / secret');
});

it('omits App-specific notes section when hint is not provided', () => {
const prompt = buildGeneralDemoPrompt(BASE_GENERAL);
expect(prompt).not.toContain('## App-specific notes');
});

it('omits App-specific notes section when hint is empty string', () => {
const prompt = buildGeneralDemoPrompt({ ...BASE_GENERAL, hint: '' });
expect(prompt).not.toContain('## App-specific notes');
});
});
Loading