From 9f68751103bff6a1679d18f6e3232bd66648462b Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Wed, 18 Jun 2025 11:14:53 +0200
Subject: [PATCH 01/16] feat: extract common code to helper

---
 packages/cli/src/helpers/test-helper.ts | 57 +++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 packages/cli/src/helpers/test-helper.ts

diff --git a/packages/cli/src/helpers/test-helper.ts b/packages/cli/src/helpers/test-helper.ts
new file mode 100644
index 000000000..ad1fc9e12
--- /dev/null
+++ b/packages/cli/src/helpers/test-helper.ts
@@ -0,0 +1,57 @@
+import { PrivateRunLocation, RunLocation } from '../services/abstract-check-runner'
+import { Session } from '../constructs'
+import type { Region } from '..'
+import { ReporterType } from '../reporters/reporter'
+import { isCI } from 'ci-info'
+
+const DEFAULT_REGION = 'eu-central-1'
+
+export async function prepareRunLocation (
+  configOptions: { runLocation?: keyof Region, privateRunLocation?: string } = {},
+  cliFlags: { runLocation?: keyof Region, privateRunLocation?: string } = {},
+  api: any,
+  accountId: string,
+): Promise<RunLocation> {
+  // Command line options take precedence
+  if (cliFlags.runLocation) {
+    const { data: availableLocations } = await api.locations.getAll()
+    if (availableLocations.some((l: { region: string | undefined }) => l.region === cliFlags.runLocation)) {
+      return { type: 'PUBLIC', region: cliFlags.runLocation }
+    }
+    throw new Error(`Unable to run checks on unsupported location "${cliFlags.runLocation}". ` +
+      `Supported locations are:\n${availableLocations.map((l: { region: any }) => `${l.region}`).join('\n')}`)
+  } else if (cliFlags.privateRunLocation) {
+    return preparePrivateRunLocation(cliFlags.privateRunLocation, api, accountId)
+  } else if (configOptions.runLocation && configOptions.privateRunLocation) {
+    throw new Error('Both runLocation and privateRunLocation fields were set in your Checkly config file.' +
+      ` Please only specify one run location. The configured locations were' +
+        ' "${configOptions.runLocation}" and "${configOptions.privateRunLocation}"`)
+  } else if (configOptions.runLocation) {
+    return { type: 'PUBLIC', region: configOptions.runLocation }
+  } else if (configOptions.privateRunLocation) {
+    return preparePrivateRunLocation(configOptions.privateRunLocation, api, accountId)
+  } else {
+    return { type: 'PUBLIC', region: DEFAULT_REGION }
+  }
+}
+
+export async function preparePrivateRunLocation (privateLocationSlugName: string, api: any, accountId: string): Promise<PrivateRunLocation> {
+  try {
+    const privateLocations = await Session.getPrivateLocations()
+    const privateLocation = privateLocations.find(({ slugName }) => slugName === privateLocationSlugName)
+    if (privateLocation) {
+      return { type: 'PRIVATE', id: privateLocation.id, slugName: privateLocationSlugName }
+    }
+    const { data: account } = await api.accounts.get(accountId)
+    throw new Error(`The specified private location "${privateLocationSlugName}" was not found on account "${account.name}".`)
+  } catch (err: any) {
+    throw new Error(`Failed to get private locations. ${err.message}.`)
+  }
+}
+
+export function prepareReportersTypes (reporterFlag: ReporterType, cliReporters: ReporterType[] = []): ReporterType[] {
+  if (!reporterFlag && !cliReporters.length) {
+    return [isCI ? 'ci' : 'list']
+  }
+  return reporterFlag ? [reporterFlag] : cliReporters
+}

From 4f25863998bd2811a2cbb4e33f517b7a02739f90 Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Wed, 18 Jun 2025 11:15:11 +0200
Subject: [PATCH 02/16] feat: use functions from helper

---
 packages/cli/src/commands/test.ts | 59 ++++---------------------------
 1 file changed, 7 insertions(+), 52 deletions(-)

diff --git a/packages/cli/src/commands/test.ts b/packages/cli/src/commands/test.ts
index f0caf3223..d52061082 100644
--- a/packages/cli/src/commands/test.ts
+++ b/packages/cli/src/commands/test.ts
@@ -26,8 +26,8 @@ import { printLn, formatCheckTitle, CheckStatus } from '../reporters/util'
 import { uploadSnapshots } from '../services/snapshot-service'
 import { isEntrypoint } from '../constructs/construct'
 import { BrowserCheckBundle } from '../constructs/browser-check-bundle'
+import { prepareReportersTypes, prepareRunLocation } from '../helpers/test-helper'
 
-const DEFAULT_REGION = 'eu-central-1'
 const MAX_RETRIES = 3
 
 export default class Test extends AuthCommand {
@@ -155,12 +155,15 @@ export default class Test extends AuthCommand {
       config: checklyConfig,
       constructs: checklyConfigConstructs,
     } = await loadChecklyConfig(configDirectory, configFilenames)
-    const location = await this.prepareRunLocation(checklyConfig.cli, {
+
+    const location = await prepareRunLocation(checklyConfig.cli, {
       runLocation: runLocation as keyof Region,
       privateRunLocation,
-    })
+    },
+      api,
+      config.getAccountId())
     const verbose = this.prepareVerboseFlag(verboseFlag, checklyConfig.cli?.verbose)
-    const reporterTypes = this.prepareReportersTypes(reporterFlag as ReporterType, checklyConfig.cli?.reporters)
+    const reporterTypes = prepareReportersTypes(reporterFlag as ReporterType, checklyConfig.cli?.reporters)
     const { data: account } = await api.accounts.get(config.getAccountId())
     const { data: availableRuntimes } = await api.runtimes.getAll()
 
@@ -378,54 +381,6 @@ export default class Test extends AuthCommand {
     return verboseFlag ?? cliVerboseFlag ?? false
   }
 
-  prepareReportersTypes (reporterFlag: ReporterType, cliReporters: ReporterType[] = []): ReporterType[] {
-    if (!reporterFlag && !cliReporters.length) {
-      return [isCI ? 'ci' : 'list']
-    }
-    return reporterFlag ? [reporterFlag] : cliReporters
-  }
-
-  async prepareRunLocation (
-    configOptions: { runLocation?: keyof Region, privateRunLocation?: string } = {},
-    cliFlags: { runLocation?: keyof Region, privateRunLocation?: string } = {},
-  ): Promise<RunLocation> {
-    // Command line options take precedence
-    if (cliFlags.runLocation) {
-      const { data: availableLocations } = await api.locations.getAll()
-      if (availableLocations.some(l => l.region === cliFlags.runLocation)) {
-        return { type: 'PUBLIC', region: cliFlags.runLocation }
-      }
-      throw new Error(`Unable to run checks on unsupported location "${cliFlags.runLocation}". ` +
-        `Supported locations are:\n${availableLocations.map(l => `${l.region}`).join('\n')}`)
-    } else if (cliFlags.privateRunLocation) {
-      return this.preparePrivateRunLocation(cliFlags.privateRunLocation)
-    } else if (configOptions.runLocation && configOptions.privateRunLocation) {
-      throw new Error('Both runLocation and privateRunLocation fields were set in your Checkly config file.' +
-        ` Please only specify one run location. The configured locations were' +
-        ' "${configOptions.runLocation}" and "${configOptions.privateRunLocation}"`)
-    } else if (configOptions.runLocation) {
-      return { type: 'PUBLIC', region: configOptions.runLocation }
-    } else if (configOptions.privateRunLocation) {
-      return this.preparePrivateRunLocation(configOptions.privateRunLocation)
-    } else {
-      return { type: 'PUBLIC', region: DEFAULT_REGION }
-    }
-  }
-
-  async preparePrivateRunLocation (privateLocationSlugName: string): Promise<PrivateRunLocation> {
-    try {
-      const privateLocations = await Session.getPrivateLocations()
-      const privateLocation = privateLocations.find(({ slugName }) => slugName === privateLocationSlugName)
-      if (privateLocation) {
-        return { type: 'PRIVATE', id: privateLocation.id, slugName: privateLocationSlugName }
-      }
-      const { data: account } = await api.accounts.get(config.getAccountId())
-      throw new Error(`The specified private location "${privateLocationSlugName}" was not found on account "${account.name}".`)
-    } catch (err: any) {
-      throw new Error(`Failed to get private locations. ${err.message}.`)
-    }
-  }
-
   prepareTestRetryStrategy (retries?: number, configRetries?: number) {
     const numRetries = retries ?? configRetries ?? 0
     if (numRetries > MAX_RETRIES) {

From f2ebb1d13c98069d08291f056461b6f016c8145b Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Wed, 18 Jun 2025 11:15:30 +0200
Subject: [PATCH 03/16] feat: create base pw-test command

---
 packages/cli/src/commands/pw-test.ts | 211 +++++++++++++++++++++++++++
 1 file changed, 211 insertions(+)
 create mode 100644 packages/cli/src/commands/pw-test.ts

diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts
new file mode 100644
index 000000000..f10da9e5d
--- /dev/null
+++ b/packages/cli/src/commands/pw-test.ts
@@ -0,0 +1,211 @@
+import { AuthCommand } from './authCommand'
+import { getCiInformation, getGitInformation, splitConfigFilePath } from '../services/util'
+import { loadChecklyConfig, PlaywrightSlimmedProp } from '../services/checkly-config-loader'
+import { prepareReportersTypes, prepareRunLocation } from '../helpers/test-helper'
+import * as api from '../rest/api'
+import config from '../services/config'
+import { parseProject } from '../services/project-parser'
+import type { Runtime } from '../rest/runtimes'
+import { Diagnostics, Session } from '../constructs'
+import { ux } from '@oclif/core'
+import { createReporters } from '../reporters/reporter'
+import TestRunner from '../services/test-runner'
+import { Events, SequenceId } from '../services/abstract-check-runner'
+import { TestResultsShortLinks } from '../rest/test-sessions'
+const DEFAULT_REGION = 'eu-central-1'
+
+
+export default class PwTestCommand extends AuthCommand {
+  static coreCommand = true
+  static hidden = false
+  static description = 'Test your Playwright Tests on Checkly'
+  static state = 'beta'
+
+  async run(): Promise<void> {
+    this.style.actionStart('Parsing your Playwright project')
+    const rawArgs = this.argv || []
+
+    const { configDirectory, configFilenames } = splitConfigFilePath()
+    const {
+      config: checklyConfig,
+      constructs: checklyConfigConstructs,
+    } = await loadChecklyConfig(configDirectory, configFilenames)
+
+    // TODO: ADD PROPER LOCATION HANDLING
+    const location = await prepareRunLocation(checklyConfig.cli, {}, api, config.getAccountId())
+
+    // TODO: SET PROPER REPORTER TYPES
+    const reporterTypes = prepareReportersTypes('list', checklyConfig.cli?.reporters)
+    const { data: account } = await api.accounts.get(config.getAccountId())
+    const { data: availableRuntimes } = await api.runtimes.getAll()
+
+    const project = await parseProject({
+      directory: configDirectory,
+      projectLogicalId: checklyConfig.logicalId,
+      // TODO: ADD PROPPER TEST SESSION NAME HANDLING
+      projectName: checklyConfig.projectName,
+      repoUrl: checklyConfig.repoUrl,
+      includeTestOnlyChecks: true,
+      checkMatch: checklyConfig.checks?.checkMatch,
+      ignoreDirectoriesMatch: checklyConfig.checks?.ignoreDirectoriesMatch,
+      checkDefaults: checklyConfig.checks,
+      availableRuntimes: availableRuntimes.reduce((acc, runtime) => {
+        acc[runtime.name] = runtime
+        return acc
+      }, <Record<string, Runtime>> {}),
+      defaultRuntimeId: account.runtimeId,
+      verifyRuntimeDependencies: false,
+      checklyConfigConstructs,
+      playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath,
+      include: checklyConfig.checks?.include,
+      playwrightChecks: PwTestCommand.createPlaywrightCheck(rawArgs),
+      checkFilter: check => {
+        return true
+      }
+    })
+
+    this.style.actionSuccess()
+
+    this.style.actionStart('Validating project resources')
+
+    const diagnostics = new Diagnostics()
+    await project.validate(diagnostics)
+
+    for (const diag of diagnostics.observations) {
+      if (diag.isFatal()) {
+        this.style.longError(diag.title, diag.message)
+      } else if (!diag.isBenign()) {
+        this.style.longWarning(diag.title, diag.message)
+      } else {
+        this.style.longInfo(diag.title, diag.message)
+      }
+    }
+
+    if (diagnostics.isFatal()) {
+      this.style.actionFailure()
+      this.style.shortError(`Unable to continue due to unresolved validation errors.`)
+      this.exit(1)
+    }
+
+    this.style.actionSuccess()
+
+    this.style.actionStart('Bundling project resources')
+    const projectBundle = await (async () => {
+      try {
+        const bundle = await project.bundle()
+        this.style.actionSuccess()
+        return bundle
+      } catch (err) {
+        this.style.actionFailure()
+        throw err
+      }
+    })()
+
+    const checkBundles = Object.values(projectBundle.data.check)
+
+    if (this.fancy) {
+      ux.action.stop()
+    }
+
+    if (!checkBundles.length) {
+      this.log(`Unable to find checks to run`)
+      return
+    }
+
+    // TODO: ADD PROPER LIST FLAG HANDLING
+    // if (list) {
+    //   this.listChecks(checkBundles.map(({ construct }) => construct))
+    //   return
+    // }
+
+    // TODO: ADD PROPER VERBOSE FLAG HANDLING
+    const reporters = createReporters(reporterTypes, location, false)
+    const repoInfo = getGitInformation(project.repoUrl)
+    const ciInfo = getCiInformation()
+    // TODO: ADD PROPER RETRY STRATEGY HANDLING
+    // const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
+
+    const runner = new TestRunner(
+      config.getAccountId(),
+      projectBundle,
+      checkBundles,
+      Session.sharedFiles,
+      location,
+      // TODO: ADD PROPER TEST SESSION TIMEOUT HANDLING
+      1000,
+      // TODO: ADD PROPER VERBOSE FLAG HANDLING
+      false,
+      // TODO: ADD PROPER RECORD FLAG HANDLING
+      true,
+      repoInfo,
+      ciInfo.environment,
+      // NO NEED TO UPLOAD SNAPSHOTS FOR PLAYWRIGHT TESTS
+      false,
+      configDirectory,
+      // TODO: ADD PROPER RETRY STRATEGY HANDLING
+      null, // testRetryStrategy
+    )
+
+    runner.on(Events.RUN_STARTED,
+      (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
+        reporters.forEach(r => r.onBegin(checks, testSessionId)),
+    )
+
+    runner.on(Events.CHECK_INPROGRESS, (check: any, sequenceId: SequenceId) => {
+      reporters.forEach(r => r.onCheckInProgress(check, sequenceId))
+    })
+
+    runner.on(Events.MAX_SCHEDULING_DELAY_EXCEEDED, () => {
+      reporters.forEach(r => r.onSchedulingDelayExceeded())
+    })
+
+    runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
+      reporters.forEach(r => r.onCheckAttemptResult(sequenceId, {
+        logicalId: check.logicalId,
+        sourceFile: check.getSourceFile(),
+        ...result,
+      }, links))
+    })
+
+    runner.on(Events.CHECK_SUCCESSFUL,
+      (sequenceId: SequenceId, check, result, testResultId, links?: TestResultsShortLinks) => {
+        if (result.hasFailures) {
+          process.exitCode = 1
+        }
+
+        reporters.forEach(r => r.onCheckEnd(sequenceId, {
+          logicalId: check.logicalId,
+          sourceFile: check.getSourceFile(),
+          ...result,
+        }, testResultId, links))
+      })
+
+    runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
+      reporters.forEach(r => r.onCheckEnd(sequenceId, {
+        ...check,
+        logicalId: check.logicalId,
+        sourceFile: check.getSourceFile(),
+        hasFailures: true,
+        runError: message,
+      }))
+      process.exitCode = 1
+    })
+    runner.on(Events.RUN_FINISHED, () => reporters.forEach(r => r.onEnd()))
+    runner.on(Events.ERROR, (err) => {
+      reporters.forEach(r => r.onError(err))
+      process.exitCode = 1
+    })
+    await runner.run()
+
+
+    }
+    static createPlaywrightCheck(args: string[]): PlaywrightSlimmedProp [] {
+      const input = args.join(' ') || ''
+     return [{
+        logicalId: 'playwright-check',
+        name: `Playwright Test: ${input}`,
+        testCommand: `npx playwright test ${input}`,
+        locations: [DEFAULT_REGION],
+      }]
+  }
+}

From 80980a9946980c14c2d5c3bfdafd65541a05ec1d Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Thu, 19 Jun 2025 10:20:24 +0200
Subject: [PATCH 04/16] feat: add pw-test command features

---
 packages/cli/src/commands/pw-test.ts          | 233 +++++++++++++++---
 packages/cli/src/commands/sync-playwright.ts  |  36 +--
 packages/cli/src/commands/test.ts             |   3 -
 packages/cli/src/helpers/test-helper.ts       |  24 ++
 .../cli/src/helpers/write-config-helpers.ts   |  62 +++++
 .../cli/src/services/checkly-config-loader.ts |  12 +-
 packages/cli/src/services/util.ts             |  19 +-
 7 files changed, 301 insertions(+), 88 deletions(-)
 create mode 100644 packages/cli/src/helpers/write-config-helpers.ts

diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts
index f10da9e5d..b6635555e 100644
--- a/packages/cli/src/commands/pw-test.ts
+++ b/packages/cli/src/commands/pw-test.ts
@@ -1,17 +1,35 @@
 import { AuthCommand } from './authCommand'
-import { getCiInformation, getGitInformation, splitConfigFilePath } from '../services/util'
-import { loadChecklyConfig, PlaywrightSlimmedProp } from '../services/checkly-config-loader'
-import { prepareReportersTypes, prepareRunLocation } from '../helpers/test-helper'
+import {
+  getCiInformation,
+  getDefaultChecklyConfig,
+  getEnvs,
+  getGitInformation,
+  splitConfigFilePath, writeChecklyConfigFile
+} from '../services/util'
+import { getChecklyConfigFile, loadChecklyConfig, PlaywrightSlimmedProp } from '../services/checkly-config-loader'
+import { prepareReportersTypes, prepareRunLocation, splitChecklyAndPlaywrightFlags } from '../helpers/test-helper'
 import * as api from '../rest/api'
 import config from '../services/config'
 import { parseProject } from '../services/project-parser'
 import type { Runtime } from '../rest/runtimes'
 import { Diagnostics, Session } from '../constructs'
-import { ux } from '@oclif/core'
-import { createReporters } from '../reporters/reporter'
+import { Flags, ux } from '@oclif/core'
+import { createReporters, ReporterType } from '../reporters/reporter'
 import TestRunner from '../services/test-runner'
-import { Events, SequenceId } from '../services/abstract-check-runner'
+import { DEFAULT_CHECK_RUN_TIMEOUT_SECONDS, Events, SequenceId } from '../services/abstract-check-runner'
 import { TestResultsShortLinks } from '../rest/test-sessions'
+import commonMessages from '../messages/common-messages'
+import type { Region } from '..'
+import path from 'node:path'
+import * as recast from 'recast'
+import {
+  addItemToArray,
+  addOrReplaceItem,
+  findPropertyByName,
+  reWriteChecklyConfigFile
+} from '../helpers/write-config-helpers'
+import * as JSON5 from 'json5'
+
 const DEFAULT_REGION = 'eu-central-1'
 
 
@@ -20,30 +38,104 @@ export default class PwTestCommand extends AuthCommand {
   static hidden = false
   static description = 'Test your Playwright Tests on Checkly'
   static state = 'beta'
+  static flags = {
+    'cly-location': Flags.string({
+      description: 'The location to run the checks at.',
+    }),
+    'cly-private-location': Flags.string({
+      description: 'The private location to run checks at.',
+      exclusive: ['location'],
+    }),
+    env: Flags.string({
+      char: 'e',
+      description: 'Env vars to be passed to the test run.',
+      exclusive: ['env-file'],
+      multiple: true,
+      default: [],
+    }),
+    'env-file': Flags.string({
+      description: 'dotenv file path to be passed. For example --env-file="./.env"',
+      exclusive: ['env'],
+    }),
+    'cly-timeout': Flags.integer({
+      default: DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
+      description: 'A timeout (in seconds) to wait for checks to complete.',
+    }),
+    'cly-verbose': Flags.boolean({
+      description: 'Always show the full logs of the checks.',
+    }),
+    'cly-reporter': Flags.string({
+      description: 'A list of custom reporters for the test output.',
+      options: ['list', 'dot', 'ci', 'github', 'json'],
+    }),
+    'cly-config': Flags.string({
+      description: commonMessages.configFile,
+    }),
+    'cly-skip-record': Flags.boolean({
+      description: 'Record test results in Checkly as a test session with full logs, traces and videos.',
+      default: false,
+    }),
+    'cly-test-session-name': Flags.string({
+      description: 'A name to use when storing results in Checkly',
+    }),
+    'cly-create-check': Flags.boolean({
+      description: 'Create a Checkly check from the Playwright test.',
+      default: true,
+    })
+  }
+
 
   async run(): Promise<void> {
     this.style.actionStart('Parsing your Playwright project')
     const rawArgs = this.argv || []
-
-    const { configDirectory, configFilenames } = splitConfigFilePath()
+    const { checklyFlags, playwrightFlags } = splitChecklyAndPlaywrightFlags(rawArgs)
+    if (!this.validChecklyFlags(checklyFlags)) {
+      this.style.actionFailure()
+      this.style.shortError('Invalid Checkly flags provided. Please check the command usage.')
+      this.exit(1)
+    }
+    const {
+      location: runLocation = DEFAULT_REGION,
+      'private-location': privateRunLocation,
+      env = [],
+      'env-file': envFile,
+      timeout = DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
+      verbose: verboseFlag,
+      reporter: reporterFlag,
+      config: configFilename,
+      'skip-record': skipRecord = false,
+      'test-session-name': testSessionName,
+      'create-check': createCheck = false,
+    } = checklyFlags
+    const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
     const {
       config: checklyConfig,
       constructs: checklyConfigConstructs,
-    } = await loadChecklyConfig(configDirectory, configFilenames)
+    } = await loadChecklyConfig(configDirectory, configFilenames, false)
+
+    const playwrightConfigPath = this.getConfigPath(playwrightFlags) ?? checklyConfig.checks?.playwrightConfigPath
+    const playwrightCheck = PwTestCommand.createPlaywrightCheck(playwrightFlags, runLocation as keyof Region)
+    if (createCheck) {
+      this.style.actionStart('Creating Checkly check from Playwright test')
+      await this.createPlaywrightCheck(playwrightCheck, playwrightConfigPath)
+      return
+    }
 
-    // TODO: ADD PROPER LOCATION HANDLING
-    const location = await prepareRunLocation(checklyConfig.cli, {}, api, config.getAccountId())
 
-    // TODO: SET PROPER REPORTER TYPES
-    const reporterTypes = prepareReportersTypes('list', checklyConfig.cli?.reporters)
+    const location = await prepareRunLocation(checklyConfig.cli, {
+      runLocation: runLocation as keyof Region,
+      privateRunLocation,
+    }, api, config.getAccountId())
+    const reporterTypes = prepareReportersTypes(reporterFlag as ReporterType, checklyConfig.cli?.reporters)
     const { data: account } = await api.accounts.get(config.getAccountId())
     const { data: availableRuntimes } = await api.runtimes.getAll()
+    const testEnvVars = await getEnvs(envFile, env)
+
 
     const project = await parseProject({
       directory: configDirectory,
       projectLogicalId: checklyConfig.logicalId,
-      // TODO: ADD PROPPER TEST SESSION NAME HANDLING
-      projectName: checklyConfig.projectName,
+      projectName: testSessionName ?? checklyConfig.projectName,
       repoUrl: checklyConfig.repoUrl,
       includeTestOnlyChecks: true,
       checkMatch: checklyConfig.checks?.checkMatch,
@@ -56,10 +148,21 @@ export default class PwTestCommand extends AuthCommand {
       defaultRuntimeId: account.runtimeId,
       verifyRuntimeDependencies: false,
       checklyConfigConstructs,
-      playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath,
+      playwrightConfigPath,
       include: checklyConfig.checks?.include,
-      playwrightChecks: PwTestCommand.createPlaywrightCheck(rawArgs),
+      playwrightChecks: [playwrightCheck],
       checkFilter: check => {
+        if (Object.keys(testEnvVars).length) {
+          check.environmentVariables = check.environmentVariables
+            ?.filter((envVar: any) => !testEnvVars[envVar.key]) || []
+          for (const [key, value] of Object.entries(testEnvVars)) {
+            check.environmentVariables.push({
+              key,
+              value,
+              locked: true,
+            })
+          }
+        }
         return true
       }
     })
@@ -112,31 +215,21 @@ export default class PwTestCommand extends AuthCommand {
       return
     }
 
-    // TODO: ADD PROPER LIST FLAG HANDLING
-    // if (list) {
-    //   this.listChecks(checkBundles.map(({ construct }) => construct))
-    //   return
-    // }
-
-    // TODO: ADD PROPER VERBOSE FLAG HANDLING
-    const reporters = createReporters(reporterTypes, location, false)
+    const reporters = createReporters(reporterTypes, location, verboseFlag)
     const repoInfo = getGitInformation(project.repoUrl)
     const ciInfo = getCiInformation()
     // TODO: ADD PROPER RETRY STRATEGY HANDLING
     // const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
-
+    const shouldRecord = !skipRecord
     const runner = new TestRunner(
       config.getAccountId(),
       projectBundle,
       checkBundles,
       Session.sharedFiles,
       location,
-      // TODO: ADD PROPER TEST SESSION TIMEOUT HANDLING
-      1000,
-      // TODO: ADD PROPER VERBOSE FLAG HANDLING
-      false,
-      // TODO: ADD PROPER RECORD FLAG HANDLING
-      true,
+      timeout,
+      verboseFlag,
+      shouldRecord,
       repoInfo,
       ciInfo.environment,
       // NO NEED TO UPLOAD SNAPSHOTS FOR PLAYWRIGHT TESTS
@@ -199,13 +292,77 @@ export default class PwTestCommand extends AuthCommand {
 
 
     }
-    static createPlaywrightCheck(args: string[]): PlaywrightSlimmedProp [] {
-      const input = args.join(' ') || ''
-     return [{
-        logicalId: 'playwright-check',
+    static createPlaywrightCheck(args: string[], runLocation: keyof Region): PlaywrightSlimmedProp {
+    const input = args.join(' ') || ''
+      const inputLogicalId = input.replace(/[^a-zA-Z0-9]/g, '-').toLowerCase().substring(0, 50)
+     return {
+        logicalId: `playwright-check-${inputLogicalId}`,
         name: `Playwright Test: ${input}`,
         testCommand: `npx playwright test ${input}`,
-        locations: [DEFAULT_REGION],
-      }]
+        locations: [runLocation],
+        frequency: 10,
+      }
+  }
+
+  private validChecklyFlags (checklyFlags: Record<string, any>) {
+    const validFlags = [
+      'location',
+      'private-location',
+      'env',
+      'env-file',
+      'list',
+      'timeout',
+      'verbose',
+      'reporter',
+      'config',
+      'skip-record',
+      'test-session-name',
+      'create-check'
+    ]
+    return Object.keys(checklyFlags).every(flag => validFlags.includes(flag))
+  }
+
+  private getConfigPath (playwrightFlags: string[]) {
+    for (let i = 0; i < playwrightFlags.length; i++) {
+      const arg = playwrightFlags[i]
+      if (arg.startsWith('--config') || arg.startsWith('-c')) {
+        return arg.includes('=') ? arg.split('=')[1] : playwrightFlags[i + 1]
+      }
+    }
+  }
+
+  private async createPlaywrightCheck (playwrightCheck: PlaywrightSlimmedProp, playwrightConfigPath: string = './playwright.config.ts') {
+    const dir = process.cwd()
+    const baseName = path.basename(dir)
+
+    const configFile = await getChecklyConfigFile()
+    if (!configFile) {
+      this.style.longInfo('No Checkly config file found', 'Creating a default checkly config file.')
+      const checklyConfig = getDefaultChecklyConfig(baseName, `./${path.relative(dir, playwrightConfigPath)}`, playwrightCheck)
+      await writeChecklyConfigFile(dir, checklyConfig)
+      this.style.actionSuccess()
+      return
+    }
+    const checklyAst = recast.parse(configFile.checklyConfig)
+    const checksAst = findPropertyByName(checklyAst, 'checks')
+    if (!checksAst) {
+      this.style.longError('Unable to automatically sync your config file.', 'This can happen if your Checkly config is ' +
+        'built using helper functions or other JS/TS features. You can still manually set Playwright config values in ' +
+        'your Checkly config: https://www.checklyhq.com/docs/cli/constructs-reference/#project')
+
+      return
+    }
+    const playwrightCheckString = `const playwrightCheck = ${JSON5.stringify(playwrightCheck, { space: 2 })}`
+    const playwrightCheckAst = recast.parse(playwrightCheckString)
+    const playwrightCheckNode = playwrightCheckAst.program.body[0].declarations[0].init;
+
+    addItemToArray(checksAst.value, playwrightCheckNode, 'playwrightChecks')
+    const checklyConfigData = recast.print(checklyAst, { tabWidth: 2 }).code
+    const writeDir = path.resolve(path.dirname(configFile.fileName))
+    await reWriteChecklyConfigFile(checklyConfigData, configFile.fileName, writeDir)
+    this.style.actionSuccess()
+
+    return
+
   }
 }
diff --git a/packages/cli/src/commands/sync-playwright.ts b/packages/cli/src/commands/sync-playwright.ts
index d766c0cf8..7469d940b 100644
--- a/packages/cli/src/commands/sync-playwright.ts
+++ b/packages/cli/src/commands/sync-playwright.ts
@@ -2,10 +2,10 @@ import { BaseCommand } from './baseCommand'
 import * as recast from 'recast'
 import { getChecklyConfigFile } from '../services/checkly-config-loader'
 import { loadPlaywrightConfig } from '../playwright/playwright-config-loader'
-import fs from 'fs'
 import path from 'path'
 import { ux } from '@oclif/core'
 import PlaywrightConfigTemplate from '../playwright/playwright-config-template'
+import { addOrReplaceItem, findPropertyByName, reWriteChecklyConfigFile } from '../helpers/write-config-helpers'
 
 export default class SyncPlaywright extends BaseCommand {
   static hidden = false
@@ -27,7 +27,7 @@ export default class SyncPlaywright extends BaseCommand {
       return this.handleError('Could not find any playwright.config file.')
     }
 
-    const checksAst = this.findPropertyByName(checklyAst, 'checks')
+    const checksAst = findPropertyByName(checklyAst, 'checks')
     if (!checksAst) {
       return this.handleError('Unable to automatically sync your config file. This can happen if your Checkly config is ' +
           'built using helper functions or other JS/TS features. You can still manually set Playwright config values in ' +
@@ -35,12 +35,12 @@ export default class SyncPlaywright extends BaseCommand {
     }
 
     const pwtConfig = new PlaywrightConfigTemplate(config).getConfigTemplate()
-    const pwtConfigAst = this.findPropertyByName(recast.parse(pwtConfig), 'playwrightConfig')
-    this.addOrReplacePlaywrightConfig(checksAst.value, pwtConfigAst)
+    const pwtConfigAst = findPropertyByName(recast.parse(pwtConfig), 'playwrightConfig')
+    addOrReplaceItem(checksAst.value, pwtConfigAst, 'playwrightConfig')
 
     const checklyConfigData = recast.print(checklyAst, { tabWidth: 2 }).code
     const dir = path.resolve(path.dirname(configFile.fileName))
-    this.reWriteChecklyConfigFile(checklyConfigData, configFile.fileName, dir)
+    await reWriteChecklyConfigFile(checklyConfigData, configFile.fileName, dir)
 
     if (this.fancy) {
       ux.action.stop('✅ ')
@@ -56,30 +56,4 @@ export default class SyncPlaywright extends BaseCommand {
     this.log(message)
     this.exit(1)
   }
-
-  private findPropertyByName (ast: any, name: string): recast.types.namedTypes.Property | undefined {
-    let node
-    recast.visit(ast, {
-      visitProperty (path: any) {
-        if (path.node.key.name === name) {
-          node = path.node
-        }
-        return false
-      },
-    })
-    return node
-  }
-
-  private addOrReplacePlaywrightConfig (ast: any, node: any) {
-    const playWrightConfig = this.findPropertyByName(ast, 'playwrightConfig')
-    if (playWrightConfig) {
-      playWrightConfig.value = node.value
-    } else {
-      ast.properties.push(node)
-    }
-  }
-
-  private reWriteChecklyConfigFile (data: string, fileName: string, dir: string) {
-    fs.writeFileSync(path.join(dir, fileName), data)
-  }
 }
diff --git a/packages/cli/src/commands/test.ts b/packages/cli/src/commands/test.ts
index d52061082..1b091a1dc 100644
--- a/packages/cli/src/commands/test.ts
+++ b/packages/cli/src/commands/test.ts
@@ -1,13 +1,10 @@
 import { Flags, Args, ux } from '@oclif/core'
 import indentString from 'indent-string'
-import { isCI } from 'ci-info'
 import * as api from '../rest/api'
 import config from '../services/config'
 import { parseProject } from '../services/project-parser'
 import {
   Events,
-  RunLocation,
-  PrivateRunLocation,
   SequenceId,
   DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
 } from '../services/abstract-check-runner'
diff --git a/packages/cli/src/helpers/test-helper.ts b/packages/cli/src/helpers/test-helper.ts
index ad1fc9e12..60fe9453a 100644
--- a/packages/cli/src/helpers/test-helper.ts
+++ b/packages/cli/src/helpers/test-helper.ts
@@ -55,3 +55,27 @@ export function prepareReportersTypes (reporterFlag: ReporterType, cliReporters:
   }
   return reporterFlag ? [reporterFlag] : cliReporters
 }
+
+export function splitChecklyAndPlaywrightFlags(args: string[]) {
+  const checklyFlags: Record<string, any> = {}
+  const playwrightFlags: string[] = []
+  let idx = 0;
+  while (idx < args.length) {
+    const arg = args[idx]
+    if (arg.startsWith('--cly-') || arg === '-e' || arg === '--env' || arg === '--env-file') {
+      if (arg.includes('=')) {
+        const split = arg.split(/=(.*)/s)
+        const k: string = split[0].replace('--cly-', '')
+        checklyFlags[k] = split[1]
+      } else {
+        const k: string = arg.replace('--cly-', '')
+        checklyFlags[k] = args[idx + 1] === undefined ? true : args[idx + 1]
+        idx++
+      }
+    } else {
+      playwrightFlags.push(arg)
+    }
+    idx++
+  }
+  return { checklyFlags, playwrightFlags }
+}
diff --git a/packages/cli/src/helpers/write-config-helpers.ts b/packages/cli/src/helpers/write-config-helpers.ts
new file mode 100644
index 000000000..5c8d6e6c8
--- /dev/null
+++ b/packages/cli/src/helpers/write-config-helpers.ts
@@ -0,0 +1,62 @@
+import * as recast from 'recast'
+import path from 'node:path'
+import fs from 'node:fs/promises'
+import { namedTypes as n } from 'ast-types';
+
+
+export function findPropertyByName (ast: any, name: string): recast.types.namedTypes.Property | undefined {
+  let node
+  recast.visit(ast, {
+    visitProperty (path: any) {
+      if (path.node.key.name === name) {
+        node = path.node
+      }
+      return false
+    },
+  })
+  return node
+}
+
+export function addOrReplaceItem(ast: any, node: any, name: string) {
+  const item = findPropertyByName(ast, name)
+  if (item) {
+    item.value = node.value
+  } else {
+    ast.properties.push(node)
+  }
+}
+
+export function addItemToArray(ast: any, node: any, name: string) {
+  if (!n.ObjectExpression.check(ast)) {
+    throw new Error('AST node is not an ObjectExpression');
+  }
+
+  const item = findPropertyByName(ast, name);
+
+  if (item) {
+    if (n.ArrayExpression.check(item.value)) {
+      item.value.elements = item.value.elements || [];
+      item.value.elements.push(node);
+    } else {
+      item.value = {
+        type: 'ArrayExpression',
+        elements: [node]
+      };
+    }
+  } else {
+    ast.properties.push({
+      type: 'Property',
+      key: { type: 'Identifier', name },
+      value: { type: 'ArrayExpression', elements: [node] },
+      kind: 'init',
+      computed: false,
+      method: false,
+      shorthand: false,
+    });
+  }
+}
+
+
+export async function reWriteChecklyConfigFile (data: string, fileName: string, dir: string) {
+  await fs.writeFile(path.join(dir, fileName), data)
+}
diff --git a/packages/cli/src/services/checkly-config-loader.ts b/packages/cli/src/services/checkly-config-loader.ts
index fba5a93bd..b341d9ea3 100644
--- a/packages/cli/src/services/checkly-config-loader.ts
+++ b/packages/cli/src/services/checkly-config-loader.ts
@@ -129,7 +129,7 @@ export async function getChecklyConfigFile (): Promise<{checklyConfig: string, f
 
 export class ConfigNotFoundError extends Error {}
 
-export async function loadChecklyConfig (dir: string, filenames = ['checkly.config.ts', 'checkly.config.mts', 'checkly.config.cts', 'checkly.config.js', 'checkly.config.mjs', 'checkly.config.cjs']): Promise<{ config: ChecklyConfig, constructs: Construct[] }> {
+export async function loadChecklyConfig (dir: string, filenames = ['checkly.config.ts', 'checkly.config.mts', 'checkly.config.cts', 'checkly.config.js', 'checkly.config.mjs', 'checkly.config.cjs'], writeChecklyConfig: boolean = true): Promise<{ config: ChecklyConfig, constructs: Construct[] }> {
   let config: ChecklyConfig | undefined
   Session.loadingChecklyConfigFile = true
   Session.checklyConfigFileConstructs = []
@@ -143,11 +143,9 @@ export async function loadChecklyConfig (dir: string, filenames = ['checkly.conf
     config = await Session.loadFile<ChecklyConfig>(filePath)
     break
   }
-
   if (!config) {
-    config = await handleMissingConfig(dir, filenames)
+    config = await handleMissingConfig(dir, filenames, writeChecklyConfig)
   }
-
   validateConfigFields(config, ['logicalId', 'projectName'] as const)
 
   const constructs = Session.checklyConfigFileConstructs
@@ -159,12 +157,14 @@ export async function loadChecklyConfig (dir: string, filenames = ['checkly.conf
   return { config, constructs }
 }
 
-async function handleMissingConfig (dir: string, filenames: string[]): Promise<ChecklyConfig> {
+async function handleMissingConfig (dir: string, filenames: string[], shouldWriteConfig: boolean = true): Promise<ChecklyConfig> {
   const baseName = path.basename(dir)
   const playwrightConfigPath = findPlaywrightConfigPath(dir)
   if (playwrightConfigPath) {
     const checklyConfig = getDefaultChecklyConfig(baseName, `./${path.relative(dir, playwrightConfigPath)}`)
-    await writeChecklyConfigFile(dir, checklyConfig)
+    if (shouldWriteConfig) {
+      await writeChecklyConfigFile(dir, checklyConfig)
+    }
     return checklyConfig
   }
   throw new ConfigNotFoundError(`Unable to locate a config at ${dir} with ${filenames.join(', ')}.`)
diff --git a/packages/cli/src/services/util.ts b/packages/cli/src/services/util.ts
index feef6b13f..804210abd 100644
--- a/packages/cli/src/services/util.ts
+++ b/packages/cli/src/services/util.ts
@@ -11,7 +11,7 @@ import archiver from 'archiver'
 import type { Archiver } from 'archiver'
 import { glob } from 'glob'
 import os from 'node:os'
-import { ChecklyConfig } from './checkly-config-loader'
+import { ChecklyConfig, PlaywrightSlimmedProp } from './checkly-config-loader'
 import { Parser } from './check-parser/parser'
 import * as JSON5 from 'json5'
 import { PlaywrightConfig } from './playwright-config'
@@ -291,20 +291,19 @@ export function cleanup (dir: string) {
   return fs.rm(dir, { recursive: true, force: true })
 }
 
-export function getDefaultChecklyConfig (directoryName: string, playwrightConfigPath: string): ChecklyConfig {
+export function getDefaultChecklyConfig (directoryName: string, playwrightConfigPath: string, playwrightCheck: PlaywrightSlimmedProp | null = null): ChecklyConfig {
+  const check = playwrightCheck || {
+    logicalId: directoryName,
+    name: directoryName,
+    frequency: 10,
+    locations: ['us-east-1'],
+  }
   return {
     logicalId: directoryName,
     projectName: directoryName,
     checks: {
       playwrightConfigPath,
-      playwrightChecks: [
-        {
-          logicalId: directoryName,
-          name: directoryName,
-          frequency: 10,
-          locations: ['us-east-1'],
-        },
-      ],
+      playwrightChecks: [check],
       frequency: 10,
       locations: ['us-east-1'],
     },

From 1f25f463d37c6e196dafbbd5598157e567786579 Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Thu, 19 Jun 2025 11:51:36 +0200
Subject: [PATCH 05/16] feat: use separator -- for flags

---
 packages/cli/src/commands/pw-test.ts    | 81 +++++++++++--------------
 packages/cli/src/helpers/test-helper.ts | 28 +++------
 2 files changed, 43 insertions(+), 66 deletions(-)

diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts
index b6635555e..a6f304e3f 100644
--- a/packages/cli/src/commands/pw-test.ts
+++ b/packages/cli/src/commands/pw-test.ts
@@ -23,8 +23,7 @@ import type { Region } from '..'
 import path from 'node:path'
 import * as recast from 'recast'
 import {
-  addItemToArray,
-  addOrReplaceItem,
+  addItemToArray, addOrReplaceItem,
   findPropertyByName,
   reWriteChecklyConfigFile
 } from '../helpers/write-config-helpers'
@@ -39,10 +38,12 @@ export default class PwTestCommand extends AuthCommand {
   static description = 'Test your Playwright Tests on Checkly'
   static state = 'beta'
   static flags = {
-    'cly-location': Flags.string({
+    'location': Flags.string({
+      char: 'l',
+      default: DEFAULT_REGION,
       description: 'The location to run the checks at.',
     }),
-    'cly-private-location': Flags.string({
+    'private-location': Flags.string({
       description: 'The private location to run checks at.',
       exclusive: ['location'],
     }),
@@ -57,56 +58,52 @@ export default class PwTestCommand extends AuthCommand {
       description: 'dotenv file path to be passed. For example --env-file="./.env"',
       exclusive: ['env'],
     }),
-    'cly-timeout': Flags.integer({
+    'timeout': Flags.integer({
       default: DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
       description: 'A timeout (in seconds) to wait for checks to complete.',
     }),
-    'cly-verbose': Flags.boolean({
+    'verbose': Flags.boolean({
       description: 'Always show the full logs of the checks.',
     }),
-    'cly-reporter': Flags.string({
+    'reporter': Flags.string({
       description: 'A list of custom reporters for the test output.',
       options: ['list', 'dot', 'ci', 'github', 'json'],
     }),
-    'cly-config': Flags.string({
+    'config': Flags.string({
       description: commonMessages.configFile,
     }),
-    'cly-skip-record': Flags.boolean({
+    'skip-record': Flags.boolean({
       description: 'Record test results in Checkly as a test session with full logs, traces and videos.',
       default: false,
     }),
-    'cly-test-session-name': Flags.string({
+    'test-session-name': Flags.string({
       description: 'A name to use when storing results in Checkly',
     }),
-    'cly-create-check': Flags.boolean({
+    'create-check': Flags.boolean({
       description: 'Create a Checkly check from the Playwright test.',
-      default: true,
+      default: false,
     })
   }
 
-
   async run(): Promise<void> {
     this.style.actionStart('Parsing your Playwright project')
-    const rawArgs = this.argv || []
-    const { checklyFlags, playwrightFlags } = splitChecklyAndPlaywrightFlags(rawArgs)
-    if (!this.validChecklyFlags(checklyFlags)) {
-      this.style.actionFailure()
-      this.style.shortError('Invalid Checkly flags provided. Please check the command usage.')
-      this.exit(1)
-    }
+
+    const { checklyFlags, playwrightFlags } = splitChecklyAndPlaywrightFlags(this.argv)
+
+    const { flags } = await this.parse(PwTestCommand, checklyFlags)
     const {
-      location: runLocation = DEFAULT_REGION,
+      location: runLocation,
       'private-location': privateRunLocation,
       env = [],
       'env-file': envFile,
-      timeout = DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
+      timeout,
       verbose: verboseFlag,
       reporter: reporterFlag,
       config: configFilename,
-      'skip-record': skipRecord = false,
+      'skip-record': skipRecord,
       'test-session-name': testSessionName,
-      'create-check': createCheck = false,
-    } = checklyFlags
+      'create-check': createCheck,
+    } = flags
     const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
     const {
       config: checklyConfig,
@@ -293,7 +290,13 @@ export default class PwTestCommand extends AuthCommand {
 
     }
     static createPlaywrightCheck(args: string[], runLocation: keyof Region): PlaywrightSlimmedProp {
-    const input = args.join(' ') || ''
+    const parseArgs = args.map(arg => {
+      if (arg.includes(' ')) {
+        arg = `"${arg}"`
+      }
+      return arg
+    })
+    const input = parseArgs.join(' ') || ''
       const inputLogicalId = input.replace(/[^a-zA-Z0-9]/g, '-').toLowerCase().substring(0, 50)
      return {
         logicalId: `playwright-check-${inputLogicalId}`,
@@ -304,24 +307,6 @@ export default class PwTestCommand extends AuthCommand {
       }
   }
 
-  private validChecklyFlags (checklyFlags: Record<string, any>) {
-    const validFlags = [
-      'location',
-      'private-location',
-      'env',
-      'env-file',
-      'list',
-      'timeout',
-      'verbose',
-      'reporter',
-      'config',
-      'skip-record',
-      'test-session-name',
-      'create-check'
-    ]
-    return Object.keys(checklyFlags).every(flag => validFlags.includes(flag))
-  }
-
   private getConfigPath (playwrightFlags: string[]) {
     for (let i = 0; i < playwrightFlags.length; i++) {
       const arg = playwrightFlags[i]
@@ -337,7 +322,8 @@ export default class PwTestCommand extends AuthCommand {
 
     const configFile = await getChecklyConfigFile()
     if (!configFile) {
-      this.style.longInfo('No Checkly config file found', 'Creating a default checkly config file.')
+      this.style.shortWarning('No Checkly config file found')
+      this.style.shortInfo('Creating a default checkly config file.')
       const checklyConfig = getDefaultChecklyConfig(baseName, `./${path.relative(dir, playwrightConfigPath)}`, playwrightCheck)
       await writeChecklyConfigFile(dir, checklyConfig)
       this.style.actionSuccess()
@@ -352,16 +338,17 @@ export default class PwTestCommand extends AuthCommand {
 
       return
     }
+    const playwrightConfigPathNode = recast.parse(`const playwrightConfig = '${playwrightConfigPath}'`)
+
     const playwrightCheckString = `const playwrightCheck = ${JSON5.stringify(playwrightCheck, { space: 2 })}`
     const playwrightCheckAst = recast.parse(playwrightCheckString)
     const playwrightCheckNode = playwrightCheckAst.program.body[0].declarations[0].init;
-
+    addOrReplaceItem(checksAst.value, playwrightConfigPathNode.value, 'playwrightConfig')
     addItemToArray(checksAst.value, playwrightCheckNode, 'playwrightChecks')
     const checklyConfigData = recast.print(checklyAst, { tabWidth: 2 }).code
     const writeDir = path.resolve(path.dirname(configFile.fileName))
     await reWriteChecklyConfigFile(checklyConfigData, configFile.fileName, writeDir)
     this.style.actionSuccess()
-
     return
 
   }
diff --git a/packages/cli/src/helpers/test-helper.ts b/packages/cli/src/helpers/test-helper.ts
index 60fe9453a..4c9c6a47d 100644
--- a/packages/cli/src/helpers/test-helper.ts
+++ b/packages/cli/src/helpers/test-helper.ts
@@ -57,25 +57,15 @@ export function prepareReportersTypes (reporterFlag: ReporterType, cliReporters:
 }
 
 export function splitChecklyAndPlaywrightFlags(args: string[]) {
-  const checklyFlags: Record<string, any> = {}
-  const playwrightFlags: string[] = []
-  let idx = 0;
-  while (idx < args.length) {
-    const arg = args[idx]
-    if (arg.startsWith('--cly-') || arg === '-e' || arg === '--env' || arg === '--env-file') {
-      if (arg.includes('=')) {
-        const split = arg.split(/=(.*)/s)
-        const k: string = split[0].replace('--cly-', '')
-        checklyFlags[k] = split[1]
-      } else {
-        const k: string = arg.replace('--cly-', '')
-        checklyFlags[k] = args[idx + 1] === undefined ? true : args[idx + 1]
-        idx++
-      }
-    } else {
-      playwrightFlags.push(arg)
-    }
-    idx++
+  const separatorIndex = args.indexOf('--');
+  let checklyFlags: string[] = [];
+  let playwrightFlags: string[] = [];
+
+  if (separatorIndex !== -1) {
+    checklyFlags = args.slice(0, separatorIndex);
+    playwrightFlags = args.slice(separatorIndex + 1);
+  } else {
+    checklyFlags = args;
   }
   return { checklyFlags, playwrightFlags }
 }

From 7c7e1512f3532f57f61359f71894f19d32d79004 Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Thu, 19 Jun 2025 11:57:22 +0200
Subject: [PATCH 06/16] fix: test

---
 packages/cli/e2e/__tests__/help.spec.ts | 1 +
 packages/cli/src/commands/pw-test.ts    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/cli/e2e/__tests__/help.spec.ts b/packages/cli/e2e/__tests__/help.spec.ts
index 3d3056785..316abca70 100644
--- a/packages/cli/e2e/__tests__/help.spec.ts
+++ b/packages/cli/e2e/__tests__/help.spec.ts
@@ -44,6 +44,7 @@ describe('help', () => {
     })
     expect(stdout).toContain(`CORE COMMANDS
   deploy   Deploy your project to your Checkly account.
+  pw-test  Test your Playwright Tests on Checkly.
   test     Test your checks on Checkly.
   trigger  Trigger your existing checks on Checkly.`)
 
diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts
index a6f304e3f..c5224f092 100644
--- a/packages/cli/src/commands/pw-test.ts
+++ b/packages/cli/src/commands/pw-test.ts
@@ -35,7 +35,7 @@ const DEFAULT_REGION = 'eu-central-1'
 export default class PwTestCommand extends AuthCommand {
   static coreCommand = true
   static hidden = false
-  static description = 'Test your Playwright Tests on Checkly'
+  static description = 'Test your Playwright Tests on Checkly.'
   static state = 'beta'
   static flags = {
     'location': Flags.string({

From 82889bf7a7af38a53a070b5246c63df65fd25da1 Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Thu, 19 Jun 2025 12:26:03 +0200
Subject: [PATCH 07/16] feat: allow using different package managers

---
 packages/cli/src/commands/pw-test.ts | 47 ++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 13 deletions(-)

diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts
index c5224f092..e9437d1f4 100644
--- a/packages/cli/src/commands/pw-test.ts
+++ b/packages/cli/src/commands/pw-test.ts
@@ -28,6 +28,7 @@ import {
   reWriteChecklyConfigFile
 } from '../helpers/write-config-helpers'
 import * as JSON5 from 'json5'
+import fs from 'node:fs/promises'
 
 const DEFAULT_REGION = 'eu-central-1'
 
@@ -111,7 +112,8 @@ export default class PwTestCommand extends AuthCommand {
     } = await loadChecklyConfig(configDirectory, configFilenames, false)
 
     const playwrightConfigPath = this.getConfigPath(playwrightFlags) ?? checklyConfig.checks?.playwrightConfigPath
-    const playwrightCheck = PwTestCommand.createPlaywrightCheck(playwrightFlags, runLocation as keyof Region)
+    const dir = path.dirname(playwrightConfigPath || '.')
+    const playwrightCheck = await PwTestCommand.createPlaywrightCheck(playwrightFlags, runLocation as keyof Region, dir)
     if (createCheck) {
       this.style.actionStart('Creating Checkly check from Playwright test')
       await this.createPlaywrightCheck(playwrightCheck, playwrightConfigPath)
@@ -286,22 +288,22 @@ export default class PwTestCommand extends AuthCommand {
       process.exitCode = 1
     })
     await runner.run()
-
-
     }
-    static createPlaywrightCheck(args: string[], runLocation: keyof Region): PlaywrightSlimmedProp {
-    const parseArgs = args.map(arg => {
-      if (arg.includes(' ')) {
-        arg = `"${arg}"`
-      }
-      return arg
-    })
-    const input = parseArgs.join(' ') || ''
+
+    static async createPlaywrightCheck(args: string[], runLocation: keyof Region, dir: string): Promise<PlaywrightSlimmedProp> {
+      const parseArgs = args.map(arg => {
+        if (arg.includes(' ')) {
+          arg = `"${arg}"`
+        }
+        return arg
+      })
+      const packageManager = await PwTestCommand.getPackageManagerExecutable(dir)
+      const input = parseArgs.join(' ') || ''
       const inputLogicalId = input.replace(/[^a-zA-Z0-9]/g, '-').toLowerCase().substring(0, 50)
-     return {
+      return {
         logicalId: `playwright-check-${inputLogicalId}`,
         name: `Playwright Test: ${input}`,
-        testCommand: `npx playwright test ${input}`,
+        testCommand: `${packageManager} playwright test ${input}`,
         locations: [runLocation],
         frequency: 10,
       }
@@ -352,4 +354,23 @@ export default class PwTestCommand extends AuthCommand {
     return
 
   }
+
+  private static async getPackageManagerExecutable(directoryPath: string): Promise<string| undefined> {
+    const packageManagers = [
+      { name: 'npx', lockFile: 'package-lock.json' },
+      { name: 'yarn', lockFile: 'yarn.lock' },
+      { name: 'pnpm', lockFile: 'pnpm-lock.yaml' },
+    ];
+
+    for (const pm of packageManagers) {
+      const lockFilePath = path.join(directoryPath, pm.lockFile);
+      try {
+        await fs.access(lockFilePath);
+        return pm.name;
+      } catch (error) {
+        continue;
+      }
+    }
+    return
+  }
 }

From 29ac4ae30958015df6a3b1259172584d8f7c4115 Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Fri, 20 Jun 2025 10:33:01 +0200
Subject: [PATCH 08/16] feat: add tests

---
 .../checkly.config.original.ts                | 22 ++++++
 .../test-pwt-native/checkly.config.ts         | 22 ++++++
 .../test-pwt-native/package-lock.json         | 74 +++++++++++++++++++
 .../fixtures/test-pwt-native/package.json     |  7 ++
 .../test-pwt-native/playwright.config.ts      | 17 +++++
 .../test-project/pw-test-project.test.ts      |  7 ++
 .../test/pw-test-example.test.ts              | 10 +++
 packages/cli/e2e/__tests__/pw-test.spec.ts    | 44 +++++++++++
 packages/cli/src/commands/pw-test.ts          | 12 ++-
 9 files changed, 211 insertions(+), 4 deletions(-)
 create mode 100644 packages/cli/e2e/__tests__/fixtures/test-pwt-native/checkly.config.original.ts
 create mode 100644 packages/cli/e2e/__tests__/fixtures/test-pwt-native/checkly.config.ts
 create mode 100644 packages/cli/e2e/__tests__/fixtures/test-pwt-native/package-lock.json
 create mode 100644 packages/cli/e2e/__tests__/fixtures/test-pwt-native/package.json
 create mode 100644 packages/cli/e2e/__tests__/fixtures/test-pwt-native/playwright.config.ts
 create mode 100644 packages/cli/e2e/__tests__/fixtures/test-pwt-native/test-project/pw-test-project.test.ts
 create mode 100644 packages/cli/e2e/__tests__/fixtures/test-pwt-native/test/pw-test-example.test.ts
 create mode 100644 packages/cli/e2e/__tests__/pw-test.spec.ts

diff --git a/packages/cli/e2e/__tests__/fixtures/test-pwt-native/checkly.config.original.ts b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/checkly.config.original.ts
new file mode 100644
index 000000000..25efa265f
--- /dev/null
+++ b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/checkly.config.original.ts
@@ -0,0 +1,22 @@
+import { defineConfig } from 'checkly'
+
+const config = defineConfig({
+  projectName: 'Test Project',
+  logicalId: 'test-project',
+  repoUrl: 'https://github.com/checkly/checkly-cli',
+  checks: {
+    locations: ['us-east-1', 'eu-west-1'],
+    tags: ['mac'],
+    runtimeId: '2022.10',
+    checkMatch: '**/*.check.ts',
+    browserChecks: {
+      // using .test.ts suffix (no .spec.ts) to avoid '@playwright/test not found error' when Jest transpile the spec.ts
+      testMatch: '**/__checks__/*.test.ts',
+    },
+  },
+  cli: {
+    runLocation: 'us-east-1',
+  },
+})
+
+export default config
diff --git a/packages/cli/e2e/__tests__/fixtures/test-pwt-native/checkly.config.ts b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/checkly.config.ts
new file mode 100644
index 000000000..25efa265f
--- /dev/null
+++ b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/checkly.config.ts
@@ -0,0 +1,22 @@
+import { defineConfig } from 'checkly'
+
+const config = defineConfig({
+  projectName: 'Test Project',
+  logicalId: 'test-project',
+  repoUrl: 'https://github.com/checkly/checkly-cli',
+  checks: {
+    locations: ['us-east-1', 'eu-west-1'],
+    tags: ['mac'],
+    runtimeId: '2022.10',
+    checkMatch: '**/*.check.ts',
+    browserChecks: {
+      // using .test.ts suffix (no .spec.ts) to avoid '@playwright/test not found error' when Jest transpile the spec.ts
+      testMatch: '**/__checks__/*.test.ts',
+    },
+  },
+  cli: {
+    runLocation: 'us-east-1',
+  },
+})
+
+export default config
diff --git a/packages/cli/e2e/__tests__/fixtures/test-pwt-native/package-lock.json b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/package-lock.json
new file mode 100644
index 000000000..35e27a0c8
--- /dev/null
+++ b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/package-lock.json
@@ -0,0 +1,74 @@
+{
+  "name": "test-pwt-native",
+  "version": "0.0.1",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "test-pwt-native",
+      "version": "0.0.1",
+      "devDependencies": {
+        "@playwright/test": "1.53.1"
+      }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.53.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.53.1.tgz",
+      "integrity": "sha512-Z4c23LHV0muZ8hfv4jw6HngPJkbbtZxTkxPNIg7cJcTc9C28N/p2q7g3JZS2SiKBBHJ3uM1dgDye66bB7LEk5w==",
+      "dev": true,
+      "dependencies": {
+        "playwright": "1.53.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.53.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.53.1.tgz",
+      "integrity": "sha512-LJ13YLr/ocweuwxyGf1XNFWIU4M2zUSo149Qbp+A4cpwDjsxRPj7k6H25LBrEHiEwxvRbD8HdwvQmRMSvquhYw==",
+      "dev": true,
+      "dependencies": {
+        "playwright-core": "1.53.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.53.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.53.1.tgz",
+      "integrity": "sha512-Z46Oq7tLAyT0lGoFx4DOuB1IA9D1TPj0QkYxpPVUnGDqHHvDpCftu1J2hM2PiWsNMoZh8+LQaarAWcDfPBc6zg==",
+      "dev": true,
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    }
+  }
+}
diff --git a/packages/cli/e2e/__tests__/fixtures/test-pwt-native/package.json b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/package.json
new file mode 100644
index 000000000..78a102b14
--- /dev/null
+++ b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/package.json
@@ -0,0 +1,7 @@
+{
+  "name": "test-pwt-native",
+  "version": "0.0.1",
+  "devDependencies": {
+    "@playwright/test": "1.53.1"
+  }
+}
diff --git a/packages/cli/e2e/__tests__/fixtures/test-pwt-native/playwright.config.ts b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/playwright.config.ts
new file mode 100644
index 000000000..c783e75de
--- /dev/null
+++ b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/playwright.config.ts
@@ -0,0 +1,17 @@
+import { defineConfig, devices } from '@playwright/test'
+
+export default defineConfig({
+  timeout: 1234,
+  projects: [
+    {
+      name: 'test-example',
+      testDir: './test',
+      use: { ...devices['Desktop Chrome'] },
+    },
+    {
+      name: 'test-project',
+      testDir: './test-project',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+})
diff --git a/packages/cli/e2e/__tests__/fixtures/test-pwt-native/test-project/pw-test-project.test.ts b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/test-project/pw-test-project.test.ts
new file mode 100644
index 000000000..30f48c108
--- /dev/null
+++ b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/test-project/pw-test-project.test.ts
@@ -0,0 +1,7 @@
+// @ts-ignore
+import { test, expect } from '@playwright/test'
+
+
+test('Tag B test project @TAG-B', async ({ page }) => {
+    expect(true).toBe(true);
+});
diff --git a/packages/cli/e2e/__tests__/fixtures/test-pwt-native/test/pw-test-example.test.ts b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/test/pw-test-example.test.ts
new file mode 100644
index 000000000..4cbe5aad1
--- /dev/null
+++ b/packages/cli/e2e/__tests__/fixtures/test-pwt-native/test/pw-test-example.test.ts
@@ -0,0 +1,10 @@
+// @ts-ignore
+import { test, expect } from '@playwright/test'
+
+test('Tag A test example @TAG-A', async ({ page }) => {
+    expect(true).toBe(true);
+});
+
+test('Tag B test example @TAG-B', async ({ page }) => {
+    expect(true).toBe(true);
+});
diff --git a/packages/cli/e2e/__tests__/pw-test.spec.ts b/packages/cli/e2e/__tests__/pw-test.spec.ts
new file mode 100644
index 000000000..7b5828771
--- /dev/null
+++ b/packages/cli/e2e/__tests__/pw-test.spec.ts
@@ -0,0 +1,44 @@
+import path from 'node:path'
+import fs from 'node:fs'
+
+import config from 'config'
+import { describe, it, expect, afterEach } from 'vitest'
+
+import { runChecklyCli } from '../run-checkly'
+import { loadChecklyConfig } from '../../src/services/checkly-config-loader'
+
+describe('pw-test', { timeout: 45000 }, () => {
+  afterEach(async() => {
+    const configPath = path.join(__dirname, 'fixtures', 'test-pwt-native')
+    fs.copyFileSync(path.join(configPath, 'checkly.config.original.ts'), path.join(configPath, 'checkly.config.ts'))
+  })
+
+  it('Playwright test should run successfully', async () => {
+    const result = await runChecklyCli({
+      args: ['pw-test', '--', `--grep`, '@TAG-B'],
+      apiKey: config.get('apiKey'),
+      accountId: config.get('accountId'),
+      directory: path.join(__dirname, 'fixtures', 'test-pwt-native'),
+      timeout: 120000, // 2 minutes
+    })
+    expect(result.status).toBe(0)
+  }, 130000)
+
+  it('Should add a Playwright test to the config', async () => {
+    const result = await runChecklyCli({
+      args: ['pw-test', '--create-check', '--', `--grep`, '@TAG-B'],
+      apiKey: config.get('apiKey'),
+      accountId: config.get('accountId'),
+      directory: path.join(__dirname, 'fixtures', 'test-pwt-native'),
+      timeout: 120000, // 2 minutes
+    })
+    expect(result.status).toBe(0)
+    const checklyConfig = await loadChecklyConfig(path.join(__dirname, 'fixtures', 'test-pwt-native'))
+    expect(checklyConfig.config?.checks).toBeDefined()
+    expect(checklyConfig.config?.checks?.playwrightConfigPath).toBe('./playwright.config.ts')
+    expect(checklyConfig.config?.checks?.playwrightChecks).toBeDefined()
+    expect(checklyConfig.config?.checks?.playwrightChecks.length).toBe(1)
+    expect(checklyConfig.config?.checks?.playwrightChecks[0].name).toBe('Playwright Test: --grep @TAG-B')
+    expect(checklyConfig.config?.checks?.playwrightChecks[0].testCommand).toBe('npx playwright test --grep @TAG-B')
+  })
+})
diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts
index e9437d1f4..4c0ad5bea 100644
--- a/packages/cli/src/commands/pw-test.ts
+++ b/packages/cli/src/commands/pw-test.ts
@@ -110,7 +110,6 @@ export default class PwTestCommand extends AuthCommand {
       config: checklyConfig,
       constructs: checklyConfigConstructs,
     } = await loadChecklyConfig(configDirectory, configFilenames, false)
-
     const playwrightConfigPath = this.getConfigPath(playwrightFlags) ?? checklyConfig.checks?.playwrightConfigPath
     const dir = path.dirname(playwrightConfigPath || '.')
     const playwrightCheck = await PwTestCommand.createPlaywrightCheck(playwrightFlags, runLocation as keyof Region, dir)
@@ -340,12 +339,17 @@ export default class PwTestCommand extends AuthCommand {
 
       return
     }
-    const playwrightConfigPathNode = recast.parse(`const playwrightConfig = '${playwrightConfigPath}'`)
+    const b = recast.types.builders;
+    const playwrightPropertyNode = b.property(
+      'init',
+      b.identifier('playwrightConfigPath'),
+      b.stringLiteral(playwrightConfigPath)
+    );
 
     const playwrightCheckString = `const playwrightCheck = ${JSON5.stringify(playwrightCheck, { space: 2 })}`
     const playwrightCheckAst = recast.parse(playwrightCheckString)
     const playwrightCheckNode = playwrightCheckAst.program.body[0].declarations[0].init;
-    addOrReplaceItem(checksAst.value, playwrightConfigPathNode.value, 'playwrightConfig')
+    addOrReplaceItem(checksAst.value, playwrightPropertyNode, 'playwrightConfigPath')
     addItemToArray(checksAst.value, playwrightCheckNode, 'playwrightChecks')
     const checklyConfigData = recast.print(checklyAst, { tabWidth: 2 }).code
     const writeDir = path.resolve(path.dirname(configFile.fileName))
@@ -371,6 +375,6 @@ export default class PwTestCommand extends AuthCommand {
         continue;
       }
     }
-    return
+    return 'npx'; // Default to npx if no lock file is found
   }
 }

From 90caccff71a1f32a71325b2d192b91691ae63943 Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Thu, 26 Jun 2025 09:58:11 +0200
Subject: [PATCH 09/16] fix: throw error if property is not array

---
 packages/cli/src/helpers/write-config-helpers.ts | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/packages/cli/src/helpers/write-config-helpers.ts b/packages/cli/src/helpers/write-config-helpers.ts
index 5c8d6e6c8..329c318c7 100644
--- a/packages/cli/src/helpers/write-config-helpers.ts
+++ b/packages/cli/src/helpers/write-config-helpers.ts
@@ -38,10 +38,7 @@ export function addItemToArray(ast: any, node: any, name: string) {
       item.value.elements = item.value.elements || [];
       item.value.elements.push(node);
     } else {
-      item.value = {
-        type: 'ArrayExpression',
-        elements: [node]
-      };
+      throw new Error(`Property "${name}" exists but is not an array`);
     }
   } else {
     ast.properties.push({

From ee1ded07be868b4270bfed0e9419411e10a030d8 Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Thu, 3 Jul 2025 11:24:35 +0200
Subject: [PATCH 10/16] wip

---
 packages/cli/src/commands/pw-test.ts               |  3 +++
 packages/cli/src/reporters/abstract-list.ts        | 12 ++++++++++++
 packages/cli/src/reporters/reporter.ts             |  1 +
 packages/cli/src/services/abstract-check-runner.ts |  9 ++++++++-
 4 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts
index 4c0ad5bea..b7bb534df 100644
--- a/packages/cli/src/commands/pw-test.ts
+++ b/packages/cli/src/commands/pw-test.ts
@@ -286,6 +286,9 @@ export default class PwTestCommand extends AuthCommand {
       reporters.forEach(r => r.onError(err))
       process.exitCode = 1
     })
+    runner.on(Events.STREAM_LOGS, (check: any, sequenceId: SequenceId, logs) => {
+      reporters.forEach(r => r.onStreamLogs(check, sequenceId, logs))
+    })
     await runner.run()
     }
 
diff --git a/packages/cli/src/reporters/abstract-list.ts b/packages/cli/src/reporters/abstract-list.ts
index c9edae8b1..b28c39475 100644
--- a/packages/cli/src/reporters/abstract-list.ts
+++ b/packages/cli/src/reporters/abstract-list.ts
@@ -20,6 +20,7 @@ export type checkFilesMap = Map<string|undefined, Map<SequenceId, {
   testResultId?: string,
   links?: TestResultsShortLinks,
   numRetries: number,
+  logs?: string[],
 }>>
 
 export default abstract class AbstractListReporter implements Reporter {
@@ -96,6 +97,17 @@ export default abstract class AbstractListReporter implements Reporter {
     printLn(chalk.red('Unable to run checks: ') + err.message)
   }
 
+  onStreamLogs (check: any, sequenceId: SequenceId, logs: any) {
+    const checkFile = this.checkFilesMap!.get(check.getSourceFile?.())!.get(sequenceId)!
+    const logList = logs.logs || []
+    if (!checkFile.logs) {
+      checkFile.logs = []
+    }
+    // checkFile.logs.push(...logs.logs)
+    logList.forEach((log: string) => printLn(log, 2))
+    return
+  }
+
   // Clear the summary which was printed by _printStatus from stdout
   // TODO: Rather than clearing the whole status bar, we could overwrite the exact lines that changed.
   // This might look a bit smoother and reduce the flickering effects.
diff --git a/packages/cli/src/reporters/reporter.ts b/packages/cli/src/reporters/reporter.ts
index c3b32a65e..7a0bb218b 100644
--- a/packages/cli/src/reporters/reporter.ts
+++ b/packages/cli/src/reporters/reporter.ts
@@ -14,6 +14,7 @@ export interface Reporter {
   onEnd(): void;
   onError(err: Error): void,
   onSchedulingDelayExceeded(): void
+  onStreamLogs(check: any, sequenceId: SequenceId, logs: string[]): void
 }
 
 export type ReporterType = 'list' | 'dot' | 'ci' | 'github' | 'json'
diff --git a/packages/cli/src/services/abstract-check-runner.ts b/packages/cli/src/services/abstract-check-runner.ts
index 73ee504cd..0587326fe 100644
--- a/packages/cli/src/services/abstract-check-runner.ts
+++ b/packages/cli/src/services/abstract-check-runner.ts
@@ -18,7 +18,8 @@ export enum Events {
   RUN_STARTED = 'RUN_STARTED',
   RUN_FINISHED = 'RUN_FINISHED',
   ERROR = 'ERROR',
-  MAX_SCHEDULING_DELAY_EXCEEDED = 'MAX_SCHEDULING_DELAY_EXCEEDED'
+  MAX_SCHEDULING_DELAY_EXCEEDED = 'MAX_SCHEDULING_DELAY_EXCEEDED',
+  STREAM_LOGS = 'STREAM_LOGS',
 }
 
 export type PrivateRunLocation = {
@@ -160,6 +161,12 @@ export default abstract class AbstractCheckRunner extends EventEmitter {
       this.disableTimeout(sequenceId)
       this.emit(Events.CHECK_FAILED, sequenceId, check, message)
       this.emit(Events.CHECK_FINISHED, check)
+    } else if (subtopic === 'stream-logs') {
+      const buffer = Buffer.from(message.data)
+      const jsonString = buffer.toString('utf-8');
+      const obj = JSON.parse(jsonString);
+      this.emit(Events.STREAM_LOGS, check, sequenceId, obj)
+
     }
   }
 

From 0c829fd1fd5bbf9996565f69a07d914642b21c9d Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Fri, 11 Jul 2025 10:56:18 +0200
Subject: [PATCH 11/16] wip

---
 packages/cli/src/commands/pw-test.ts               | 6 ++++++
 packages/cli/src/reporters/abstract-list.ts        | 4 ++--
 packages/cli/src/rest/test-sessions.ts             | 1 +
 packages/cli/src/services/abstract-check-runner.ts | 6 ++----
 packages/cli/src/services/test-runner.ts           | 4 ++++
 5 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts
index b7bb534df..78fb98f0d 100644
--- a/packages/cli/src/commands/pw-test.ts
+++ b/packages/cli/src/commands/pw-test.ts
@@ -83,6 +83,10 @@ export default class PwTestCommand extends AuthCommand {
     'create-check': Flags.boolean({
       description: 'Create a Checkly check from the Playwright test.',
       default: false,
+    }),
+    'stream-logs': Flags.boolean({
+      description: 'Stream logs from the test run to the console.',
+      default: false,
     })
   }
 
@@ -104,6 +108,7 @@ export default class PwTestCommand extends AuthCommand {
       'skip-record': skipRecord,
       'test-session-name': testSessionName,
       'create-check': createCheck,
+      'stream-logs': streamLogs,
     } = flags
     const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
     const {
@@ -235,6 +240,7 @@ export default class PwTestCommand extends AuthCommand {
       configDirectory,
       // TODO: ADD PROPER RETRY STRATEGY HANDLING
       null, // testRetryStrategy
+      streamLogs,
     )
 
     runner.on(Events.RUN_STARTED,
diff --git a/packages/cli/src/reporters/abstract-list.ts b/packages/cli/src/reporters/abstract-list.ts
index b28c39475..ed6f77699 100644
--- a/packages/cli/src/reporters/abstract-list.ts
+++ b/packages/cli/src/reporters/abstract-list.ts
@@ -97,9 +97,9 @@ export default abstract class AbstractListReporter implements Reporter {
     printLn(chalk.red('Unable to run checks: ') + err.message)
   }
 
-  onStreamLogs (check: any, sequenceId: SequenceId, logs: any) {
+  onStreamLogs (check: any, sequenceId: SequenceId, logs: string[] | undefined) {
     const checkFile = this.checkFilesMap!.get(check.getSourceFile?.())!.get(sequenceId)!
-    const logList = logs.logs || []
+    const logList = logs || []
     if (!checkFile.logs) {
       checkFile.logs = []
     }
diff --git a/packages/cli/src/rest/test-sessions.ts b/packages/cli/src/rest/test-sessions.ts
index 479f09dd4..9021a222b 100644
--- a/packages/cli/src/rest/test-sessions.ts
+++ b/packages/cli/src/rest/test-sessions.ts
@@ -12,6 +12,7 @@ type RunTestSessionRequest = {
   repoInfo?: GitInformation | null,
   environment?: string | null,
   shouldRecord: boolean,
+  streamLogs: boolean | null,
 }
 
 type TriggerTestSessionRequest = {
diff --git a/packages/cli/src/services/abstract-check-runner.ts b/packages/cli/src/services/abstract-check-runner.ts
index 0587326fe..1f84b7eb4 100644
--- a/packages/cli/src/services/abstract-check-runner.ts
+++ b/packages/cli/src/services/abstract-check-runner.ts
@@ -162,10 +162,8 @@ export default abstract class AbstractCheckRunner extends EventEmitter {
       this.emit(Events.CHECK_FAILED, sequenceId, check, message)
       this.emit(Events.CHECK_FINISHED, check)
     } else if (subtopic === 'stream-logs') {
-      const buffer = Buffer.from(message.data)
-      const jsonString = buffer.toString('utf-8');
-      const obj = JSON.parse(jsonString);
-      this.emit(Events.STREAM_LOGS, check, sequenceId, obj)
+      const { logs } = message
+      this.emit(Events.STREAM_LOGS, check, sequenceId, logs)
 
     }
   }
diff --git a/packages/cli/src/services/test-runner.ts b/packages/cli/src/services/test-runner.ts
index c12ed1836..82339087d 100644
--- a/packages/cli/src/services/test-runner.ts
+++ b/packages/cli/src/services/test-runner.ts
@@ -21,6 +21,7 @@ export default class TestRunner extends AbstractCheckRunner {
   updateSnapshots: boolean
   baseDirectory: string
   testRetryStrategy: RetryStrategy | null
+  streamLogs: boolean | null
 
   constructor (
     accountId: string,
@@ -36,6 +37,7 @@ export default class TestRunner extends AbstractCheckRunner {
     updateSnapshots: boolean,
     baseDirectory: string,
     testRetryStrategy: RetryStrategy | null,
+    streamLogs: boolean | null,
   ) {
     super(accountId, timeout, verbose)
     this.projectBundle = projectBundle
@@ -48,6 +50,7 @@ export default class TestRunner extends AbstractCheckRunner {
     this.updateSnapshots = updateSnapshots
     this.baseDirectory = baseDirectory
     this.testRetryStrategy = testRetryStrategy
+    this.streamLogs = streamLogs
   }
 
   async scheduleChecks (
@@ -89,6 +92,7 @@ export default class TestRunner extends AbstractCheckRunner {
         repoInfo: this.repoInfo,
         environment: this.environment,
         shouldRecord: this.shouldRecord,
+        streamLogs: this.streamLogs,
       })
       const { testSessionId, sequenceIds } = data
       const checks = this.checkBundles.map(({ construct: check }) => ({ check, sequenceId: sequenceIds?.[check.logicalId] }))

From 55fb7fc23d6260414f9d652e72163e4336fb9011 Mon Sep 17 00:00:00 2001
From: Ferran Diaz <ferran@checklyhq.com>
Date: Mon, 14 Jul 2025 11:46:58 +0200
Subject: [PATCH 12/16] feat: add log streaming

---
 packages/cli/src/rest/test-sessions.ts   | 2 +-
 packages/cli/src/services/test-runner.ts | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/cli/src/rest/test-sessions.ts b/packages/cli/src/rest/test-sessions.ts
index 9021a222b..94b94e00c 100644
--- a/packages/cli/src/rest/test-sessions.ts
+++ b/packages/cli/src/rest/test-sessions.ts
@@ -12,7 +12,7 @@ type RunTestSessionRequest = {
   repoInfo?: GitInformation | null,
   environment?: string | null,
   shouldRecord: boolean,
-  streamLogs: boolean | null,
+  streamLogs?: boolean,
 }
 
 type TriggerTestSessionRequest = {
diff --git a/packages/cli/src/services/test-runner.ts b/packages/cli/src/services/test-runner.ts
index 82339087d..a6140f6ca 100644
--- a/packages/cli/src/services/test-runner.ts
+++ b/packages/cli/src/services/test-runner.ts
@@ -21,7 +21,7 @@ export default class TestRunner extends AbstractCheckRunner {
   updateSnapshots: boolean
   baseDirectory: string
   testRetryStrategy: RetryStrategy | null
-  streamLogs: boolean | null
+  streamLogs?: boolean
 
   constructor (
     accountId: string,
@@ -37,7 +37,7 @@ export default class TestRunner extends AbstractCheckRunner {
     updateSnapshots: boolean,
     baseDirectory: string,
     testRetryStrategy: RetryStrategy | null,
-    streamLogs: boolean | null,
+    streamLogs?: boolean,
   ) {
     super(accountId, timeout, verbose)
     this.projectBundle = projectBundle
@@ -50,7 +50,7 @@ export default class TestRunner extends AbstractCheckRunner {
     this.updateSnapshots = updateSnapshots
     this.baseDirectory = baseDirectory
     this.testRetryStrategy = testRetryStrategy
-    this.streamLogs = streamLogs
+    this.streamLogs = streamLogs ?? false
   }
 
   async scheduleChecks (

From edae1f92d621f0f4ea32b2c040e1c471b0229f77 Mon Sep 17 00:00:00 2001
From: Daniel Paulus <daniel.paulus@icloud.com>
Date: Mon, 21 Jul 2025 19:58:13 +0200
Subject: [PATCH 13/16] feat: add new mcp command

---
 packages/cli/src/commands/mcp.ts           | 138 +++++++++++
 packages/cli/src/templates/packed-files.ts | 256 +++++++++++++++++++++
 2 files changed, 394 insertions(+)
 create mode 100644 packages/cli/src/commands/mcp.ts
 create mode 100644 packages/cli/src/templates/packed-files.ts

diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts
new file mode 100644
index 000000000..d2b7381a4
--- /dev/null
+++ b/packages/cli/src/commands/mcp.ts
@@ -0,0 +1,138 @@
+import { AuthCommand } from './authCommand'
+import * as fs from 'node:fs/promises'
+import * as path from 'node:path'
+import * as os from 'node:os'
+import { spawn } from 'node:child_process'
+import { EXAMPLE_SPEC_TS, CHECKLY_CONFIG_TS, TSCONFIG_JSON, PACKAGE_JSON } from '../templates/packed-files'
+
+// Default playwright.config.ts since it's not in packed-files
+const PLAYWRIGHT_CONFIG_TS = `import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: 'html',
+  use: {
+    trace: 'on-first-retry',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});`
+
+export default class McpCommand extends AuthCommand {
+  static coreCommand = true
+  static hidden = false
+  static description = 'Run MCP server tests with Playwright on Checkly.'
+  static state = 'beta'
+
+  async run(): Promise<void> {
+    let tempDir: string | null = null
+    let pinggyUrl: string | null = null
+      // Output initial "in progress" message
+    this.log('starting mcp server in progress (this can take a while)..')
+    this.log('it will run for 20 minutes and then exit automatically')
+    try {
+      // Create temporary directory
+      tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-mcp-'))
+      
+      // Write all files to temp directory
+      await fs.writeFile(path.join(tempDir, 'example.spec.ts'), EXAMPLE_SPEC_TS)
+      await fs.writeFile(path.join(tempDir, 'checkly.config.ts'), CHECKLY_CONFIG_TS)
+      await fs.writeFile(path.join(tempDir, 'tsconfig.json'), TSCONFIG_JSON)
+      await fs.writeFile(path.join(tempDir, 'package.json'), PACKAGE_JSON)
+      await fs.writeFile(path.join(tempDir, 'playwright.config.ts'), PLAYWRIGHT_CONFIG_TS)
+      
+      // Run npm install in the temp directory
+      const { execa } = await import('execa')
+      await execa('npm', ['install'], {
+        cwd: tempDir,
+        stdio: 'pipe' // Changed to pipe to suppress output
+      })
+      
+    
+      
+      // Save current directory
+      const originalCwd = process.cwd()
+      
+      // Change to temp directory
+      process.chdir(tempDir)
+      
+      try {
+        // Run pw-test command with stream-logs enabled using spawn to capture output
+        const checklyPath = path.join(originalCwd, 'bin', 'run')
+        const args = ['pw-test', '--stream-logs', ...this.argv]
+        
+        const pwTestProcess = spawn(checklyPath, args, {
+          cwd: tempDir,
+          env: { ...process.env },
+          stdio: ['inherit', 'pipe', 'pipe']
+        })
+        
+        // Capture stdout
+        pwTestProcess.stdout?.on('data', (data) => {
+          const output = data.toString()
+          
+          // Parse for Pinggy URL
+          const urlMatch = output.match(/🌐 PINGGY PUBLIC URL:\s*(https:\/\/[a-z0-9-]+\.a\.free\.pinggy\.link)/)
+          if (urlMatch && !pinggyUrl) {
+            pinggyUrl = urlMatch[1]
+            // Output JSON immediately when URL is found
+            this.log(JSON.stringify({ serverUrl: pinggyUrl }))
+          }
+        })
+        
+        // Capture stderr as well
+        pwTestProcess.stderr?.on('data', (data) => {
+          const output = data.toString()
+          
+          // Also check stderr for URLs (pinggy might output there)
+          const urlMatch = output.match(/🌐 PINGGY PUBLIC URL:\s*(https:\/\/[a-z0-9-]+\.a\.free\.pinggy\.link)/)
+          if (urlMatch && !pinggyUrl) {
+            pinggyUrl = urlMatch[1]
+            // Output JSON immediately when URL is found
+            this.log(JSON.stringify({ serverUrl: pinggyUrl }))
+          }
+        })
+        
+        // Wait for process to complete
+        await new Promise<void>((resolve, reject) => {
+          pwTestProcess.on('close', (code) => {
+            if (code === 0) {
+              resolve()
+            } else {
+              reject(new Error(`Process exited with code ${code}`))
+            }
+          })
+          
+          pwTestProcess.on('error', (error) => {
+            reject(error)
+          })
+        })
+        
+      } finally {
+        // Always restore original directory
+        process.chdir(originalCwd)
+      }
+      
+    } catch (error) {
+      // Suppress error output to keep it clean
+      process.exit(1)
+    } finally {
+      // Clean up temp directory
+      if (tempDir) {
+        try {
+          await fs.rm(tempDir, { recursive: true, force: true })
+        } catch (cleanupError) {
+          // Suppress cleanup errors
+        }
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/packages/cli/src/templates/packed-files.ts b/packages/cli/src/templates/packed-files.ts
new file mode 100644
index 000000000..a0487fdf9
--- /dev/null
+++ b/packages/cli/src/templates/packed-files.ts
@@ -0,0 +1,256 @@
+// Packed files generated automatically
+
+export const EXAMPLE_SPEC_TS = `import { spawn } from 'child_process';
+import { test } from '@playwright/test';
+import * as net from 'net';
+
+test('has title', async ({ page }) => {
+  const browserPath = '/checkly/browsers/chromium-1181/chrome-linux/chrome';
+  const MCP_PORT = 8931;
+  const TEN_MINUTES_MS = 20 * 60 * 1000; // 20 minutes in milliseconds
+
+  // Set timeout for the test
+  test.setTimeout(TEN_MINUTES_MS);
+
+  // Function to check if a port is in use
+  function isPortInUse(port: number): Promise<boolean> {
+    return new Promise((resolve) => {
+      const server = net.createServer();
+      
+      server.listen(port, () => {
+        server.once('close', () => {
+          resolve(false); // Port is available
+        });
+        server.close();
+      });
+      
+      server.on('error', () => {
+        resolve(true); // Port is in use
+      });
+    });
+  }
+
+  // Function to run pinggy to expose MCP port
+  async function runPinggy(port: number) {
+    console.log(\`Starting pinggy to expose port \${port}...\`);
+    
+    const pinggyProcess = spawn('ssh', [
+      '-o', 'StrictHostKeyChecking=no',
+      '-o', 'UserKnownHostsFile=/dev/null',
+      '-o', 'LogLevel=ERROR',
+      '-p', '443', 
+      '-R', \`0:localhost:\${port}\`, 
+      'a.pinggy.io'
+    ], {
+      stdio: 'pipe',
+      shell: false
+    });
+
+    // Handle pinggy output
+    pinggyProcess.stdout?.on('data', (data) => {
+      const output = data.toString();
+      console.log(\`Pinggy stdout: \${output}\`);
+      
+      // Look for HTTPS URL in the output
+      const httpsMatch = output.match(/https:\\/\\/[a-z0-9-]+\\.a\\.free\\.pinggy\\.link/);
+      if (httpsMatch) {
+        const publicUrl = httpsMatch[0];
+        console.log(\`🌐 PINGGY PUBLIC URL: \${publicUrl}\`);
+      }
+    });
+
+    pinggyProcess.stderr?.on('data', (data) => {
+      const output = data.toString();
+      console.log(\`Pinggy stderr: \${output}\`);
+      
+      // Also check stderr for URLs (pinggy might output there)
+      const httpsMatch = output.match(/https:\\/\\/[a-z0-9-]+\\.a\\.free\\.pinggy\\.link/);
+      if (httpsMatch) {
+        const publicUrl = httpsMatch[0];
+        console.log(\`🌐 PINGGY PUBLIC URL: \${publicUrl}\`);
+      }
+    });
+
+    pinggyProcess.on('close', (code) => {
+      console.log(\`Pinggy process exited with code \${code}\`);
+    });
+
+    pinggyProcess.on('error', (error) => {
+      console.error(\`Pinggy process error: \${error}\`);
+    });
+
+    return pinggyProcess;
+  }
+
+  // Function to run MCP server
+  async function runMCPServer() {
+    // Check if port is already in use
+    const portInUse = await isPortInUse(MCP_PORT);
+    if (portInUse) {
+      console.log(\`Port \${MCP_PORT} is already in use. MCP server may already be running.\`);
+      return null;
+    }
+
+    console.log('Starting MCP server...');
+
+    const mcpProcess = spawn('npx', ['-y', '@playwright/mcp@latest', '--port', \`\${MCP_PORT}\`, '--executable-path', browserPath, '--isolated'], {
+      stdio: 'pipe',
+      shell: true
+    });
+
+    // Handle process output
+    mcpProcess.stdout?.on('data', (data) => {
+      console.log(\`MCP stdout: \${data.toString().trim()}\`);
+    });
+
+    mcpProcess.stderr?.on('data', (data) => {
+      console.log(\`MCP stderr: \${data.toString().trim()}\`);
+    });
+
+    mcpProcess.on('close', (code) => {
+      console.log(\`MCP process exited with code \${code}\`);
+    });
+
+    mcpProcess.on('error', (error) => {
+      console.error(\`MCP process error: \${error}\`);
+    });
+
+    return mcpProcess;
+  }
+
+  // Main async function to run all setup
+  async function setupAndRun() {
+    console.log("Running MCP server with a timeout of 30 seconds...");
+    const mcpServerProcess = await runMCPServer();
+    
+    if (mcpServerProcess) {
+      console.log("MCP server started. You can now run your Playwright tests.");
+      
+      // Start pinggy to expose the MCP server
+      console.log("Starting pinggy to expose MCP server...");
+      const pinggyProcess = await runPinggy(MCP_PORT);
+    } else {
+      console.log("MCP server not started (port already in use).");
+    }
+  }
+
+  // Run the setup
+  await setupAndRun();
+  
+  console.log("logging in a test");
+  
+  // Wait for the specified timeout
+  await new Promise(resolve => setTimeout(resolve, TEN_MINUTES_MS));
+});`;
+
+export const CHECKLY_CONFIG_TS = `import { defineConfig } from 'checkly'
+import { Frequency } from 'checkly/constructs'
+
+export default defineConfig({
+  projectName: 'MCP Server Instance',
+  logicalId: 'cool-website-monitoring',
+  repoUrl: 'https://github.com/acme/website',
+  checks: {
+    playwrightConfigPath: './playwright.config.ts',
+    playwrightChecks: [
+      {
+        /**
+         * Create a multi-browser check that runs
+         * every 10 mins in two locations.
+         */
+        logicalId: 'multi-browser',
+        name: 'Playwright MCP Server',
+        // Use one project (or multiple projects) defined in your Playwright config
+        pwProjects: ['chromium'],
+        // Use one tag (or multiple tags) defined in your spec files
+        //pwTags: '@smoke-tests',
+        frequency: Frequency.EVERY_10M,
+        locations: ['us-east-1', 'eu-west-1'],
+      },
+      // {
+      //   /**
+      //    * Create a check that runs the \`@critical\` tagged tests
+      //    * every 5 mins in three locations.
+      //    */
+      //   logicalId: 'critical-tagged',
+      //   name: 'Critical Tagged tests',
+      //   // Use one project (or multiple projects) defined in your Playwright config
+      //   pwProjects: ['chromium'],
+      //   // Use one tag (or multiple tags) defined in your spec files
+      //   pwTags: '@critical',
+      //   frequency: Frequency.EVERY_5M,
+      //   locations: ['us-east-1', 'eu-central-1', 'ap-southeast-2'],
+      // },
+    ],
+  },
+  cli: {
+    runLocation: 'us-east-1',
+    retries: 0,
+  },
+})
+`;
+
+export const TSCONFIG_JSON = `{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "rootDir": "./",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": [
+    "*.ts"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist"
+  ],
+  "ts-node": {
+    "esm": false,
+    "compilerOptions": {
+      "module": "commonjs"
+    }
+  }
+}
+`;
+
+export const PACKAGE_JSON = `{
+  "name": "hackathon_pwmcp",
+  "version": "1.0.0",
+  "description": "MCP Manager API for Playwright server instances with ngrok",
+  "main": "mcp-manager-api.js",
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/mcp-manager-api.js",
+    "dev": "ts-node mcp-manager-api.ts",
+    "test": "npx playwright test"
+  },
+  "keywords": ["mcp", "playwright", "ngrok", "api", "typescript"],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@playwright/test": "^1.54.1",
+    "@types/express": "^5.0.3",
+    "@types/node": "^24.0.15",
+    "@types/uuid": "^10.0.0",
+    "checkly": "^0.0.0-pr.1111.e932b3e",
+    "jiti": "^2.4.2",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.8.3"
+  },
+  "dependencies": {
+    "express": "^4.18.2",
+    "ngrok": "^5.0.0-beta.2",
+    "uuid": "^10.0.0"
+  }
+}
+`;
+

From ffcdb5e2f536a396ccf4c6ca670bbc97dce8bbd5 Mon Sep 17 00:00:00 2001
From: Daniel Paulus <daniel.paulus@icloud.com>
Date: Mon, 21 Jul 2025 20:06:22 +0200
Subject: [PATCH 14/16] chore: add verbose logs

---
 packages/cli/src/commands/mcp.ts | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts
index d2b7381a4..16f9347bb 100644
--- a/packages/cli/src/commands/mcp.ts
+++ b/packages/cli/src/commands/mcp.ts
@@ -1,4 +1,5 @@
 import { AuthCommand } from './authCommand'
+import { Flags } from '@oclif/core'
 import * as fs from 'node:fs/promises'
 import * as path from 'node:path'
 import * as os from 'node:os'
@@ -31,8 +32,19 @@ export default class McpCommand extends AuthCommand {
   static hidden = false
   static description = 'Run MCP server tests with Playwright on Checkly.'
   static state = 'beta'
+  
+  static flags = {
+    verbose: Flags.boolean({
+      char: 'v',
+      description: 'Show all logs, errors, and stdout from processes',
+      default: false,
+    }),
+  }
 
   async run(): Promise<void> {
+    const { flags } = await this.parse(McpCommand)
+    const { verbose } = flags
+    
     let tempDir: string | null = null
     let pinggyUrl: string | null = null
       // Output initial "in progress" message
@@ -51,9 +63,12 @@ export default class McpCommand extends AuthCommand {
       
       // Run npm install in the temp directory
       const { execa } = await import('execa')
+      if (verbose) {
+        this.log('Installing dependencies...')
+      }
       await execa('npm', ['install'], {
         cwd: tempDir,
-        stdio: 'pipe' // Changed to pipe to suppress output
+        stdio: verbose ? 'inherit' : 'pipe' // Show output in verbose mode
       })
       
     
@@ -79,6 +94,11 @@ export default class McpCommand extends AuthCommand {
         pwTestProcess.stdout?.on('data', (data) => {
           const output = data.toString()
           
+          // In verbose mode, show all output
+          if (verbose) {
+            process.stdout.write(output)
+          }
+          
           // Parse for Pinggy URL
           const urlMatch = output.match(/🌐 PINGGY PUBLIC URL:\s*(https:\/\/[a-z0-9-]+\.a\.free\.pinggy\.link)/)
           if (urlMatch && !pinggyUrl) {
@@ -92,6 +112,11 @@ export default class McpCommand extends AuthCommand {
         pwTestProcess.stderr?.on('data', (data) => {
           const output = data.toString()
           
+          // In verbose mode, show all error output
+          if (verbose) {
+            process.stderr.write(output)
+          }
+          
           // Also check stderr for URLs (pinggy might output there)
           const urlMatch = output.match(/🌐 PINGGY PUBLIC URL:\s*(https:\/\/[a-z0-9-]+\.a\.free\.pinggy\.link)/)
           if (urlMatch && !pinggyUrl) {
@@ -122,7 +147,10 @@ export default class McpCommand extends AuthCommand {
       }
       
     } catch (error) {
-      // Suppress error output to keep it clean
+      // Show errors in verbose mode
+      if (verbose && error instanceof Error) {
+        this.error(error.message)
+      }
       process.exit(1)
     } finally {
       // Clean up temp directory

From a772cb3cd1cedb650db61e51ff6e92234d132cea Mon Sep 17 00:00:00 2001
From: Daniel Paulus <daniel.paulus@icloud.com>
Date: Mon, 21 Jul 2025 20:08:14 +0200
Subject: [PATCH 15/16] chore: ups

---
 packages/cli/src/commands/mcp.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts
index 16f9347bb..26b2b9452 100644
--- a/packages/cli/src/commands/mcp.ts
+++ b/packages/cli/src/commands/mcp.ts
@@ -82,7 +82,7 @@ export default class McpCommand extends AuthCommand {
       try {
         // Run pw-test command with stream-logs enabled using spawn to capture output
         const checklyPath = path.join(originalCwd, 'bin', 'run')
-        const args = ['pw-test', '--stream-logs', ...this.argv]
+        const args = ['pw-test', '--stream-logs']
         
         const pwTestProcess = spawn(checklyPath, args, {
           cwd: tempDir,

From f6acb9c6a34ce22f80554b87f1feaa6379360603 Mon Sep 17 00:00:00 2001
From: Daniel Paulus <daniel.paulus@icloud.com>
Date: Mon, 21 Jul 2025 20:21:16 +0200
Subject: [PATCH 16/16] chore: fix lol bug from claude

---
 packages/cli/src/commands/mcp.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts
index 26b2b9452..6a9fcec4b 100644
--- a/packages/cli/src/commands/mcp.ts
+++ b/packages/cli/src/commands/mcp.ts
@@ -81,10 +81,12 @@ export default class McpCommand extends AuthCommand {
       
       try {
         // Run pw-test command with stream-logs enabled using spawn to capture output
-        const checklyPath = path.join(originalCwd, 'bin', 'run')
-        const args = ['pw-test', '--stream-logs']
+        // Use the same node and checkly command that was used to run this command
+        const nodeExecutable = process.argv[0]
+        const checklyExecutable = process.argv[1]
+        const args = [checklyExecutable, 'pw-test', '--stream-logs']
         
-        const pwTestProcess = spawn(checklyPath, args, {
+        const pwTestProcess = spawn(nodeExecutable, args, {
           cwd: tempDir,
           env: { ...process.env },
           stdio: ['inherit', 'pipe', 'pipe']