diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..ffae2195 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,118 @@ +name: LMS CLI Tests + +# Group workflow runs by branch and cancel in-progress runs on new commits to avoid PR wastefulness +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize, reopened, labeled] + +jobs: + build-and-test: + name: Build and Test LMS CLI + runs-on: ubuntu-latest + container: + image: lmstudio/llmster-preview:cpu + timeout-minutes: 30 + steps: + - name: Install git on container + run: | + apt update + apt install -y git + - name: Determine lmstudio.js branch + id: branch + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + echo "$PR_BODY" > pr_body.txt + + # Always look below separators first + LMS_JS_BRANCH=$(awk '/^-+$/{flag=1; next} flag && /lmstudio-js-branch:/{gsub(/.*lmstudio-js-branch:[ \t]*/, ""); gsub(/[ \t].*/, ""); print; exit}' pr_body.txt || true) + + # If not found in separator sections, try anywhere in PR body + if [ -z "$LMS_JS_BRANCH" ]; then + LMS_JS_BRANCH=$(grep -oP 'lmstudio-js-branch:\s*\K\S+' pr_body.txt || true) + fi + + # Fallback to main if not found + if [ -z "$LMS_JS_BRANCH" ]; then + LMS_JS_BRANCH="main" + fi + + echo "branch=$LMS_JS_BRANCH" >> $GITHUB_OUTPUT + + - name: Checkout lmstudio.js repo + uses: actions/checkout@v4 + with: + repository: lmstudio-ai/lmstudio-js + ref: ${{ steps.branch.outputs.branch }} + path: lmstudio.js + submodules: true + - name: Fetch and checkout lms-cli branch + run: | + cd lmstudio.js/packages/lms-cli + git fetch origin ${{ github.head_ref }}:${{ github.head_ref }} + git checkout ${{ github.head_ref }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: lmstudio.js/package-lock.json + + - name: Install dependencies + run: | + cd lmstudio.js + npm install + + - name: Build lmstudio.js + run: | + cd lmstudio.js + npm run build + + - name: Run daemon and tests + id: run + run: | + cd /app + echo "Starting up llmd..." + ./daemon-run.sh & + DAEMON_PID=$! + LM_HOME="${HOME}/.lmstudio" + MAX_WAIT=120 + START_TIME=$(date +%s) + # Check for the ready files + while true; do + # Check for any .ready file + if ls "${LM_HOME}"/.ready* 1> /dev/null 2>&1; then + break + fi + CURRENT_TIME=$(date +%s) + ELAPSED=$((CURRENT_TIME - START_TIME)) + if [ $ELAPSED -ge $MAX_WAIT ]; then + echo "Timed out waiting for llmster to start after ${MAX_WAIT} seconds." + break + fi + sleep 1 + done + + # Setup model + # mkdir -p /.lmstudio/hub/models/google + # cd /.lmstudio/hub/models/google + # lms clone google/gemma-3-1b + # mkdir -p /.lmstudio/models/lmstudio-community/gemma-3-1b-it-GGUF + # cd /.lmstudio/models/lmstudio-community/gemma-3-1b-it-GGUF + # if ! command -v wget &> /dev/null; then + # apt-get update + # apt-get install -y wget + # fi + # wget --quiet -P /tmp https://models.lmstudio.ai/models/gemma-3-1b-it-Q4_K_M.gguf && mv /tmp/gemma-3-1b-it-Q4_K_M.gguf gemma-3-1b-it-Q4_K_M.gguf + + lms get google/gemma-3-1b -y + + # Run tests + cd $GITHUB_WORKSPACE/lmstudio.js + npm run test-cli -- --verbose diff --git a/src/subcommands/chat.heavy.test.ts b/src/subcommands/chat.heavy.test.ts new file mode 100644 index 00000000..2f51b19c --- /dev/null +++ b/src/subcommands/chat.heavy.test.ts @@ -0,0 +1,106 @@ +import path from "path"; +import { TEST_CLI_PATH, testRunCommandSync } from "../test-utils.js"; + +// We skip chat tests to because we don't have max_tokens here. +describe.skip("chat heavy", () => { + const cliPath = path.join(__dirname, TEST_CLI_PATH); + const modelIdentifier = "test-model"; + const modelToUse = "gemma-3-1b"; + + beforeAll(async () => { + // Ensure the test model is loaded + const { status } = testRunCommandSync("node", [ + cliPath, + "load", + modelToUse, + "--identifier", + modelIdentifier, + "--yes", + ]); + if (status !== 0) { + throw new Error(`Failed to load test model: ${modelIdentifier}`); + } + }, 30000); + + afterAll(async () => { + // Clean up by unloading the model + const { status } = testRunCommandSync("node", [cliPath, "unload", modelIdentifier]); + if (status !== 0) { + console.warn(`Failed to unload test model: ${modelIdentifier}`); + } + }, 10000); + + it("should respond to simple prompt with specific model", () => { + const { status, stdout, stderr } = testRunCommandSync("node", [ + cliPath, + "chat", + modelIdentifier, + "--prompt", + '"What is 2+2? Answer briefly:"', + ]); + + if (status !== 0) console.error("Chat stderr:", stderr); + expect(status).toBe(0); + expect(stdout.toLowerCase()).toContain("4"); + }, 15000); + + it("should use custom system prompt", () => { + const { status, stdout, stderr } = testRunCommandSync("node", [ + cliPath, + "chat", + modelIdentifier, + "--prompt", + '"What is your role?"', + "--system-prompt", + '"You are a helpful assistant."', + ]); + + if (status !== 0) console.error("Chat stderr:", stderr); + expect(status).toBe(0); + expect(stdout.toLowerCase()).toMatch(/(assistant|help)/); + }, 15000); + + it("should display stats when --stats flag is used", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "chat", + modelIdentifier, + "--prompt", + '"Hi"', + "--stats", + ]); + + if (status !== 0) console.error("Chat stderr:", stderr); + expect(status).toBe(0); + expect(stderr).toContain("Prediction Stats:"); + expect(stderr).toContain("Stop Reason:"); + expect(stderr).toContain("Tokens/Second:"); + }, 15000); + + it("should work with default model when no model specified", () => { + const { status, stdout, stderr } = testRunCommandSync("node", [ + cliPath, + "chat", + "--prompt", + "\"Say hello. Respond with just 'hello'\"", + ]); + + if (status !== 0) console.error("Chat stderr:", stderr); + expect(status).toBe(0); + expect(stdout.toLowerCase()).toContain("hello"); + }, 15000); + + it("should fail gracefully with non-existent model", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "chat", + "non-existent-model", + "--prompt", + '"test"', + ]); + + expect(status).not.toBe(0); + expect(stderr).toContain("not found"); + expect(stderr).toContain("lms ls"); + }); +}); diff --git a/src/subcommands/importCmd.heavy.test.ts b/src/subcommands/importCmd.heavy.test.ts new file mode 100644 index 00000000..631746b1 --- /dev/null +++ b/src/subcommands/importCmd.heavy.test.ts @@ -0,0 +1,446 @@ +import path from "path"; +import fs from "fs"; +import os from "os"; +import { TEST_CLI_PATH, testRunCommandSync } from "../test-utils.js"; + +// Helper function to create a minimal valid GGUF file +function createValidGGUFFile(filePath: string) { + const buffer = Buffer.alloc(64); + // GGUF magic number (4 bytes): "GGUF" + buffer.write("GGUF", 0, "ascii"); + // Version (4 bytes): version 3 + buffer.writeUInt32LE(3, 4); + // Tensor count (8 bytes): 0 tensors + buffer.writeBigUInt64LE(0n, 8); + // Metadata kv count (8 bytes): 0 metadata + buffer.writeBigUInt64LE(0n, 16); + fs.writeFileSync(filePath, buffer); +} + +describe("import command", () => { + const cliPath = path.join(__dirname, TEST_CLI_PATH); + const testModelPath = path.join(__dirname, "../../../test-fixtures/test-model.gguf"); + const lmstudioModelsPath = path.join(os.homedir(), ".lmstudio", "models"); + + beforeAll(() => { + // Create a test model file + const testDir = path.dirname(testModelPath); + if (!fs.existsSync(testDir)) { + fs.mkdirSync(testDir, { recursive: true }); + } + if (!fs.existsSync(testModelPath)) { + createValidGGUFFile(testModelPath); + } + }); + + afterAll(() => { + // Clean up test model file + if (fs.existsSync(testModelPath)) { + fs.unlinkSync(testModelPath); + } + }); + + describe("dry run tests", () => { + let testFilePath: string; + let testId: string; + + beforeEach(() => { + // Create unique test file for each test + testId = `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + testFilePath = path.join(lmstudioModelsPath, "test-files", `${testId}.gguf`); + // Ensure test directory exists + const testDir = path.dirname(testFilePath); + if (!fs.existsSync(testDir)) { + fs.mkdirSync(testDir, { recursive: true }); + } + createValidGGUFFile(testFilePath); + }); + + afterEach(() => { + // Clean up test file + if (fs.existsSync(testFilePath)) { + fs.unlinkSync(testFilePath); + } + + // Clean up any potential target files in LM Studio directory + const targetDir = path.join(lmstudioModelsPath, "test", "model"); + const targetPath = path.join(targetDir, path.basename(testFilePath)); + if (fs.existsSync(targetPath)) { + fs.unlinkSync(targetPath); + } + }); + + it("should perform dry run without actually moving file", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--dry-run", + "--yes", + "--user-repo", + "test/model", + ]); + + expect(status).toBe(0); + expect(stderr).toContain("Would move"); + expect(stderr).toContain("--dry-run"); + + // Assert file was NOT moved + expect(fs.existsSync(testFilePath)).toBe(true); + expect(fs.existsSync(testFilePath)).toBe(true); + }); + + it("should perform dry run without actually copying file", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--dry-run", + "--copy", + "--yes", + "--user-repo", + "test/model", + ]); + + expect(status).toBe(0); + expect(stderr).toContain("Would copy"); + + // Assert original file still exists and no target file was created + expect(fs.existsSync(testFilePath)).toBe(true); + const targetPath = path.join( + lmstudioModelsPath, + "test", + "model", + path.basename(testFilePath), + ); + expect(fs.existsSync(targetPath)).toBe(false); + }); + + it("should perform dry run without actually creating hard link", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--dry-run", + "--hard-link", + "--yes", + "--user-repo", + "test/model", + ]); + + expect(status).toBe(0); + expect(stderr).toContain("Would create a hard link"); + + // Assert original file still exists and no target file was created + expect(fs.existsSync(testFilePath)).toBe(true); + const targetPath = path.join( + lmstudioModelsPath, + "test", + "model", + path.basename(testFilePath), + ); + expect(fs.existsSync(targetPath)).toBe(false); + }); + + it("should perform dry run without actually creating symbolic link", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--dry-run", + "--symbolic-link", + "--yes", + "--user-repo", + "test/model", + ]); + + expect(status).toBe(0); + expect(stderr).toContain("Would create a symbolic link"); + + // Assert original file still exists and no target file was created + expect(fs.existsSync(testFilePath)).toBe(true); + const targetPath = path.join( + lmstudioModelsPath, + "test", + "model", + path.basename(testFilePath), + ); + expect(fs.existsSync(targetPath)).toBe(false); + }); + }); + + // Skip for now as tests do not run inside the container. + describe("actual import tests", () => { + let testFilePath: string; + let testId: string; + let targetPath: string; + + beforeEach(() => { + // Create unique test file for each test + testId = `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + testFilePath = path.join(lmstudioModelsPath, "test-files", `${testId}.gguf`); + // Ensure test directory exists + const testDir = path.dirname(testFilePath); + if (!fs.existsSync(testDir)) { + fs.mkdirSync(testDir, { recursive: true }); + } + createValidGGUFFile(testFilePath); + + targetPath = path.join(lmstudioModelsPath, "test", "model", path.basename(testFilePath)); + }); + + afterEach(() => { + // Clean up test files + [testFilePath, targetPath].forEach(filePath => { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + }); + + // Clean up test directories if empty + const targetDir = path.join(lmstudioModelsPath, "test", "model"); + const testUserDir = path.join(lmstudioModelsPath, "test"); + + try { + if (fs.existsSync(targetDir) && fs.readdirSync(targetDir).length === 0) { + fs.rmdirSync(targetDir); + } + if (fs.existsSync(testUserDir) && fs.readdirSync(testUserDir).length === 0) { + fs.rmdirSync(testUserDir); + } + } catch (e) { + // Ignore cleanup errors + } + }); + + it("should actually move file when not in dry run mode", () => { + const { status, stderr, stdout } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--yes", + "--user-repo", + "test/model", + ]); + + if (status !== 0) { + console.log("STDOUT:", stdout); + console.log("STDERR:", stderr); + } + + expect(status).toBe(0); + expect(stderr).toContain("File moved to"); + + // Assert file was moved + expect(fs.existsSync(testFilePath)).toBe(false); + expect(fs.existsSync(targetPath)).toBe(true); + expect(fs.existsSync(targetPath)).toBe(true); + }); + + it("should actually copy file when using --copy flag", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--copy", + "--yes", + "--user-repo", + "test/model", + ]); + + expect(status).toBe(0); + expect(stderr).toContain("File copied to"); + + // Assert file was copied (both original and target exist) + expect(fs.existsSync(testFilePath)).toBe(true); + expect(fs.existsSync(targetPath)).toBe(true); + expect(fs.existsSync(testFilePath)).toBe(true); + expect(fs.existsSync(targetPath)).toBe(true); + }); + + it("should actually create hard link when using --hard-link flag", () => { + const { status, stderr, stdout } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--hard-link", + "--yes", + "--user-repo", + "test/model", + ]); + + if (status !== 0) { + console.log("STDOUT:", stdout); + console.log("STDERR:", stderr); + } + + expect(status).toBe(0); + expect(stderr).toContain("Hard link created at"); + + // Assert hard link was created (both files exist and have same content) + expect(fs.existsSync(testFilePath)).toBe(true); + expect(fs.existsSync(targetPath)).toBe(true); + expect(fs.existsSync(testFilePath)).toBe(true); + expect(fs.existsSync(targetPath)).toBe(true); + + // Verify it's actually a hard link by checking inode numbers + const originalStat = fs.statSync(testFilePath); + const targetStat = fs.statSync(targetPath); + expect(originalStat.ino).toBe(targetStat.ino); + }); + + it("should actually create symbolic link when using --symbolic-link flag", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--symbolic-link", + "--yes", + "--user-repo", + "test/model", + ]); + + expect(status).toBe(0); + expect(stderr).toContain("Symbolic link created at"); + + // Assert symbolic link was created + expect(fs.existsSync(testFilePath)).toBe(true); + expect(fs.existsSync(targetPath)).toBe(true); + expect(fs.existsSync(testFilePath)).toBe(true); + expect(fs.existsSync(targetPath)).toBe(true); + + // Verify it's actually a symbolic link + const targetStat = fs.lstatSync(targetPath); + expect(targetStat.isSymbolicLink()).toBe(true); + expect(fs.readlinkSync(targetPath)).toBe(testFilePath); + }); + + it("should create directory structure when importing", () => { + const deepTargetPath = path.join( + lmstudioModelsPath, + "deep-user", + "nested-repo", + path.basename(testFilePath), + ); + + const { status, stderr, stdout } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--yes", + "--user-repo", + "deep-user/nested-repo", + ]); + + if (status !== 0) { + console.log("STDOUT:", stdout); + console.log("STDERR:", stderr); + } + + expect(status).toBe(0); + + // Assert directory structure was created + expect(fs.existsSync(path.join(lmstudioModelsPath, "deep-user", "nested-repo"))).toBe(true); + expect(fs.existsSync(deepTargetPath)).toBe(true); + + // Clean up deep structure + if (fs.existsSync(deepTargetPath)) { + fs.unlinkSync(deepTargetPath); + } + ["nested-repo", "deep-user"].forEach(dir => { + const dirPath = path.join(lmstudioModelsPath, dir); + try { + if (fs.existsSync(dirPath) && fs.readdirSync(dirPath).length === 0) { + fs.rmdirSync(dirPath); + } + } catch (e) { + // Ignore cleanup errors + } + }); + }); + }); + + describe("error handling", () => { + it("should fail when multiple operation flags are specified", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + testModelPath, + "--dry-run", + "--copy", + "--hard-link", + "--yes", + ]); + + expect(status).not.toBe(0); + expect(stderr).toContain("Cannot specify"); + }); + + it("should handle non-existent file gracefully", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + "/non/existent/file.gguf", + "--dry-run", + "--yes", + "--user-repo", + "test/model", + ]); + + expect(status).not.toBe(0); + expect(stderr).toContain("File does not exist"); + }); + + it("should fail when target file already exists", () => { + const testId = `existing-test-${Date.now()}`; + const testFilePath = path.join(lmstudioModelsPath, "test-files", `${testId}.gguf`); + // Ensure test directory exists + const testDir = path.dirname(testFilePath); + if (!fs.existsSync(testDir)) { + fs.mkdirSync(testDir, { recursive: true }); + } + createValidGGUFFile(testFilePath); + + const targetDir = path.join(lmstudioModelsPath, "test", "existing"); + const targetPath = path.join(targetDir, path.basename(testFilePath)); + + // Create target directory and file + fs.mkdirSync(targetDir, { recursive: true }); + createValidGGUFFile(targetPath); + + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "import", + testFilePath, + "--yes", + "--user-repo", + "test/existing", + ]); + + expect(status).not.toBe(0); + expect(stderr).toContain("Target file already exists"); + + // Assert original file still exists + expect(fs.existsSync(testFilePath)).toBe(true); + expect(fs.existsSync(targetPath)).toBe(true); + + // Clean up + [testFilePath, targetPath].forEach(filePath => { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + }); + try { + if (fs.existsSync(targetDir) && fs.readdirSync(targetDir).length === 0) { + fs.rmdirSync(targetDir); + } + const testUserDir = path.join(lmstudioModelsPath, "test"); + if (fs.existsSync(testUserDir) && fs.readdirSync(testUserDir).length === 0) { + fs.rmdirSync(testUserDir); + } + } catch (e) { + // Ignore cleanup errors + } + }); + }); +}); diff --git a/src/subcommands/importCmd.ts b/src/subcommands/importCmd.ts index 7d44ad98..b9f891cc 100644 --- a/src/subcommands/importCmd.ts +++ b/src/subcommands/importCmd.ts @@ -380,6 +380,7 @@ async function warnAboutMove( } if (yes) { logger.warn("Warning about move suppressed by the --yes flag."); + return; } logger.debug("Asking user to confirm moving the file"); process.stderr.write(text` diff --git a/src/subcommands/list.heavy.test.ts b/src/subcommands/list.heavy.test.ts new file mode 100644 index 00000000..7d5790b7 --- /dev/null +++ b/src/subcommands/list.heavy.test.ts @@ -0,0 +1,107 @@ +import path from "path"; +import { TEST_CLI_PATH, testRunCommandSync, TEST_MODEL_EXPECTED } from "../test-utils.js"; + +describe("list", () => { + const cliPath = path.join(__dirname, TEST_CLI_PATH); + + describe("ls command", () => { + it("should show downloaded models", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "ls"]); + expect(status).toBe(0); + expect(stdout).toContain("models"); + }); + + it("should filter LLM models only", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "ls", "--llm"]); + expect(status).toBe(0); + expect(stdout).toContain("LLM"); + expect(stdout).not.toContain("EMBEDDING"); + expect(stdout).toContain("PARAMS"); + expect(stdout).toContain(TEST_MODEL_EXPECTED); + }); + + it("should filter embedding models only", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "ls", "--embedding"]); + expect(status).toBe(0); + expect(stdout).toContain("EMBEDDING"); + expect(stdout).not.toContain("LLM"); + expect(stdout).toContain("PARAMS"); + }); + + it("should output JSON format", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "ls", "--json"]); + expect(status).toBe(0); + if (stdout.trim()) { + expect(() => JSON.parse(stdout)).not.toThrow(); + expect(stdout).toContain(TEST_MODEL_EXPECTED); + } + }); + + it("should handle combined flags with json", () => { + const { status, stdout } = testRunCommandSync("node", [ + cliPath, + "ls", + "--embedding", + "--json", + ]); + expect(status).toBe(0); + if (stdout.trim()) { + expect(() => JSON.parse(stdout)).not.toThrow(); + } + }); + }); + + describe("ps command", () => { + beforeAll(() => { + // Ensure the server is running before tests + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "load", + TEST_MODEL_EXPECTED, + "--identifier", + TEST_MODEL_EXPECTED, + "--yes", + ]); + if (status !== 0) { + console.error("Server stderr:", stderr); + } + expect(status).toBe(0); + }); + + afterAll(() => { + // Cleanup: Unload the model after tests + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "unload", + TEST_MODEL_EXPECTED, + ]); + if (status !== 0) { + console.error("Unload stderr:", stderr); + } + expect(status).toBe(0); + }); + + it("should show loaded models", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "ps"]); + expect(status).toBe(0); + expect(stdout).toContain(TEST_MODEL_EXPECTED); + }); + + it("should output JSON format", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "ps", "--json"]); + expect(status).toBe(0); + if (stdout.trim()) { + expect(() => JSON.parse(stdout)).not.toThrow(); + expect( + JSON.parse(stdout).some((model: any) => model.identifier === TEST_MODEL_EXPECTED), + ).toBe(true); + } + }); + + it("should handle no loaded models gracefully", () => { + const { status } = testRunCommandSync("node", [cliPath, "ps"]); + // Command might show "No models are currently loaded" but should not fail + expect(status).toBe(0); + }); + }); +}); diff --git a/src/subcommands/list.ts b/src/subcommands/list.ts index d99914e9..23e5be18 100644 --- a/src/subcommands/list.ts +++ b/src/subcommands/list.ts @@ -185,7 +185,7 @@ export const ps = addCreateClientOptions( } if (loadedModels.length === 0) { - logger.error( + logger.info( text` No models are currently loaded diff --git a/src/subcommands/load.heavy.test.ts b/src/subcommands/load.heavy.test.ts new file mode 100644 index 00000000..569130b6 --- /dev/null +++ b/src/subcommands/load.heavy.test.ts @@ -0,0 +1,189 @@ +import path from "path"; +import { TEST_CLI_PATH, testRunCommandSync, TEST_MODEL_EXPECTED } from "../test-utils.js"; + +describe("load", () => { + const cliPath = path.join(__dirname, TEST_CLI_PATH); + + // Helper function to check if model is loaded using ps command + const verifyModelLoaded = ( + expectedIdentifier: string, + expectedTtlMs: number | null = null, + expectedContextLength: number | null = null, + ) => { + const { status, stdout, stderr } = testRunCommandSync("node", [cliPath, "ps", "--json"]); + if (status !== 0) console.error("PS stderr:", stderr); + expect(status).toBe(0); + expect(stdout).toContain(expectedIdentifier); + + const psData = JSON.parse(stdout); + const model = psData.find( + (m: any) => m.path !== undefined && m.path.toLowerCase().includes(TEST_MODEL_EXPECTED), + ); + expect(model).toBeTruthy(); + + if (expectedTtlMs !== null) { + expect(model.ttlMs).toBe(expectedTtlMs); + } + + if (expectedContextLength !== null) { + expect(model.contextLength).toBe(expectedContextLength); + } + + return model.identifier; + }; + + const unloadAllModels = () => { + const { status } = testRunCommandSync("node", [cliPath, "unload", "--all"]); + if (status !== 0) { + console.error("Failed to unload all models during cleanup."); + } + }; + + beforeAll(() => { + // Ensure cleanup of any loaded models before tests + unloadAllModels(); + }); + + afterAll(() => { + // Ensure cleanup of any loaded models after tests + unloadAllModels(); + }); + + describe("load command", () => { + it("should load model without identifier and verify with ps", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "load", + TEST_MODEL_EXPECTED, + "--yes", + ]); + if (status !== 0) console.error("Load stderr:", stderr); + expect(status).toBe(0); + + // Verify model is loaded and get identifier + const modelIdentifier = verifyModelLoaded(TEST_MODEL_EXPECTED); + + // Unload the model using the extracted identifier + const { status: unloadStatus, stderr: unloadStderr } = testRunCommandSync("node", [ + cliPath, + "unload", + modelIdentifier, + ]); + if (unloadStatus !== 0) console.error("Unload stderr:", unloadStderr); + expect(unloadStatus).toBe(0); + }); + + it("should load model with basic flags and verify with ps", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "load", + TEST_MODEL_EXPECTED, + "--identifier", + "basic-model", + "--yes", + ]); + if (status !== 0) console.error("Load stderr:", stderr); + expect(status).toBe(0); + + // Verify model is loaded + verifyModelLoaded("basic-model"); + + // Unload the model + const { status: unloadStatus, stderr: unloadStderr } = testRunCommandSync("node", [ + cliPath, + "unload", + "basic-model", + ]); + if (unloadStatus !== 0) console.error("Unload stderr:", unloadStderr); + expect(unloadStatus).toBe(0); + }); + + it("should handle advanced flags (GPU, TTL, context-length)", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "load", + TEST_MODEL_EXPECTED, + "--identifier", + "advanced-model", + "--ttl", + "1800", + "--gpu", + "0.8", + "--context-length", + "4096", + "--yes", + ]); + if (status !== 0) console.error("Load stderr:", stderr); + expect(status).toBe(0); + + // Verify model is loaded with correct TTL and context length + verifyModelLoaded("advanced-model", 1800000, 4096); + + // Cleanup + testRunCommandSync("node", [cliPath, "unload", "advanced-model"]); + }); + + it("should handle GPU options (off, max, numeric)", () => { + // Test GPU off + const { status: status1, stderr: stderr1 } = testRunCommandSync("node", [ + cliPath, + "load", + TEST_MODEL_EXPECTED, + "--identifier", + "gpu-off-model", + "--gpu", + "off", + "--yes", + ]); + if (status1 !== 0) console.error("Load stderr:", stderr1); + expect(status1).toBe(0); + testRunCommandSync("node", [cliPath, "unload", "gpu-off-model"]); + + // Test GPU max + const { status: status2, stderr: stderr2 } = testRunCommandSync("node", [ + cliPath, + "load", + TEST_MODEL_EXPECTED, + "--identifier", + "gpu-max-model", + "--gpu", + "max", + "--yes", + ]); + if (status2 !== 0) console.error("Load stderr:", stderr2); + expect(status2).toBe(0); + testRunCommandSync("node", [cliPath, "unload", "gpu-max-model"]); + }); + + it("should handle error cases gracefully", () => { + // Non-existent model with exact flag + const { status: status1, stderr: stderr1 } = testRunCommandSync("node", [ + cliPath, + "load", + "non-existent-model", + "--exact", + ]); + expect(status1).not.toBe(0); + expect(stderr1).toBeTruthy(); + + // Non-existent model with yes flag + const { status: status2, stderr: stderr2 } = testRunCommandSync("node", [ + cliPath, + "load", + "non-existent-model", + "--yes", + ]); + expect(status2).not.toBe(0); + expect(stderr2).toBeTruthy(); + + // Exact flag without path + const { status: status3, stderr: stderr3 } = testRunCommandSync("node", [ + cliPath, + "load", + "--exact", + ]); + expect(status3).not.toBe(0); + expect(stderr3).toBeTruthy(); + }); + }); +}); diff --git a/src/subcommands/status.heavy.test.ts b/src/subcommands/status.heavy.test.ts new file mode 100644 index 00000000..ade22615 --- /dev/null +++ b/src/subcommands/status.heavy.test.ts @@ -0,0 +1,46 @@ +import path from "path"; +import { TEST_CLI_PATH, testRunCommandSync } from "../test-utils.js"; + +// Skipping tests because non-privileged mode. +describe.skip("status", () => { + const cliPath = path.join(__dirname, TEST_CLI_PATH); + + beforeAll(() => { + // Start the server regardless of its current state + const { status } = testRunCommandSync("node", [cliPath, "server", "start", "--port", "1234"]); + if (status !== 0) { + throw new Error("Failed to start the server before tests."); + } + }); + + afterAll(() => { + // Make sure server is up even after the tests + const { status } = testRunCommandSync("node", [cliPath, "server", "start", "--port", "1234"]); + if (status !== 0) { + console.error("Failed to start the server after tests."); + } + }); + + describe("status command", () => { + it("should show LM Studio status", () => { + const { status } = testRunCommandSync("node", [cliPath, "status"]); + expect(status).toBe(0); + }); + }); + + it("update status when server state is updated", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "status"]); + expect(status).toBe(0); + expect(stdout).toContain("ON"); + + const { status: statusForSwitch } = testRunCommandSync("node", [cliPath, "server", "stop"]); + expect(statusForSwitch).toBe(0); + + const { status: statusAfterStop, stdout: stdoutAfterStop } = testRunCommandSync("node", [ + cliPath, + "status", + ]); + expect(statusAfterStop).toBe(0); + expect(stdoutAfterStop).toContain("OFF"); + }); +}); diff --git a/src/subcommands/unload.heavy.test.ts b/src/subcommands/unload.heavy.test.ts new file mode 100644 index 00000000..1ef655d7 --- /dev/null +++ b/src/subcommands/unload.heavy.test.ts @@ -0,0 +1,134 @@ +import path from "path"; +import { TEST_CLI_PATH, testRunCommandSync, TEST_MODEL_EXPECTED } from "../test-utils.js"; + +describe("unload", () => { + const cliPath = path.join(__dirname, TEST_CLI_PATH); + + // Helper function to load a model + const loadModel = (identifier: string) => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "load", + TEST_MODEL_EXPECTED, + "--identifier", + identifier, + "--yes", + ]); + if (status !== 0) console.error("Load stderr:", stderr); + expect(status).toBe(0); + }; + + // Helper function to verify model is loaded + const verifyModelLoaded = (identifier: string) => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "ps"]); + expect(status).toBe(0); + expect(stdout).toContain(identifier); + }; + + // Helper function to verify model is not loaded + const verifyModelNotLoaded = (identifier: string) => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "ps"]); + expect(status).toBe(0); + expect(stdout).not.toContain(identifier); + }; + + // Helper function to unload model by identifier + const unloadModel = (identifier: string) => { + const { status, stderr } = testRunCommandSync("node", [cliPath, "unload", identifier]); + if (status !== 0) console.error("Unload stderr:", stderr); + expect(status).toBe(0); + }; + + // Helper function to unload all models + const unloadAllModels = () => { + const { status, stderr } = testRunCommandSync("node", [cliPath, "unload", "--all"]); + if (status !== 0) console.error("Unload --all stderr:", stderr); + expect(status).toBe(0); + }; + + beforeAll(() => { + // Have a clean state where all models are unloaded before tests + unloadAllModels(); + }); + afterAll(() => { + // Cleanup: Ensure all models are unloaded after tests + unloadAllModels(); + }); + describe("unload command", () => { + it("should handle unload with specific identifier", () => { + // Load model and verify + loadModel("test-unload-model"); + loadModel("last-model-to-unload"); + verifyModelLoaded("test-unload-model"); + verifyModelLoaded("last-model-to-unload"); + + // Unload the specific model + unloadModel("test-unload-model"); + + // Verify it's no longer loaded + verifyModelNotLoaded("test-unload-model"); + verifyModelLoaded("last-model-to-unload"); + + // Cleanup + unloadModel("last-model-to-unload"); + verifyModelNotLoaded("last-model-to-unload"); + }); + + it("should handle unload --all flag", () => { + // Load multiple models + loadModel("model-1"); + loadModel("model-2"); + + // Verify both are loaded + verifyModelLoaded("model-1"); + verifyModelLoaded("model-2"); + + // Unload all models + unloadAllModels(); + + // Verify no models are loaded + verifyModelNotLoaded("model-1"); + verifyModelNotLoaded("model-2"); + }); + + it("should handle unload --all with short flag", () => { + // Load a model + loadModel("short-flag-test"); + + // Unload all with short flag + const { status, stderr } = testRunCommandSync("node", [cliPath, "unload", "-a"]); + if (status !== 0) console.error("Unload -a stderr:", stderr); + expect(status).toBe(0); + }); + + it("should fail gracefully with non-existent model identifier", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "unload", + "non-existent-model", + ]); + expect(status).not.toBe(0); + expect(stderr).toBeTruthy(); + expect(stderr).toContain("Cannot find a model"); + }); + + it("should fail when both identifier and --all flag are provided", () => { + const { status, stderr } = testRunCommandSync("node", [ + cliPath, + "unload", + "some-model", + "--all", + ]); + expect(status).not.toBe(0); + expect(stderr).toBeTruthy(); + }); + + it("should handle unload when no models are loaded", () => { + // Make sure no models are loaded + unloadAllModels(); + + // Try to unload all when nothing is loaded + unloadAllModels(); // Should succeed but show "No models to unload" + }); + }); +}); diff --git a/src/subcommands/unload.ts b/src/subcommands/unload.ts index ec94893f..260d9df0 100644 --- a/src/subcommands/unload.ts +++ b/src/subcommands/unload.ts @@ -38,6 +38,7 @@ export const unload = addLogLevelOptions( `, ).message, ); + process.exit(1); } const models = ( await Promise.all([client.llm.listLoaded(), client.embedding.listLoaded()]) @@ -84,7 +85,7 @@ export const unload = addLogLevelOptions( `, ).message, ); - return; + process.exit(1); } logger.debug(`Unloading "${identifier}"...`); await client.llm.unload(identifier); diff --git a/src/subcommands/version.heavy.test.ts b/src/subcommands/version.heavy.test.ts new file mode 100644 index 00000000..e284dad6 --- /dev/null +++ b/src/subcommands/version.heavy.test.ts @@ -0,0 +1,23 @@ +import path from "path"; +import { TEST_CLI_PATH, testRunCommandSync } from "../test-utils.js"; + +describe("version", () => { + const cliPath = path.join(__dirname, TEST_CLI_PATH); + + describe("version command", () => { + it("should display version with ASCII art", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "version"]); + expect(status).toBe(0); + expect(stdout).toContain("lms - LM Studio CLI"); + expect(stdout).toContain("GitHub: https://github.com/lmstudio-ai/lms"); + }); + + it("should output JSON format when --json flag is used", () => { + const { status, stdout } = testRunCommandSync("node", [cliPath, "version", "--json"]); + expect(status).toBe(0); + const parsed = JSON.parse(stdout); + expect(parsed).toHaveProperty("version"); + expect(typeof parsed.version).toBe("string"); + }); + }); +}); diff --git a/src/test-utils.ts b/src/test-utils.ts new file mode 100644 index 00000000..46e33ea2 --- /dev/null +++ b/src/test-utils.ts @@ -0,0 +1,32 @@ +import { spawnSync, type SpawnSyncOptions } from "child_process"; + +export interface TestExecResult { + stdout: string; + stderr: string; + status: number; +} +export const TEST_MODEL_EXPECTED = "gemma-3-1b"; + +/** + * Runs a command synchronously and returns the result. + */ +export function testRunCommandSync( + cmd: string, + args: string[], + options: SpawnSyncOptions = {}, +): TestExecResult { + const result = spawnSync(cmd, args, { + stdio: "pipe", + encoding: "utf-8", + shell: true, + ...options, + }); + + return { + stdout: result.stdout?.toString() ?? "", + stderr: result.stderr?.toString() ?? "", + status: result.status ?? (result.error ? 1 : 0), + }; +} + +export const TEST_CLI_PATH = "../../../../publish/cli/dist/index.js";