diff --git a/python/llm/src/ipex_llm/cli/llm-cli b/python/llm/src/ipex_llm/cli/llm-cli
index fdb182ea134..e3019ef8738 100755
--- a/python/llm/src/ipex_llm/cli/llm-cli
+++ b/python/llm/src/ipex_llm/cli/llm-cli
@@ -62,8 +62,7 @@ while [[ $# -gt 0 ]]; do
   case "$1" in
   -h | --help)
     display_help
-    filteredArguments+=("'$1'")
-    shift
+    exit 0
     ;;
   -x | --model_family | --model-family)
     model_family="$2"
diff --git a/python/llm/test/cli/test_cli.py b/python/llm/test/cli/test_cli.py
new file mode 100644
index 00000000000..78171511e34
--- /dev/null
+++ b/python/llm/test/cli/test_cli.py
@@ -0,0 +1,11 @@
+import subprocess
+import os
+
+def test_llm_cli_help_fix():
+    """
+    Tests that the llm-cli script exits gracefully after the fix.
+    """
+    cli_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../src/ipex_llm/cli/llm-cli'))
+    result = subprocess.run([cli_path, "--help"], capture_output=True, text=True)
+    assert "Invalid model_family" not in result.stdout
+    assert result.returncode == 0