Audio helpers

catgrad-llm/Cargo.toml

@@ -32,6 +32,8 @@ serde_with = { version = "3.17", default-features = false, features = ["macros"]
 serde_path_to_error = "0.1"
 ureq = "2.12.1"
 url = "2.5.7"
+hound = "3.5.1"
+rustfft = "6.4.1"
 
 
 [dev-dependencies]

catgrad-llm/scripts/llm.py

@@ -10,7 +10,7 @@
 
 from transformers import (
     AutoModelForCausalLM,
-    AutoModelForImageTextToText,
+    AutoModelForMultimodalLM,
     AutoProcessor,
     AutoTokenizer,
     logging,
@@ -247,6 +247,7 @@ def run_tool_chat(tokenizer, model, prompt, args):
     parser.add_argument("-p", "--prompt", type=str, default="Category theory is")
     parser.add_argument("-s", "--seq-len", type=int, default=10)
     parser.add_argument("-i", "--image", type=str, default=None)
+    parser.add_argument("-a", "--audio", type=str, default=None)
     parser.add_argument("-r", "--raw", action="store_true")
     parser.add_argument("-t", "--thinking", action="store_true")
     parser.add_argument(
@@ -266,19 +267,19 @@ def run_tool_chat(tokenizer, model, prompt, args):
     if args.tool_use and args.raw:
         parser.error("--tool-use does not support --raw")
 
-    if args.image is None:
+    if args.image is None and args.audio is None:
         tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision)
         try:
             model = AutoModelForCausalLM.from_pretrained(
                 args.model, revision=args.revision, dtype=args.dtype
             )
         except:
-            model = AutoModelForImageTextToText.from_pretrained(
+            model = AutoModelForMultimodalLM.from_pretrained(
                 args.model, revision=args.revision, dtype=args.dtype
             )
     else:
         processor = AutoProcessor.from_pretrained(args.model, revision=args.revision)
-        model = AutoModelForImageTextToText.from_pretrained(
+        model = AutoModelForMultimodalLM.from_pretrained(
             args.model, revision=args.revision, dtype=args.dtype
         )
 
@@ -290,6 +291,7 @@ def run_tool_chat(tokenizer, model, prompt, args):
 
     if (
         args.image is None
+        and args.audio is None
         and not args.raw
         and not args.tool_use
         and tokenizer.chat_template is not None
@@ -309,7 +311,7 @@ def run_tool_chat(tokenizer, model, prompt, args):
     model.generation_config.top_p = None
     model.generation_config.top_k = None
 
-    if args.image is None:
+    if args.image is None and args.audio is None:
         if args.tool_use:
             output = run_tool_chat(tokenizer, model, prompt, args)
         else:
@@ -322,13 +324,16 @@ def run_tool_chat(tokenizer, model, prompt, args):
             )
             output = tokenizer.decode(logits[0], skip_special_tokens=True)
     else:
+        content = [{"type": "text", "text": prompt}]
+        if args.image:
+            content += [{"type": "image", "path": args.image}]
+        if args.audio:
+            content += [{"type": "audio", "path": args.audio}]
+
         messages = [
             {
                 "role": "user",
-                "content": [
-                    {"type": "text", "text": prompt},
-                    {"type": "image", "path": args.image},
-                ],
+                "content": content,
             }
         ]
         try:

catgrad-llm/src/helpers/tool_calls.rs

@@ -50,6 +50,10 @@ pub fn parse_qwen3_5_tool_calls(output: &str) -> Result<Option<ToolUseStep>> {
     )
 }
 
+pub fn parse_granite_tool_calls(output: &str) -> Result<Option<ToolUseStep>> {
+    parse_qwen3_tool_calls(output)
+}
+
 pub fn parse_lfm2_tool_calls(output: &str) -> Result<Option<ToolUseStep>> {
     parse_python_tool_calls(output, "<|tool_call_start|>", "<|tool_call_end|>")
 }

catgrad-llm/src/models/granite.rs

@@ -77,6 +77,10 @@ impl LLMModel for GraniteModel {
     fn dtype(&self) -> Dtype {
         self.dtype
     }
+
+    fn parse_tool_calls(&self, output: &str) -> crate::Result<Option<ToolUseStep>> {
+        parse_granite_tool_calls(output)
+    }
 }
 
 impl GraniteModel {