fix using math verify

Author: codelion

optillm/mars/mars.py

@@ -348,22 +348,36 @@ def _synthesize_final_solution(
         if extracted_answer is not None:
             logger.info(f"🗳️  VOTING: Agent {solution.agent_id} extracted answer '{extracted_answer}' via unified extraction (confidence: {solution.confidence:.2f})")
 
-            # Handle both numeric and non-numeric answers
-            if isinstance(extracted_answer, (int, float)):
-                # Numeric answer - add to numerical voting
-                numerical_answers.append((int(extracted_answer), solution))
-                extracted_answers_info.append((str(int(extracted_answer)), solution, "unified_numeric"))
-            elif isinstance(extracted_answer, str):
-                # Non-numeric answer (formulas, sets, etc.) - store for synthesis
-                extracted_answers_info.append((extracted_answer, solution, "unified_formula"))
-                logger.info(f"🗳️  VOTING: Non-numeric answer stored for synthesis: '{extracted_answer}'")
-            elif isinstance(extracted_answer, set):
-                # Set answers (e.g., for Problem 1) - convert to string for synthesis
-                set_str = "{" + ", ".join(map(str, sorted(extracted_answer))) + "}"
-                extracted_answers_info.append((set_str, solution, "unified_set"))
-                logger.info(f"🗳️  VOTING: Set answer stored for synthesis: '{set_str}'")
+            # Math-verify returns a list of all possible matches
+            # Iterate through list to find first valid answer
+            answers_to_process = []
+            if isinstance(extracted_answer, list):
+                answers_to_process = extracted_answer
             else:
-                # Other types - convert to string
+                answers_to_process = [extracted_answer]
+
+            # Process each answer in the list
+            for ans in answers_to_process:
+                # Handle both numeric and non-numeric answers
+                if isinstance(ans, (int, float)):
+                    # Numeric answer - add to numerical voting
+                    numerical_answers.append((int(ans), solution))
+                    extracted_answers_info.append((str(int(ans)), solution, "unified_numeric"))
+                    break  # Use first numeric answer found
+                elif isinstance(ans, str) and ans.strip():
+                    # Non-numeric answer (formulas, sets, etc.) - store for synthesis
+                    extracted_answers_info.append((ans, solution, "unified_formula"))
+                    logger.info(f"🗳️  VOTING: Non-numeric answer stored for synthesis: '{ans}'")
+                    break  # Use first valid string
+                elif isinstance(ans, set):
+                    # Set answers (e.g., for Problem 1) - convert to string for synthesis
+                    set_str = "{" + ", ".join(map(str, sorted(ans))) + "}"
+                    extracted_answers_info.append((set_str, solution, "unified_set"))
+                    logger.info(f"🗳️  VOTING: Set answer stored for synthesis: '{set_str}'")
+                    break  # Use first set found
+
+            # If no valid answer found after iterating list, log as other type
+            if not any(isinstance(ans, (int, float, str, set)) for ans in answers_to_process if isinstance(ans, str) and ans.strip()):
                 extracted_answers_info.append((str(extracted_answer), solution, "unified_other"))
                 logger.info(f"🗳️  VOTING: Other answer type stored for synthesis: '{extracted_answer}'")
         else:

scripts/eval_aime_benchmark.py

@@ -105,6 +105,20 @@ def extract_answer(response: str) -> Optional[int]:
     if extracted_answer is None:
         return None
 
+    # Math-verify returns a list of all possible matches
+    # Check if extracted_answer is a list and find first valid integer
+    if isinstance(extracted_answer, list):
+        for item in extracted_answer:
+            if isinstance(item, (int, float)):
+                answer = int(item)
+                if 0 <= answer <= 999:
+                    return answer
+            elif isinstance(item, str) and item.isdigit():
+                answer = int(item)
+                if 0 <= answer <= 999:
+                    return answer
+        return None
+
     # Convert to integer if needed - AIME answers are always integers
     if isinstance(extracted_answer, (int, float)):
         answer = int(extracted_answer)

scripts/eval_imo25_benchmark.py

@@ -118,6 +118,25 @@ def extract_answer_from_solution(solution: str, problem_id: int) -> str:
     if extracted_answer is None:
         return None
 
+    # Math-verify returns a list of all possible matches
+    # Iterate through list to find first valid format for this problem
+    if isinstance(extracted_answer, list):
+        for item in extracted_answer:
+            # Try each type conversion
+            if isinstance(item, set):
+                sorted_elements = sorted(list(item))
+                return "{" + ", ".join(map(str, sorted_elements)) + "}"
+            elif isinstance(item, (int, float)):
+                if problem_id == 3:
+                    return f"c = {int(item)}"
+                else:
+                    return str(int(item))
+            elif isinstance(item, str) and item.strip():
+                # Skip empty strings, return first non-empty string
+                return item
+        # If no valid item found in list, convert list to string
+        return str(extracted_answer)
+
     # Convert extracted answer to string format expected by evaluation
     if isinstance(extracted_answer, set):
         # Convert set to string format: {0, 1, 2, 3}