add to inspect

src/lighteval/main_inspect.py

@@ -52,6 +52,11 @@ def get_inspect_ai_task(
     name = lighteval_task_config.name
     sample_fields = lighteval_task_config.sample_fields
 
+    if sample_fields is None:
+        raise ValueError(
+            f"Task {name} is not supported by inspect_ai yet. You can either define it or use a different backend, `lighteval --help`"
+        )
+
     dataset_repo = lighteval_task_config.hf_repo
     dataset_subset = lighteval_task_config.hf_subset
     dataset_split = lighteval_task_config.evaluation_splits[0]

src/lighteval/tasks/tasks/aimo.py

@@ -17,7 +17,10 @@
 paper:
 """
 
-from lighteval.metrics.metrics import Metrics
+from inspect_ai.dataset import Sample
+from inspect_ai.solver import generate
+
+from lighteval.metrics.metrics import Metrics, math_scorer
 from lighteval.metrics.normalizations import math_normalizer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
@@ -32,9 +35,16 @@ def aimo_prompt(line, task_name: str = None):
     )
 
 
+def record_to_sample(record):
+    return Sample(input=record["problem"], target=str(record["answer"]))
+
+
 task = LightevalTaskConfig(
     name="aimo_progress_prize_1",
     prompt_function=aimo_prompt,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
     hf_subset="",
     hf_repo="lighteval/aimo_progress_prize_1",
     hf_avail_splits=["train"],

src/lighteval/tasks/tasks/anli.py

@@ -22,6 +22,12 @@
 https://arxiv.org/abs/1910.14599
 """
 
+from string import ascii_uppercase
+
+from inspect_ai.dataset import Sample
+from inspect_ai.scorer import choice
+from inspect_ai.solver import multiple_choice
+
 from lighteval.metrics.metrics import Metrics
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
@@ -36,6 +42,12 @@ def anli_prompt(line, task_name: str = None):
     )
 
 
+def record_to_sample(record):
+    choices = ["True", "Neither", "False"]
+    query = f"{record['premise']}\nQuestion: {record['hypothesis']}"
+    return Sample(input=query, target=ascii_uppercase[record["label"]], choices=choices)
+
+
 anli_r1 = LightevalTaskConfig(
     name="anli:r1",
     prompt_function=anli_prompt,
@@ -49,6 +61,9 @@ def anli_prompt(line, task_name: str = None):
     metrics=[Metrics.loglikelihood_acc],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 
@@ -65,6 +80,9 @@ def anli_prompt(line, task_name: str = None):
     metrics=[Metrics.loglikelihood_acc],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 
@@ -81,6 +99,9 @@ def anli_prompt(line, task_name: str = None):
     metrics=[Metrics.loglikelihood_acc],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 TASKS_TABLE = [

src/lighteval/tasks/tasks/arc.py

@@ -22,6 +22,10 @@
 https://arxiv.org/abs/1803.05457
 """
 
+from inspect_ai.dataset import Sample
+from inspect_ai.scorer import choice
+from inspect_ai.solver import multiple_choice
+
 from lighteval.metrics.metrics import Metrics
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
@@ -36,6 +40,14 @@ def arc_prompt(line, task_name: str = None):
     )
 
 
+def record_to_sample(record):
+    query = record["question"].strip()
+    target = record["answerKey"]
+    choices = record["choices"]["text"]
+
+    return Sample(input=query, target=target, choices=choices)
+
+
 arc_challenge = LightevalTaskConfig(
     name="arc:challenge",
     prompt_function=arc_prompt,
@@ -51,6 +63,9 @@ def arc_prompt(line, task_name: str = None):
     ],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 arc_easy = LightevalTaskConfig(
@@ -68,6 +83,9 @@ def arc_prompt(line, task_name: str = None):
     ],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 TASKS_TABLE = [arc_challenge, arc_easy]

src/lighteval/tasks/tasks/arithmetic.py

@@ -19,15 +19,25 @@
 https://arxiv.org/abs/2005.14165
 """
 
-from lighteval.metrics.metrics import Metrics
+from inspect_ai.dataset import Sample
+from inspect_ai.solver import generate
+
+from lighteval.metrics.metrics import Metrics, math_scorer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
 
+# TODO: convert dataset to parquet
+
+
 def arithmetic_prompt(line, task_name: str = None):
     return Doc(task_name=task_name, query=line["context"], choices=[line["completion"]], gold_index=[0])
 
 
+def record_to_sample(record):
+    return Sample(input=record["context"], target=record["completion"])
+
+
 arithmetic_1dc = LightevalTaskConfig(
     name="arithmetic:1dc",
     prompt_function=arithmetic_prompt,
@@ -41,6 +51,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_2da = LightevalTaskConfig(
@@ -56,6 +69,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_2dm = LightevalTaskConfig(
@@ -71,6 +87,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_2ds = LightevalTaskConfig(
@@ -86,6 +105,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_3da = LightevalTaskConfig(
@@ -101,6 +123,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_3ds = LightevalTaskConfig(
@@ -116,6 +141,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_4da = LightevalTaskConfig(
@@ -131,6 +159,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_4ds = LightevalTaskConfig(
@@ -146,6 +177,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_5da = LightevalTaskConfig(
@@ -161,6 +195,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_5ds = LightevalTaskConfig(
@@ -176,6 +213,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 TASKS_TABLE = [

src/lighteval/tasks/tasks/asdiv.py

@@ -19,7 +19,10 @@
 https://arxiv.org/abs/2410.12853
 """
 
-from lighteval.metrics.metrics import Metrics
+from inspect_ai.dataset import Sample
+from inspect_ai.solver import generate
+
+from lighteval.metrics.metrics import Metrics, math_scorer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
@@ -33,6 +36,12 @@ def asdiv_prompt(line, task_name: str = None):
     )
 
 
+def record_to_sample(record):
+    query = f"{record['body']}\n{record['question']}"
+    target = record["answer"].split(" (")[0]
+    return Sample(input=query, target=target)
+
+
 asdiv = LightevalTaskConfig(
     name="asdiv",
     prompt_function=asdiv_prompt,
@@ -46,6 +55,9 @@ def asdiv_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 TASKS_TABLE = [asdiv]

src/lighteval/tasks/tasks/babi_qa.py

@@ -26,6 +26,9 @@
 from lighteval.tasks.requests import Doc
 
 
+# TODO: clean dataset and convert to inspect-ai
+
+
 def babi_qa_prompt(line, task_name: str = None):
     def process_path(path: str) -> str:
         steps = path.split(",")