initial unitxt evaluator

Signed-off-by: Roni Friedman-Melamed <Roni.friedman-melamed@il.ibm.com>

Author: Roni-Friedman

my_tasks/my_task.yaml

@@ -0,0 +1,3 @@
+task: my_task
+include: unitxt
+recipe: card=cards.wnli,template=templates.classification.multi_class.relation.default,max_train_instances=5,loader_limit=20,num_demos=3,demos_pool_size=10

my_tasks/unitxt

@@ -0,0 +1 @@
+class: !function /Users/ronches/miniforge3/envs/lmeval/lib/python3.10/site-packages/lm_eval/tasks/unitxt/task.Unitxt

src/instructlab/eval/unitxt.py

@@ -0,0 +1,65 @@
+"""
+Unitxt - Flexible, Shareable and Reusable Data Preparation and Evaluation for Generative AI
+https://github.com/IBM/unitxt
+https://arxiv.org/abs/2401.14019
+"""
+
+# Standard
+import os
+
+# First Party
+from instructlab.eval.mmlu import MMLUBranchEvaluator
+
+# Local
+from .logger_config import setup_logger
+
+logger = setup_logger(__name__)
+
+class UnitxtEvaluator(MMLUBranchEvaluator):
+    name = "unitxt"
+    def __init__(
+        self,
+        model_path,
+        tasks_dir: str,
+        tasks: list[str],    
+        # unitxt_recipe: str,
+    ):
+        # tasks,tasks_dir = self.prepare_files(unitxt_recipe)
+        super().__init__(
+            model_path = model_path,
+            tasks_dir = tasks_dir,
+            tasks = tasks,
+            few_shots = 0
+        )
+
+    def prepare_files(self, unitxt_recipe)->tuple:
+        tasks = ''
+        tasks_dir = ''
+        return tasks,tasks_dir
+
+    def run(self,server_url: str | None = None) -> tuple:
+        """
+        Runs evaluation
+
+        Returns:
+            overall_scores       Average scores for the task group
+            individual_scores   Individual scores for each task in the task group
+        """
+        logger.debug(locals())
+        os.environ["TOKENIZERS_PARALLELISM"] = "true"
+        results = self._run_mmlu(server_url=server_url)
+        with open('my_tasks/output.txt', 'w') as f:
+            print(results, file=f)
+        taskname = self.tasks[0]
+        global_scores = results[taskname]
+        global_scores.pop('alias')
+        instance_scores = None
+        # instances = results['samples'][taskname]
+        # instance_scores = {}
+        # metrics = [metric.replace('metrics.','') for metric in instances[0]['doc']['metrics']]
+        # for i,instance in enumerate(instances):
+        #     scores = {}
+        #     for metric in metrics:
+        #         scores[metric] = instance[metric][0]
+        #     instance_scores[i] = scores
+        return global_scores,instance_scores

tests/test_unitxt.py

@@ -0,0 +1,22 @@
+# First Party
+from instructlab.eval.unitxt import UnitxtEvaluator
+
+
+def test_unitxt():
+    print("===> Executing 'test_unitxt'...")
+    try:
+        model_path = "instructlab/granite-7b-lab"
+        tasks = ["my_task"]
+        unitxt = UnitxtEvaluator(
+            model_path=model_path, tasks_dir='./my_tasks/', tasks=tasks
+        )
+        overall_score, _ = unitxt.run()
+        print(overall_score)
+    except Exception as exc:
+        print(f"'test_unitxt_branch' failed: {exc}")
+        return False
+    return True
+
+
+if __name__ == "__main__":
+    assert test_unitxt() == True