create unitxt files on the fly

Signed-off-by: Roni Friedman-Melamed <Roni.friedman-melamed@il.ibm.com>

Author: Roni-Friedman

src/instructlab/eval/mmlu.py

@@ -153,7 +153,7 @@ def run(self, server_url: str | None = None) -> tuple:
 
         return overall_score, individual_scores
 
-    def _run_mmlu(self, server_url: str | None = None) -> dict:
+    def _run_mmlu(self, server_url: str | None = None, return_all_results:bool = False) -> dict:
         if server_url is not None:
             # Requires lm_eval >= 0.4.4
             model_args = f"base_url={server_url}/completions,model={self.model_path},tokenizer_backend=huggingface"
@@ -177,7 +177,10 @@ def _run_mmlu(self, server_url: str | None = None) -> dict:
             device=self.device,
             task_manager=tm,
         )
-        results = mmlu_output["results"]
+        if return_all_results:
+            results = mmlu_output
+        else:
+            results = mmlu_output["results"]
         return results
 
     # This method converts general errors from simple_evaluate

src/instructlab/eval/unitxt.py

@@ -5,7 +5,12 @@
 """
 
 # Standard
-import os
+import os, shutil
+import yaml
+from uuid import uuid4
+
+# Third Party
+from lm_eval.tasks.unitxt import task
 
 # First Party
 from instructlab.eval.mmlu import MMLUBranchEvaluator
@@ -16,26 +21,43 @@
 logger = setup_logger(__name__)
 
 class UnitxtEvaluator(MMLUBranchEvaluator):
+    """
+    An evaluator class, running Unitxt evaluation
+
+    Attributes:
+        model_path      absolute path to or name of a huggingface model
+        unitxt_recipe   unitxt recipe (see unitxt.ai for more information)
+                        A Recipe holds a complete specification of a unitxt pipeline 
+                        Example: card=cards.wnli,template=templates.classification.multi_class.relation.default,max_train_instances=5,loader_limit=20,num_demos=3,demos_pool_size=10
+    
+    """
     name = "unitxt"
     def __init__(
         self,
-        model_path,
-        tasks_dir: str,
-        tasks: list[str],    
-        # unitxt_recipe: str,
+        model_path,   
+        unitxt_recipe: str,
     ):
-        # tasks,tasks_dir = self.prepare_files(unitxt_recipe)
+        tasks,tasks_dir = self.prepare_unitxt_files(unitxt_recipe)
         super().__init__(
             model_path = model_path,
             tasks_dir = tasks_dir,
             tasks = tasks,
             few_shots = 0
         )
 
-    def prepare_files(self, unitxt_recipe)->tuple:
-        tasks = ''
-        tasks_dir = ''
-        return tasks,tasks_dir
+    def prepare_unitxt_files(self, unitxt_recipe)->tuple:
+        temp_task = str(uuid4())
+        temp_tasks_dir = f'unitxt_temp_{temp_task}'
+        yaml_file = os.path.join(temp_tasks_dir,f"{temp_task}.yaml")
+        create_unitxt_pointer(temp_tasks_dir)
+        create_unitxt_yaml(yaml_file=yaml_file, unitxt_recipe=unitxt_recipe, task_name=temp_task)
+        return temp_task,temp_tasks_dir
+
+    def remove_temp_files(self):
+        if self.tasks_dir.startswith('temp_'): #to avoid unintended deletion if this class is inherited
+            shutil.rmtree(self.tasks_dir)
+        else:
+            logger.warning("unitxt tasks dir did not start with 'temp_' and therefor was not deleted")
 
     def run(self,server_url: str | None = None) -> tuple:
         """
@@ -47,19 +69,44 @@ def run(self,server_url: str | None = None) -> tuple:
         """
         logger.debug(locals())
         os.environ["TOKENIZERS_PARALLELISM"] = "true"
-        results = self._run_mmlu(server_url=server_url)
+        results = self._run_mmlu(server_url=server_url, return_all_results=True)
         with open('my_tasks/output.txt', 'w') as f:
             print(results, file=f)
         taskname = self.tasks[0]
-        global_scores = results[taskname]
+        global_scores = results['results'][taskname]
         global_scores.pop('alias')
-        instance_scores = None
-        # instances = results['samples'][taskname]
-        # instance_scores = {}
-        # metrics = [metric.replace('metrics.','') for metric in instances[0]['doc']['metrics']]
-        # for i,instance in enumerate(instances):
-        #     scores = {}
-        #     for metric in metrics:
-        #         scores[metric] = instance[metric][0]
-        #     instance_scores[i] = scores
+        try:
+            instances = results['samples'][taskname]
+            instance_scores = {}
+            metrics = [metric.replace('metrics.','') for metric in instances[0]['doc']['metrics']]
+            for i,instance in enumerate(instances):
+                scores = {}
+                for metric in metrics:
+                    scores[metric] = instance[metric][0]
+                instance_scores[i] = scores
+        except Exception as e:
+            logger.error("Error in extracting single instance scores")
+            logger.error(e)
+            logger.error(e.__traceback__)
+            instance_scores = None
+        self.remove_temp_files()
         return global_scores,instance_scores
+
+
+def create_unitxt_yaml(yaml_file,unitxt_recipe, task_name):
+    data = {
+    'task': f'{task_name}',
+    'include': 'unitxt',
+    'recipe': f'{unitxt_recipe}'
+    }
+    with open(yaml_file, 'w') as file:
+        yaml.dump(data, file, default_flow_style=False)
+    logger.info(f"task {task} unitxt recipe written to {yaml_file}")
+
+def create_unitxt_pointer(tasks_dir):
+    class_line = "class: !function " + task.__file__.replace("task.py", "task.Unitxt")
+    output_file = os.path.join(tasks_dir,'unitxt')
+    os.makedirs(os.path.dirname(output_file), exist_ok=True)
+    with open(output_file, 'w') as f:
+        f.write(class_line)
+    logger.info(f"Unitxt task pointer written to {output_file}")

tests/test_unitxt.py

@@ -1,16 +1,16 @@
 # First Party
-from instructlab.eval.unitxt import UnitxtEvaluator
+from instruclab.eval.unitxt import UnitxtEvaluator
 
 
 def test_unitxt():
     print("===> Executing 'test_unitxt'...")
     try:
         model_path = "instructlab/granite-7b-lab"
-        tasks = ["my_task"]
+        unitxt_recipe = "card=cards.wnli,template=templates.classification.multi_class.relation.default,max_train_instances=5,loader_limit=20,num_demos=3,demos_pool_size=10"
         unitxt = UnitxtEvaluator(
-            model_path=model_path, tasks_dir='./my_tasks/', tasks=tasks
+            model_path=model_path, unitxt_recipe=unitxt_recipe
         )
-        overall_score, _ = unitxt.run()
+        overall_score, single_scores = unitxt.run()
         print(overall_score)
     except Exception as exc:
         print(f"'test_unitxt_branch' failed: {exc}")