Fix preprocessing logic

Author: davidjurado

brats/preprocessing/mlcube/mlcube.yaml

@@ -28,4 +28,4 @@ tasks:
   statistics:
       parameters:
         inputs: {data_path: data/, parameters_file: {type: file, default: parameters.yaml}}
-        outputs: {output_path: {type: file, default: statistics.yaml}}
+        outputs: {output_path: {type: file, default: statistics.yaml}}

brats/preprocessing/project/mlcube.py

@@ -9,7 +9,7 @@
 
 
 class PreprocessTask:
-    """Runs preprocessing given the input data path"""
+    """Run preprocessing given the input data path"""
 
     @staticmethod
     def run(
@@ -26,6 +26,35 @@ def run(
         process = subprocess.Popen("./run.sh", cwd=".", env=env)
         process.wait()
 
+class SanityCheckTask:
+    """Run sanity check"""
+
+    @staticmethod
+    def run(
+        data_path: str, parameters_file: str
+    ) -> None:
+
+        cmd = f"python3 sanity_check.py --data_path={data_path}"
+        splitted_cmd = cmd.split()
+
+        process = subprocess.Popen(splitted_cmd, cwd=".")
+        process.wait()
+
+
+class StatisticsTask:
+    """Run statistics"""
+
+    @staticmethod
+    def run(
+        data_path: str, parameters_file: str, output_path: str
+    ) -> None:
+
+        cmd = f"python3 statistics.py --data_path={data_path} --out_file={output_path}"
+        splitted_cmd = cmd.split()
+
+        process = subprocess.Popen(splitted_cmd, cwd=".")
+        process.wait()
+
 
 @app.command("prepare")
 def prepare(
@@ -42,19 +71,15 @@ def sanity_check(
     data_path: str = typer.Option(..., "--data_path"),
     parameters_file: str = typer.Option(..., "--parameters_file"),
 ):
-    pass
+    SanityCheckTask.run(data_path, parameters_file)
 
 @app.command("statistics")
 def statistics(
     data_path: str = typer.Option(..., "--data_path"),
     parameters_file: str = typer.Option(..., "--parameters_file"),
     output_path: str = typer.Option(..., "--output_path")
 ):
-    stats = {
-        "stat": 1
-    }
-    with open(output_path, "w") as f:
-        yaml.dump(stats, f)
+    StatisticsTask.run(data_path, parameters_file, output_path)
 
 
 if __name__ == "__main__":

brats/preprocessing/project/sanity_check.py

@@ -0,0 +1,28 @@
+"""Sanity check logic"""
+import os
+import argparse
+
+
+def sanity_check(data):
+    """Runs a few checks to ensure data quality and integrity
+    Args:
+        names_df (pd.DataFrame): DataFrame containing transformed data.
+    """
+    # Here you must add all the checks you consider important regarding the
+    # state of the data
+    assert len(data) > 0
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Medperf Model Sanity Check Example")
+    parser.add_argument(
+        "--data_path",
+        dest="data",
+        type=str,
+        help="directory containing the prepared data",
+    )
+
+    args = parser.parse_args()
+
+    data = os.listdir(args.data)
+    sanity_check(data)

brats/preprocessing/project/statistics.py

@@ -0,0 +1,39 @@
+import os
+import yaml
+import argparse
+
+
+def get_statistics(data_path):
+    """Computes statistics about the data. This statistics are uploaded
+    to the Medperf platform under the data owner's approval. Include
+    every statistic you consider useful for determining the nature of the
+    data, but keep in mind that we want to keep the data as private as 
+    possible.
+    """
+
+    len_data = len(os.listdir(data_path))
+
+    stats = {
+        "data length": len_data
+    }
+
+    return stats
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("MedPerf Statistics Example")
+    parser.add_argument(
+        "--data_path",
+        type=str,
+        help="directory containing the prepared data",
+    )
+    parser.add_argument(
+        "--out_file", dest="out_file", type=str, help="file to store statistics"
+    )
+
+    args = parser.parse_args()
+
+    stats = get_statistics(args.data_path)
+
+    with open(args.out_file, "w") as f:
+        yaml.dump(stats, f)