Modified Unit-tests so that they always report scores using '.' as decimal separator

README.md

@@ -1,3 +1,5 @@
+### This is a fork of [brateval](https://github.com/READ-BioMed/brateval).
+
 # BRAT-Eval v0.3.2
 
 

pom.xml

@@ -22,6 +22,12 @@
       <version>${junit-jupiter.version}</version>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>com.opencsv</groupId>
+      <artifactId>opencsv</artifactId>
+      <version>5.9</version>
+    </dependency>
   </dependencies>
 
     <distributionManagement>

src/main/java/au/com/nicta/csp/brateval/CompareEntities.java

@@ -1,8 +1,9 @@
 package au.com.nicta.csp.brateval;
 
+import com.opencsv.CSVReader;
+
+import java.io.*;
 import java.nio.file.Path;
-import java.io.File;
-import java.io.IOException;
 import java.util.*;
 
 
@@ -18,16 +19,71 @@ public class CompareEntities {
     static TaxonomyConfig taxonomy = TaxonomyConfig.singleton();
 
     public static void main(String argc[]) throws Exception {
+        Locale.setDefault(new Locale("en", "US")); //Reported scores use "." as decimal seperator
         Options.common = new Options(argc);
         verbose_output = Options.common.verbose;
         taxonomy.readConfigFile(Options.common.configFile);
         show_full_taxonomy = Options.common.show_full_taxonomy;
 
         String evalFolder = Options.common.evalFolder;
         String goldFolder = Options.common.goldFolder;
+        String outFolder = Options.common.outputFolder;
 
         System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString());
+
+        //<Here we fetch the System.out stream of Brateval>
+        java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream();
+        PrintStream stdout = System.out;
+        System.setOut(new java.io.PrintStream(out)); //Catch sysrtem out
         evaluate(goldFolder, evalFolder, Options.common.matchType);
+        System.setOut(stdout); //Reset System.out to default
+        System.out.println(out);
+        //</Here we fetch the System.out stream of Brateval>
+
+        //<Here we parse the system out stream>
+        int precisionPos = 4; int recallPos=5; int f1Pos=6;
+
+        List<Double> macroPrecisionScores = new ArrayList<>();
+        List<Double> macroRecallScores = new ArrayList<>();
+        List<Double> macroF1Scores = new ArrayList<>();
+
+        FileWriter output = new FileWriter(outFolder +File.separator +"scores.txt");
+
+        CSVReader reader = new CSVReader(new StringReader(out.toString()));
+        String[] nextLine;
+        while ((nextLine = reader.readNext()) != null) {
+            if (nextLine != null) { //Until end
+                if (nextLine.length == 7 && !nextLine[0].equals("")){
+                    if(nextLine[0].equals("all")){
+                        output.append("Task2aMicroP: " +nextLine[precisionPos] +"\n");
+                        output.append("Task2aMicroR: " +nextLine[recallPos] +"\n");
+                        output.append("Task2aMicroF1: " +nextLine[f1Pos] +"\n");
+
+                        //System.out.println("MicroP=" +nextLine[precisionPos]);
+                        //System.out.println("MicroR=" +nextLine[recallPos]);
+                        //System.out.println("MicroF1=" +nextLine[f1Pos]);
+                    }
+                    else{
+                        macroPrecisionScores.add(Double.parseDouble(nextLine[precisionPos]));
+                        macroRecallScores.add(Double.parseDouble(nextLine[recallPos]));
+                        macroF1Scores.add(Double.parseDouble(nextLine[f1Pos]));
+                    }
+
+                }
+            }
+        }
+        output.append("Task2aMacroP: " +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()) +"\n");
+        output.append("Task2aMacroR: " +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n");
+        output.append("Task2aMacroF1: " +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n");
+
+        //System.out.println("MacroP=" +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()));
+        //System.out.println("MacroR=" +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble()));
+        //System.out.println("MacroF1=" +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble()));
+
+        output.close();
+        //</Here we parse the system out stream>
+
+
     }
 
     static void report(TableOut summary, int level, String et, int TP, int FP, int FN) {

src/main/java/au/com/nicta/csp/brateval/CompareRelations.java

@@ -1,8 +1,9 @@
 package au.com.nicta.csp.brateval;
 
+import com.opencsv.CSVReader;
+
+import java.io.*;
 import java.nio.file.Paths;
-import java.io.File;
-import java.io.IOException;
 import java.util.*;
 
 /**
@@ -18,17 +19,21 @@ public class CompareRelations
     static  boolean show_full_taxonomy = false;
 	static  TaxonomyConfig taxonomy = TaxonomyConfig.singleton();
 
-	public static void main (String argc []) throws Exception
+    static String outFolder;
+
+    public static void main (String argc []) throws Exception
 	{
-		Options.common = new Options(argc);
+        Locale.setDefault(new Locale("en", "US")); //Reported scores use "." as decimal seperator
+        Options.common = new Options(argc);
 		verbose_output = Options.common.verbose;
 		taxonomy = new TaxonomyConfig(Options.common.configFile);
 		show_full_taxonomy = Options.common.show_full_taxonomy;
 
 		String evalFolder = Options.common.evalFolder;
 		String goldFolder = Options.common.goldFolder;
+        outFolder = Options.common.outputFolder;
 
-		System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString() );
+        System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString() );
 		evaluate(goldFolder, evalFolder, Options.common.matchType);
     }
 
@@ -45,14 +50,14 @@ static void report(int level, String rt, int TP, int FP, int FN, int MFP, int MF
         { f_measure = (2*precision*recall)/(double)(precision+recall); }
 
         System.out.println(rt
-                + "|tp:" + TP
-                + "|fp:" + FP
-                + "|fn:" + FN
-                + "|precision:" + String.format("%1.4f", precision)
-                + "|recall:" + String.format("%1.4f", recall)
-                + "|f1:" + String.format("%1.4f", f_measure)
-                + "|fpm:" + MFP
-                + "|fnm:" + MFN
+                + "," + TP
+                + "," + FP
+                + "," + FN
+                + "," + String.format("%1.4f", precision)
+                + "," + String.format("%1.4f", recall)
+                + "," + String.format("%1.4f", f_measure)
+                + "," + MFP
+                + "," + MFN
         );
     }
 
@@ -213,8 +218,14 @@ public static void evaluate(String goldFolderPath, String evalFolderPath, MatchT
                 }
             }
         }
+        //<Catches the System-out-output>
+        java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream();
+        PrintStream stdout = System.out;
+        System.setOut(new java.io.PrintStream(out)); //Catch sysrtem out
+        //</Catches the System-out-output>
 
         System.out.println("Summary:");
+        System.out.println("type,tp,fp,fn,precision,recall,f1,fpm,fnm");
 
         taxonomy.traverseRelations(new HierList.Visitor<TaxonomyConfig.RelationDesc>() {
             public void pre(int level, TaxonomyConfig.RelationDesc curr,
@@ -292,5 +303,55 @@ public void post(int level, TaxonomyConfig.RelationDesc curr,
             report(0, rt, TP, FP, FN, MFP, MFN);
         }
         report(0, "all", totalTP, totalFP, totalFN, totalMFP, totalMFN);
+
+        //<Reset System.out to default>
+        System.setOut(stdout);
+        System.out.println(out);
+        //</Reset System.out to default>
+
+        //<Here we parse the system out stream>
+        int precisionPos = 4; int recallPos=5; int f1Pos=6;
+
+        List<Double> macroPrecisionScores = new ArrayList<>();
+        List<Double> macroRecallScores = new ArrayList<>();
+        List<Double> macroF1Scores = new ArrayList<>();
+
+        FileWriter output = new FileWriter(outFolder +File.separator +"scores.txt");
+        String[] nextLine;
+        CSVReader reader = new CSVReader(new StringReader(out.toString()));
+        while ((nextLine = reader.readNext()) != null) {
+            if (nextLine != null) { //Until end
+                if (nextLine.length == 9  && !nextLine[0].equals("type")){
+                    if(nextLine[0].equals("all")){
+
+                        output.append("Task2bMicroP: " +nextLine[precisionPos] +"\n");
+                        output.append("Task2bMicroR: " +nextLine[recallPos] +"\n");
+                        output.append("Task2bMicroF1: " +nextLine[f1Pos] +"\n");
+
+                        //System.out.println("MicroP=" +nextLine[precisionPos]);
+                        //System.out.println("MicroR=" +nextLine[recallPos]);
+                        //System.out.println("MicroF1=" +nextLine[f1Pos]);
+                    }
+                    else{
+                        //System.out.println(String.join(",", nextLine));
+                        macroPrecisionScores.add(Double.parseDouble(nextLine[precisionPos]));
+                        macroRecallScores.add(Double.parseDouble(nextLine[recallPos]));
+                        macroF1Scores.add(Double.parseDouble(nextLine[f1Pos]));
+                    }
+                }
+
+            }
+        }
+
+        output.append("Task2bMacroP: " +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()) +"\n");
+        output.append("Task2bMacroR: " +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n");
+        output.append("Task2bMacroF1: " +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n");
+
+        //System.out.println("MacroP=" +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()));
+        //System.out.println("MacroR=" +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble()));
+        //System.out.println("MacroF1=" +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble()));
+
+        output.close();
+        //</Here we parse the system out stream>
     }
 }

src/main/java/au/com/nicta/csp/brateval/Options.java

@@ -26,12 +26,13 @@ public enum OutFmt {
 	}
 
 
-	public OutFmt outFmt = OutFmt.PLAIN;
+	public OutFmt outFmt = OutFmt.CSV;
 
 	// options related to annotation folders
 	public String goldFolder; // name of folder for gold standard annotations
 	public String evalFolder; // name of folder for annotations to be compared to gold standard
 	public String configFile; // location/name of annotation.conf file (path + name) [by default, "annotation.conf" in the current directory will be used]
+	public String outputFolder;
 
 	// options related to matching
 	public MatchType matchType; // specify how spans are treated for matching
@@ -50,6 +51,7 @@ public Options(String [] argv) {
 		goldFolder = null;
 		evalFolder = null;
 		configFile = "annotation.conf";
+		outputFolder="";
 
 		for (int j=0; j<argv.length; ++j) {
 			if (argv[j].charAt(0) == '-') {
@@ -99,6 +101,10 @@ else if (spanSelection.equalsIgnoreCase("exact") || spanSelection.equalsIgnoreCa
 							typeSelection = "HIERARCHICAL";
 						matchType.setTypeMatchType(TypeMatch.valueOf(toEnumName(typeSelection)));
 						break;
+					case "-o" : case "-outputfolder":
+						j++;
+						outputFolder = argv[j];
+						break;
 					default:
 						throw new IllegalArgumentException("Unsupported option" + argv[j]);
 				}

src/test/java/au/com/nicta/csp/brateval/CompareEntitiesTest.java

@@ -7,6 +7,7 @@
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Locale;
 
 class CompareEntitiesTest {
 
@@ -36,6 +37,7 @@ public void restoreStreams() {
 
     @BeforeEach
      void setUp() throws URISyntaxException {
+        Locale.setDefault(new Locale("en", "US")); //Reported scores use "." as decimal seperator
         dataDir = Paths.get(this.getClass().getClassLoader().getResource("data").toURI()).toString();
         GOLD_PATH = getPath(dataDir,"corpora","chemu_sample");
         BRATEVAL_PATH = getPath(dataDir,"bratevals","chemu_sample");

src/test/java/au/com/nicta/csp/brateval/CompareRelationsTest.java

@@ -7,6 +7,7 @@
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Locale;
 
 import static au.com.nicta.csp.brateval.CompareEntitiesTest.buildBratEvalFileName;
 import static au.com.nicta.csp.brateval.CompareEntitiesTest.getPath;
@@ -39,6 +40,7 @@ public void restoreStreams() {
 
     @BeforeEach
     void setUp() throws URISyntaxException {
+        Locale.setDefault(new Locale("en", "US")); //Reported scores use "." as decimal seperator
 
         dataDir = Paths.get(this.getClass().getClassLoader().getResource("data").toURI()).toString();