diff --git a/README.md b/README.md
index 1ed8069..fd2659f 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+### This is a fork of [brateval](https://github.com/READ-BioMed/brateval).
+
# BRAT-Eval v0.3.2
diff --git a/pom.xml b/pom.xml
index 687066f..bd01f30 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,6 +22,12 @@
${junit-jupiter.version}
test
+
+
+ com.opencsv
+ opencsv
+ 5.9
+
diff --git a/src/main/java/au/com/nicta/csp/brateval/CompareEntities.java b/src/main/java/au/com/nicta/csp/brateval/CompareEntities.java
index aa0a601..f1f8f1c 100755
--- a/src/main/java/au/com/nicta/csp/brateval/CompareEntities.java
+++ b/src/main/java/au/com/nicta/csp/brateval/CompareEntities.java
@@ -1,8 +1,9 @@
package au.com.nicta.csp.brateval;
+import com.opencsv.CSVReader;
+
+import java.io.*;
import java.nio.file.Path;
-import java.io.File;
-import java.io.IOException;
import java.util.*;
@@ -18,6 +19,7 @@ public class CompareEntities {
static TaxonomyConfig taxonomy = TaxonomyConfig.singleton();
public static void main(String argc[]) throws Exception {
+ Locale.setDefault(new Locale("en", "US")); //Reported scores use "." as decimal seperator
Options.common = new Options(argc);
verbose_output = Options.common.verbose;
taxonomy.readConfigFile(Options.common.configFile);
@@ -25,9 +27,63 @@ public static void main(String argc[]) throws Exception {
String evalFolder = Options.common.evalFolder;
String goldFolder = Options.common.goldFolder;
+ String outFolder = Options.common.outputFolder;
System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString());
+
+ //
+ java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream();
+ PrintStream stdout = System.out;
+ System.setOut(new java.io.PrintStream(out)); //Catch sysrtem out
evaluate(goldFolder, evalFolder, Options.common.matchType);
+ System.setOut(stdout); //Reset System.out to default
+ System.out.println(out);
+ //
+
+ //
+ int precisionPos = 4; int recallPos=5; int f1Pos=6;
+
+ List macroPrecisionScores = new ArrayList<>();
+ List macroRecallScores = new ArrayList<>();
+ List macroF1Scores = new ArrayList<>();
+
+ FileWriter output = new FileWriter(outFolder +File.separator +"scores.txt");
+
+ CSVReader reader = new CSVReader(new StringReader(out.toString()));
+ String[] nextLine;
+ while ((nextLine = reader.readNext()) != null) {
+ if (nextLine != null) { //Until end
+ if (nextLine.length == 7 && !nextLine[0].equals("")){
+ if(nextLine[0].equals("all")){
+ output.append("Task2aMicroP: " +nextLine[precisionPos] +"\n");
+ output.append("Task2aMicroR: " +nextLine[recallPos] +"\n");
+ output.append("Task2aMicroF1: " +nextLine[f1Pos] +"\n");
+
+ //System.out.println("MicroP=" +nextLine[precisionPos]);
+ //System.out.println("MicroR=" +nextLine[recallPos]);
+ //System.out.println("MicroF1=" +nextLine[f1Pos]);
+ }
+ else{
+ macroPrecisionScores.add(Double.parseDouble(nextLine[precisionPos]));
+ macroRecallScores.add(Double.parseDouble(nextLine[recallPos]));
+ macroF1Scores.add(Double.parseDouble(nextLine[f1Pos]));
+ }
+
+ }
+ }
+ }
+ output.append("Task2aMacroP: " +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()) +"\n");
+ output.append("Task2aMacroR: " +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n");
+ output.append("Task2aMacroF1: " +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n");
+
+ //System.out.println("MacroP=" +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()));
+ //System.out.println("MacroR=" +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble()));
+ //System.out.println("MacroF1=" +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble()));
+
+ output.close();
+ //
+
+
}
static void report(TableOut summary, int level, String et, int TP, int FP, int FN) {
diff --git a/src/main/java/au/com/nicta/csp/brateval/CompareRelations.java b/src/main/java/au/com/nicta/csp/brateval/CompareRelations.java
index 35d383e..5ad446f 100755
--- a/src/main/java/au/com/nicta/csp/brateval/CompareRelations.java
+++ b/src/main/java/au/com/nicta/csp/brateval/CompareRelations.java
@@ -1,8 +1,9 @@
package au.com.nicta.csp.brateval;
+import com.opencsv.CSVReader;
+
+import java.io.*;
import java.nio.file.Paths;
-import java.io.File;
-import java.io.IOException;
import java.util.*;
/**
@@ -18,17 +19,21 @@ public class CompareRelations
static boolean show_full_taxonomy = false;
static TaxonomyConfig taxonomy = TaxonomyConfig.singleton();
- public static void main (String argc []) throws Exception
+ static String outFolder;
+
+ public static void main (String argc []) throws Exception
{
- Options.common = new Options(argc);
+ Locale.setDefault(new Locale("en", "US")); //Reported scores use "." as decimal seperator
+ Options.common = new Options(argc);
verbose_output = Options.common.verbose;
taxonomy = new TaxonomyConfig(Options.common.configFile);
show_full_taxonomy = Options.common.show_full_taxonomy;
String evalFolder = Options.common.evalFolder;
String goldFolder = Options.common.goldFolder;
+ outFolder = Options.common.outputFolder;
- System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString() );
+ System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString() );
evaluate(goldFolder, evalFolder, Options.common.matchType);
}
@@ -45,14 +50,14 @@ static void report(int level, String rt, int TP, int FP, int FN, int MFP, int MF
{ f_measure = (2*precision*recall)/(double)(precision+recall); }
System.out.println(rt
- + "|tp:" + TP
- + "|fp:" + FP
- + "|fn:" + FN
- + "|precision:" + String.format("%1.4f", precision)
- + "|recall:" + String.format("%1.4f", recall)
- + "|f1:" + String.format("%1.4f", f_measure)
- + "|fpm:" + MFP
- + "|fnm:" + MFN
+ + "," + TP
+ + "," + FP
+ + "," + FN
+ + "," + String.format("%1.4f", precision)
+ + "," + String.format("%1.4f", recall)
+ + "," + String.format("%1.4f", f_measure)
+ + "," + MFP
+ + "," + MFN
);
}
@@ -213,8 +218,14 @@ public static void evaluate(String goldFolderPath, String evalFolderPath, MatchT
}
}
}
+ //
+ java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream();
+ PrintStream stdout = System.out;
+ System.setOut(new java.io.PrintStream(out)); //Catch sysrtem out
+ //
System.out.println("Summary:");
+ System.out.println("type,tp,fp,fn,precision,recall,f1,fpm,fnm");
taxonomy.traverseRelations(new HierList.Visitor() {
public void pre(int level, TaxonomyConfig.RelationDesc curr,
@@ -292,5 +303,55 @@ public void post(int level, TaxonomyConfig.RelationDesc curr,
report(0, rt, TP, FP, FN, MFP, MFN);
}
report(0, "all", totalTP, totalFP, totalFN, totalMFP, totalMFN);
+
+ //
+ System.setOut(stdout);
+ System.out.println(out);
+ //
+
+ //
+ int precisionPos = 4; int recallPos=5; int f1Pos=6;
+
+ List macroPrecisionScores = new ArrayList<>();
+ List macroRecallScores = new ArrayList<>();
+ List macroF1Scores = new ArrayList<>();
+
+ FileWriter output = new FileWriter(outFolder +File.separator +"scores.txt");
+ String[] nextLine;
+ CSVReader reader = new CSVReader(new StringReader(out.toString()));
+ while ((nextLine = reader.readNext()) != null) {
+ if (nextLine != null) { //Until end
+ if (nextLine.length == 9 && !nextLine[0].equals("type")){
+ if(nextLine[0].equals("all")){
+
+ output.append("Task2bMicroP: " +nextLine[precisionPos] +"\n");
+ output.append("Task2bMicroR: " +nextLine[recallPos] +"\n");
+ output.append("Task2bMicroF1: " +nextLine[f1Pos] +"\n");
+
+ //System.out.println("MicroP=" +nextLine[precisionPos]);
+ //System.out.println("MicroR=" +nextLine[recallPos]);
+ //System.out.println("MicroF1=" +nextLine[f1Pos]);
+ }
+ else{
+ //System.out.println(String.join(",", nextLine));
+ macroPrecisionScores.add(Double.parseDouble(nextLine[precisionPos]));
+ macroRecallScores.add(Double.parseDouble(nextLine[recallPos]));
+ macroF1Scores.add(Double.parseDouble(nextLine[f1Pos]));
+ }
+ }
+
+ }
+ }
+
+ output.append("Task2bMacroP: " +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()) +"\n");
+ output.append("Task2bMacroR: " +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n");
+ output.append("Task2bMacroF1: " +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n");
+
+ //System.out.println("MacroP=" +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()));
+ //System.out.println("MacroR=" +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble()));
+ //System.out.println("MacroF1=" +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble()));
+
+ output.close();
+ //
}
}
\ No newline at end of file
diff --git a/src/main/java/au/com/nicta/csp/brateval/Options.java b/src/main/java/au/com/nicta/csp/brateval/Options.java
index 2eab9d8..8777ff6 100755
--- a/src/main/java/au/com/nicta/csp/brateval/Options.java
+++ b/src/main/java/au/com/nicta/csp/brateval/Options.java
@@ -26,12 +26,13 @@ public enum OutFmt {
}
- public OutFmt outFmt = OutFmt.PLAIN;
+ public OutFmt outFmt = OutFmt.CSV;
// options related to annotation folders
public String goldFolder; // name of folder for gold standard annotations
public String evalFolder; // name of folder for annotations to be compared to gold standard
public String configFile; // location/name of annotation.conf file (path + name) [by default, "annotation.conf" in the current directory will be used]
+ public String outputFolder;
// options related to matching
public MatchType matchType; // specify how spans are treated for matching
@@ -50,6 +51,7 @@ public Options(String [] argv) {
goldFolder = null;
evalFolder = null;
configFile = "annotation.conf";
+ outputFolder="";
for (int j=0; j