diff --git a/README.md b/README.md index 1ed8069..fd2659f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +### This is a fork of [brateval](https://github.com/READ-BioMed/brateval). + # BRAT-Eval v0.3.2 diff --git a/pom.xml b/pom.xml index 687066f..bd01f30 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,12 @@ ${junit-jupiter.version} test + + + com.opencsv + opencsv + 5.9 + diff --git a/src/main/java/au/com/nicta/csp/brateval/CompareEntities.java b/src/main/java/au/com/nicta/csp/brateval/CompareEntities.java index aa0a601..f1f8f1c 100755 --- a/src/main/java/au/com/nicta/csp/brateval/CompareEntities.java +++ b/src/main/java/au/com/nicta/csp/brateval/CompareEntities.java @@ -1,8 +1,9 @@ package au.com.nicta.csp.brateval; +import com.opencsv.CSVReader; + +import java.io.*; import java.nio.file.Path; -import java.io.File; -import java.io.IOException; import java.util.*; @@ -18,6 +19,7 @@ public class CompareEntities { static TaxonomyConfig taxonomy = TaxonomyConfig.singleton(); public static void main(String argc[]) throws Exception { + Locale.setDefault(new Locale("en", "US")); //Reported scores use "." as decimal seperator Options.common = new Options(argc); verbose_output = Options.common.verbose; taxonomy.readConfigFile(Options.common.configFile); @@ -25,9 +27,63 @@ public static void main(String argc[]) throws Exception { String evalFolder = Options.common.evalFolder; String goldFolder = Options.common.goldFolder; + String outFolder = Options.common.outputFolder; System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString()); + + // + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + PrintStream stdout = System.out; + System.setOut(new java.io.PrintStream(out)); //Catch sysrtem out evaluate(goldFolder, evalFolder, Options.common.matchType); + System.setOut(stdout); //Reset System.out to default + System.out.println(out); + // + + // + int precisionPos = 4; int recallPos=5; int f1Pos=6; + + List macroPrecisionScores = new ArrayList<>(); + List macroRecallScores = new ArrayList<>(); + List macroF1Scores = new ArrayList<>(); + + FileWriter output = new FileWriter(outFolder +File.separator +"scores.txt"); + + CSVReader reader = new CSVReader(new StringReader(out.toString())); + String[] nextLine; + while ((nextLine = reader.readNext()) != null) { + if (nextLine != null) { //Until end + if (nextLine.length == 7 && !nextLine[0].equals("")){ + if(nextLine[0].equals("all")){ + output.append("Task2aMicroP: " +nextLine[precisionPos] +"\n"); + output.append("Task2aMicroR: " +nextLine[recallPos] +"\n"); + output.append("Task2aMicroF1: " +nextLine[f1Pos] +"\n"); + + //System.out.println("MicroP=" +nextLine[precisionPos]); + //System.out.println("MicroR=" +nextLine[recallPos]); + //System.out.println("MicroF1=" +nextLine[f1Pos]); + } + else{ + macroPrecisionScores.add(Double.parseDouble(nextLine[precisionPos])); + macroRecallScores.add(Double.parseDouble(nextLine[recallPos])); + macroF1Scores.add(Double.parseDouble(nextLine[f1Pos])); + } + + } + } + } + output.append("Task2aMacroP: " +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()) +"\n"); + output.append("Task2aMacroR: " +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n"); + output.append("Task2aMacroF1: " +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n"); + + //System.out.println("MacroP=" +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble())); + //System.out.println("MacroR=" +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble())); + //System.out.println("MacroF1=" +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble())); + + output.close(); + // + + } static void report(TableOut summary, int level, String et, int TP, int FP, int FN) { diff --git a/src/main/java/au/com/nicta/csp/brateval/CompareRelations.java b/src/main/java/au/com/nicta/csp/brateval/CompareRelations.java index 35d383e..5ad446f 100755 --- a/src/main/java/au/com/nicta/csp/brateval/CompareRelations.java +++ b/src/main/java/au/com/nicta/csp/brateval/CompareRelations.java @@ -1,8 +1,9 @@ package au.com.nicta.csp.brateval; +import com.opencsv.CSVReader; + +import java.io.*; import java.nio.file.Paths; -import java.io.File; -import java.io.IOException; import java.util.*; /** @@ -18,17 +19,21 @@ public class CompareRelations static boolean show_full_taxonomy = false; static TaxonomyConfig taxonomy = TaxonomyConfig.singleton(); - public static void main (String argc []) throws Exception + static String outFolder; + + public static void main (String argc []) throws Exception { - Options.common = new Options(argc); + Locale.setDefault(new Locale("en", "US")); //Reported scores use "." as decimal seperator + Options.common = new Options(argc); verbose_output = Options.common.verbose; taxonomy = new TaxonomyConfig(Options.common.configFile); show_full_taxonomy = Options.common.show_full_taxonomy; String evalFolder = Options.common.evalFolder; String goldFolder = Options.common.goldFolder; + outFolder = Options.common.outputFolder; - System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString() ); + System.out.println("Evaluating Folder: " + evalFolder + " against Gold Folder: " + goldFolder + " Match settings : " + Options.common.matchType.toString() ); evaluate(goldFolder, evalFolder, Options.common.matchType); } @@ -45,14 +50,14 @@ static void report(int level, String rt, int TP, int FP, int FN, int MFP, int MF { f_measure = (2*precision*recall)/(double)(precision+recall); } System.out.println(rt - + "|tp:" + TP - + "|fp:" + FP - + "|fn:" + FN - + "|precision:" + String.format("%1.4f", precision) - + "|recall:" + String.format("%1.4f", recall) - + "|f1:" + String.format("%1.4f", f_measure) - + "|fpm:" + MFP - + "|fnm:" + MFN + + "," + TP + + "," + FP + + "," + FN + + "," + String.format("%1.4f", precision) + + "," + String.format("%1.4f", recall) + + "," + String.format("%1.4f", f_measure) + + "," + MFP + + "," + MFN ); } @@ -213,8 +218,14 @@ public static void evaluate(String goldFolderPath, String evalFolderPath, MatchT } } } + // + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + PrintStream stdout = System.out; + System.setOut(new java.io.PrintStream(out)); //Catch sysrtem out + // System.out.println("Summary:"); + System.out.println("type,tp,fp,fn,precision,recall,f1,fpm,fnm"); taxonomy.traverseRelations(new HierList.Visitor() { public void pre(int level, TaxonomyConfig.RelationDesc curr, @@ -292,5 +303,55 @@ public void post(int level, TaxonomyConfig.RelationDesc curr, report(0, rt, TP, FP, FN, MFP, MFN); } report(0, "all", totalTP, totalFP, totalFN, totalMFP, totalMFN); + + // + System.setOut(stdout); + System.out.println(out); + // + + // + int precisionPos = 4; int recallPos=5; int f1Pos=6; + + List macroPrecisionScores = new ArrayList<>(); + List macroRecallScores = new ArrayList<>(); + List macroF1Scores = new ArrayList<>(); + + FileWriter output = new FileWriter(outFolder +File.separator +"scores.txt"); + String[] nextLine; + CSVReader reader = new CSVReader(new StringReader(out.toString())); + while ((nextLine = reader.readNext()) != null) { + if (nextLine != null) { //Until end + if (nextLine.length == 9 && !nextLine[0].equals("type")){ + if(nextLine[0].equals("all")){ + + output.append("Task2bMicroP: " +nextLine[precisionPos] +"\n"); + output.append("Task2bMicroR: " +nextLine[recallPos] +"\n"); + output.append("Task2bMicroF1: " +nextLine[f1Pos] +"\n"); + + //System.out.println("MicroP=" +nextLine[precisionPos]); + //System.out.println("MicroR=" +nextLine[recallPos]); + //System.out.println("MicroF1=" +nextLine[f1Pos]); + } + else{ + //System.out.println(String.join(",", nextLine)); + macroPrecisionScores.add(Double.parseDouble(nextLine[precisionPos])); + macroRecallScores.add(Double.parseDouble(nextLine[recallPos])); + macroF1Scores.add(Double.parseDouble(nextLine[f1Pos])); + } + } + + } + } + + output.append("Task2bMacroP: " +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble()) +"\n"); + output.append("Task2bMacroR: " +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n"); + output.append("Task2bMacroF1: " +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble())+"\n"); + + //System.out.println("MacroP=" +String.format("%1.4f",macroPrecisionScores.stream().mapToDouble(var -> var).average().getAsDouble())); + //System.out.println("MacroR=" +String.format("%1.4f",macroRecallScores.stream().mapToDouble(var -> var).average().getAsDouble())); + //System.out.println("MacroF1=" +String.format("%1.4f",macroF1Scores.stream().mapToDouble(var -> var).average().getAsDouble())); + + output.close(); + // } } \ No newline at end of file diff --git a/src/main/java/au/com/nicta/csp/brateval/Options.java b/src/main/java/au/com/nicta/csp/brateval/Options.java index 2eab9d8..8777ff6 100755 --- a/src/main/java/au/com/nicta/csp/brateval/Options.java +++ b/src/main/java/au/com/nicta/csp/brateval/Options.java @@ -26,12 +26,13 @@ public enum OutFmt { } - public OutFmt outFmt = OutFmt.PLAIN; + public OutFmt outFmt = OutFmt.CSV; // options related to annotation folders public String goldFolder; // name of folder for gold standard annotations public String evalFolder; // name of folder for annotations to be compared to gold standard public String configFile; // location/name of annotation.conf file (path + name) [by default, "annotation.conf" in the current directory will be used] + public String outputFolder; // options related to matching public MatchType matchType; // specify how spans are treated for matching @@ -50,6 +51,7 @@ public Options(String [] argv) { goldFolder = null; evalFolder = null; configFile = "annotation.conf"; + outputFolder=""; for (int j=0; j