diff --git a/src/SIL.Machine/QualityEstimation/BookScores.cs b/src/SIL.Machine/QualityEstimation/BookScores.cs new file mode 100644 index 00000000..0e6623f0 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/BookScores.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; + +namespace SIL.Machine.QualityEstimation +{ + internal class BookScores + { + private readonly Dictionary> _verseUsabilities = new Dictionary>(); + + public readonly Dictionary Scores = new Dictionary(); + + public void AddScore(string book, Score score) => Scores[book] = score; + + public Score GetScore(string book) => Scores.TryGetValue(book, out Score score) ? score : null; + + public void AppendVerseUsability(string book, double usability) + { + if (!_verseUsabilities.TryGetValue(book, out List list)) + { + list = new List(); + _verseUsabilities[book] = list; + } + + list.Add(usability); + } + + public List GetVerseUsabilities(string book) => + _verseUsabilities.TryGetValue(book, out List list) ? new List(list) : new List(); + } +} diff --git a/src/SIL.Machine/QualityEstimation/BookUsability.cs b/src/SIL.Machine/QualityEstimation/BookUsability.cs new file mode 100644 index 00000000..645cc0a0 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/BookUsability.cs @@ -0,0 +1,13 @@ +namespace SIL.Machine.QualityEstimation +{ + public class BookUsability : UsabilityBase + { + public BookUsability(string book, UsabilityLabel label, double projectedChrF3, double usability) + : base(label, projectedChrF3, usability) + { + Book = book; + } + + public string Book { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/ChapterScores.cs b/src/SIL.Machine/QualityEstimation/ChapterScores.cs new file mode 100644 index 00000000..0f76a804 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/ChapterScores.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; + +namespace SIL.Machine.QualityEstimation +{ + internal class ChapterScores + { + private readonly Dictionary>> _verseUsabilities = + new Dictionary>>(); + + public readonly Dictionary> Scores = + new Dictionary>(); + + public void AddScore(string book, int chapter, Score score) + { + if (!Scores.TryGetValue(book, out Dictionary chapters)) + { + chapters = new Dictionary(); + Scores[book] = chapters; + } + + chapters[chapter] = score; + } + + public Score GetScore(string book, int chapter) => + Scores.TryGetValue(book, out Dictionary chapters) + && chapters.TryGetValue(chapter, out Score score) + ? score + : null; + + public void AppendVerseUsability(string book, int chapter, double usability) + { + if (!_verseUsabilities.TryGetValue(book, out Dictionary> chapters)) + { + chapters = new Dictionary>(); + _verseUsabilities[book] = chapters; + } + + if (!chapters.TryGetValue(chapter, out List list)) + { + list = new List(); + chapters[chapter] = list; + } + + list.Add(usability); + } + + public List GetVerseUsabilities(string book, int chapter) => + _verseUsabilities.TryGetValue(book, out Dictionary> chapters) + && chapters.TryGetValue(chapter, out List list) + ? new List(list) + : new List(); + } +} diff --git a/src/SIL.Machine/QualityEstimation/ChapterUsability.cs b/src/SIL.Machine/QualityEstimation/ChapterUsability.cs new file mode 100644 index 00000000..012f2a15 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/ChapterUsability.cs @@ -0,0 +1,13 @@ +namespace SIL.Machine.QualityEstimation +{ + public class ChapterUsability : BookUsability + { + public ChapterUsability(string book, int chapter, UsabilityLabel label, double projectedChrF3, double usability) + : base(book, label, projectedChrF3, usability) + { + Chapter = chapter; + } + + public int Chapter { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/ChrF3QualityEstimation.cs b/src/SIL.Machine/QualityEstimation/ChrF3QualityEstimation.cs new file mode 100644 index 00000000..e3f7d7b5 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/ChrF3QualityEstimation.cs @@ -0,0 +1,319 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Corpora; + +namespace SIL.Machine.QualityEstimation +{ + /// + /// Provides chrF3 quality estimation support for pre-translations. + /// + public class ChrF3QualityEstimation + { + private readonly BookScores _bookScores = new BookScores(); + private readonly ChapterScores _chapterScores = new ChapterScores(); + private readonly double _intercept; + private readonly List _sequenceScores = new List(); + private readonly double _slope; + private readonly TxtFileScores _txtFileScores = new TxtFileScores(); + private readonly List _verseScores = new List(); + + public ChrF3QualityEstimation(double slope, double intercept) + { + _slope = slope; + _intercept = intercept; + } + + /// + /// The threshold values used to calculate the usability label for every book. + /// + public Thresholds BookThresholds { get; set; } = new Thresholds(greenThreshold: 0.745, yellowThreshold: 0.62); + + /// + /// The threshold values used to calculate the usability label for every chapter. + /// + public Thresholds ChapterThresholds { get; set; } = + new Thresholds(greenThreshold: 0.745, yellowThreshold: 0.62); + + /// + /// The threshold values used to calculate the usability label for every verse. + /// + public Thresholds VerseThresholds { get; set; } = new Thresholds(greenThreshold: 0.745, yellowThreshold: 0.62); + + /// + /// The usable parameters to calculate the usable probabilities. + /// + public UsabilityParameters Usable { get; set; } = UsabilityParameters.Usable; + + /// + /// The unusable parameters to calculate the usable probabilities. + /// + public UsabilityParameters Unusable { get; set; } = UsabilityParameters.Unusable; + + /// + /// Estimate the quality of the pre-translations from text files. + /// + /// The confidence values. + /// The usability scores for every line in the text files, and for the text files. + public (List usabilitySequences, List usabilityTxtFiles) EstimateQuality( + IEnumerable<(MultiKeyRef key, double confidence)> confidences + ) + { + ProjectChrF3(confidences); + return ComputeSequenceUsability(); + } + + /// + /// Estimate the quality of the pre-translations from USFM files. + /// + /// The confidence values. + /// The usability scores for every verse, chapter, and book. + public ( + List usabilityVerses, + List usabilityChapters, + List usabilityBooks + ) EstimateQuality(IEnumerable<(ScriptureRef key, double confidence)> confidences) + { + ProjectChrF3(confidences); + return ComputeVerseUsability(); + } + + /// + /// Calculates the geometric mean for a collection of values. + /// + /// + /// The geometric mean. + private static double GeometricMean(IList values) + { + // Geometric mean requires positive values + if (values == null || !values.Any() || values.Any(x => x <= 0)) + return 0; + + // Compute the sum of the natural logarithms of all values, + // and divide by the count of numbers and take the exponential + return Math.Exp(values.Sum(Math.Log) / values.Count); + } + + private double CalculateUsableProbability(double chrF3) + { + double usableWeight = Math.Exp(-Math.Pow(chrF3 - Usable.Mean, 2) / (2 * Usable.Variance)) * Usable.Count; + double unusableWeight = + Math.Exp(-Math.Pow(chrF3 - Unusable.Mean, 2) / (2 * Unusable.Variance)) * Unusable.Count; + return usableWeight / (usableWeight + unusableWeight); + } + + private List ComputeBookUsability() + { + var usabilityBooks = new List(); + foreach (string book in _bookScores.Scores.Keys) + { + Score score = _bookScores.GetScore(book); + if (score is null) + continue; + + List bookUsabilities = _bookScores.GetVerseUsabilities(book); + double averageProbability = bookUsabilities.Average(); + usabilityBooks.Add( + new BookUsability( + book, + label: BookThresholds.ReturnLabel(averageProbability), + usability: averageProbability, + projectedChrF3: score.ProjectedChrF3 + ) + ); + } + + return usabilityBooks; + } + + private List ComputeChapterUsability() + { + var usabilityChapters = new List(); + foreach (KeyValuePair> chapterScoresByBook in _chapterScores.Scores) + { + string book = chapterScoresByBook.Key; + foreach (int chapter in chapterScoresByBook.Value.Keys) + { + Score score = _chapterScores.GetScore(book, chapter); + if (score is null) + continue; + + List chapterUsabilities = _chapterScores.GetVerseUsabilities(book, chapter); + double averageProbability = chapterUsabilities.Average(); + usabilityChapters.Add( + new ChapterUsability( + book, + chapter, + label: ChapterThresholds.ReturnLabel(averageProbability), + usability: averageProbability, + projectedChrF3: score.ProjectedChrF3 + ) + ); + } + } + + return usabilityChapters; + } + + private (List, List) ComputeSequenceUsability() + { + var usabilitySequences = new List(); + foreach (SequenceScore sequenceScore in _sequenceScores) + { + double probability = CalculateUsableProbability(sequenceScore.ProjectedChrF3); + _txtFileScores.AppendSequenceUsability(sequenceScore.TargetDraftFileStem, probability); + usabilitySequences.Add( + new SequenceUsability( + targetDraftFile: sequenceScore.TargetDraftFileStem, + sequenceNumber: sequenceScore.SequenceNumber, + label: VerseThresholds.ReturnLabel(probability), + usability: probability, + projectedChrF3: sequenceScore.ProjectedChrF3 + ) + ); + } + + return (usabilitySequences, ComputeTxtFileUsability()); + } + + private List ComputeTxtFileUsability() + { + var usabilityTxtFiles = new List(); + foreach (string targetDraftFileStem in _txtFileScores.Scores.Keys) + { + Score score = _txtFileScores.GetScore(targetDraftFileStem); + if (score is null) + continue; + + List txtFileUsabilities = _txtFileScores.GetSequenceUsabilities(targetDraftFileStem); + double averageProbability = txtFileUsabilities.Average(); + usabilityTxtFiles.Add( + new TxtFileUsability( + targetDraftFileStem, + label: BookThresholds.ReturnLabel(averageProbability), + usability: averageProbability, + projectedChrF3: score.ProjectedChrF3 + ) + ); + } + + return usabilityTxtFiles; + } + + private (List, List, List) ComputeVerseUsability() + { + var usabilityVerses = new List(); + foreach (VerseScore verseScore in _verseScores.Where(v => v.ScriptureRef.VerseNum > 0)) + { + double probability = CalculateUsableProbability(verseScore.ProjectedChrF3); + _chapterScores.AppendVerseUsability( + verseScore.ScriptureRef.Book, + verseScore.ScriptureRef.ChapterNum, + probability + ); + _bookScores.AppendVerseUsability(verseScore.ScriptureRef.Book, probability); + usabilityVerses.Add( + new VerseUsability( + book: verseScore.ScriptureRef.Book, + chapter: verseScore.ScriptureRef.ChapterNum, + verse: verseScore.ScriptureRef.Verse, + label: VerseThresholds.ReturnLabel(probability), + usability: probability, + projectedChrF3: verseScore.ProjectedChrF3 + ) + ); + } + + return (usabilityVerses, ComputeChapterUsability(), ComputeBookUsability()); + } + + private void ProjectChrF3(IEnumerable<(MultiKeyRef, double)> confidences) + { + var confidencesByTxtFile = new Dictionary>(); + foreach ((MultiKeyRef key, double confidence) in confidences) + { + if (key.Keys.Count >= 0 && int.TryParse(key.Keys[0].ToString(), out int sequenceNumber)) + { + string targetDraftFileStem = key.TextId; + var score = new SequenceScore(_slope, confidence, _intercept, sequenceNumber, targetDraftFileStem); + _sequenceScores.Add(score); + + // Record the confidence by text file + if (!confidencesByTxtFile.TryGetValue(targetDraftFileStem, out List txtFileConfidences)) + { + txtFileConfidences = new List(); + confidencesByTxtFile[targetDraftFileStem] = txtFileConfidences; + } + + txtFileConfidences.Add(confidence); + } + } + + foreach (KeyValuePair> txtFileConfidences in confidencesByTxtFile) + { + _txtFileScores.AddScore( + txtFileConfidences.Key, + new Score(_slope, confidence: GeometricMean(txtFileConfidences.Value), _intercept) + ); + } + } + + private void ProjectChrF3(IEnumerable<(ScriptureRef, double)> confidences) + { + var confidencesByBook = new Dictionary>(); + var confidencesByBookAndChapter = new Dictionary<(string, int), List>(); + foreach ((ScriptureRef key, double confidence) in confidences) + { + var score = new VerseScore(_slope, confidence, _intercept, key); + _verseScores.Add(score); + string book = key.Book; + int chapter = key.ChapterNum; + + // Record the confidence by and chapter + if ( + !confidencesByBookAndChapter.TryGetValue( + (book, chapter), + out List bookAndChapterConfidences + ) + ) + { + bookAndChapterConfidences = new List(); + confidencesByBookAndChapter[(book, chapter)] = bookAndChapterConfidences; + } + + bookAndChapterConfidences.Add(confidence); + + // Record the confidence by book + if (!confidencesByBook.TryGetValue(book, out List bookConfidences)) + { + bookConfidences = new List(); + confidencesByBook[book] = bookConfidences; + } + + bookConfidences.Add(confidence); + } + + foreach (KeyValuePair> bookConfidences in confidencesByBook) + { + _bookScores.AddScore( + bookConfidences.Key, + new Score(_slope, confidence: GeometricMean(bookConfidences.Value), _intercept) + ); + } + + foreach ( + KeyValuePair< + (string Book, int Chapter), + List + > bookAndChapterConfidences in confidencesByBookAndChapter + ) + { + _chapterScores.AddScore( + bookAndChapterConfidences.Key.Book, + bookAndChapterConfidences.Key.Chapter, + new Score(_slope, confidence: GeometricMean(bookAndChapterConfidences.Value), _intercept) + ); + } + } + } +} diff --git a/src/SIL.Machine/QualityEstimation/Score.cs b/src/SIL.Machine/QualityEstimation/Score.cs new file mode 100644 index 00000000..ee98dd6a --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/Score.cs @@ -0,0 +1,15 @@ +namespace SIL.Machine.QualityEstimation +{ + internal class Score + { + public Score(double slope, double confidence, double intercept) + { + Confidence = confidence; + ProjectedChrF3 = slope * confidence + intercept; + } + + public double Confidence { get; } + + public double ProjectedChrF3 { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/SequenceScore.cs b/src/SIL.Machine/QualityEstimation/SequenceScore.cs new file mode 100644 index 00000000..d85821b3 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/SequenceScore.cs @@ -0,0 +1,21 @@ +namespace SIL.Machine.QualityEstimation +{ + internal class SequenceScore : Score + { + public SequenceScore( + double slope, + double confidence, + double intercept, + int sequenceNumber, + string targetDraftFileStem + ) + : base(slope, confidence, intercept) + { + SequenceNumber = sequenceNumber; + TargetDraftFileStem = targetDraftFileStem; + } + + public int SequenceNumber { get; } + public string TargetDraftFileStem { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/SequenceUsability.cs b/src/SIL.Machine/QualityEstimation/SequenceUsability.cs new file mode 100644 index 00000000..828dc266 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/SequenceUsability.cs @@ -0,0 +1,19 @@ +namespace SIL.Machine.QualityEstimation +{ + public class SequenceUsability : TxtFileUsability + { + public SequenceUsability( + string targetDraftFile, + int sequenceNumber, + UsabilityLabel label, + double projectedChrF3, + double usability + ) + : base(targetDraftFile, label, projectedChrF3, usability) + { + SequenceNumber = sequenceNumber; + } + + public int SequenceNumber { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/Thresholds.cs b/src/SIL.Machine/QualityEstimation/Thresholds.cs new file mode 100644 index 00000000..095c9178 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/Thresholds.cs @@ -0,0 +1,20 @@ +namespace SIL.Machine.QualityEstimation +{ + public class Thresholds + { + public Thresholds(double greenThreshold, double yellowThreshold) + { + GreenThreshold = greenThreshold; + YellowThreshold = yellowThreshold; + } + + public double GreenThreshold { get; } + + public double YellowThreshold { get; } + + public UsabilityLabel ReturnLabel(double probability) => + probability >= GreenThreshold ? UsabilityLabel.Green + : probability >= YellowThreshold ? UsabilityLabel.Yellow + : UsabilityLabel.Red; + } +} diff --git a/src/SIL.Machine/QualityEstimation/TxtFileScores.cs b/src/SIL.Machine/QualityEstimation/TxtFileScores.cs new file mode 100644 index 00000000..8f5e563e --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/TxtFileScores.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; + +namespace SIL.Machine.QualityEstimation +{ + internal class TxtFileScores + { + private readonly Dictionary> _sequenceUsabilities = new Dictionary>(); + + public readonly Dictionary Scores = new Dictionary(); + + public void AddScore(string targetDraftFileStem, Score score) => Scores[targetDraftFileStem] = score; + + public Score GetScore(string targetDraftFileStem) => + Scores.TryGetValue(targetDraftFileStem, out Score score) ? score : null; + + public void AppendSequenceUsability(string targetDraftFileStem, double usability) + { + if (!_sequenceUsabilities.TryGetValue(targetDraftFileStem, out List list)) + { + list = new List(); + _sequenceUsabilities[targetDraftFileStem] = list; + } + + list.Add(usability); + } + + public List GetSequenceUsabilities(string targetDraftFileStem) => + _sequenceUsabilities.TryGetValue(targetDraftFileStem, out List list) + ? new List(list) + : new List(); + } +} diff --git a/src/SIL.Machine/QualityEstimation/TxtFileUsability.cs b/src/SIL.Machine/QualityEstimation/TxtFileUsability.cs new file mode 100644 index 00000000..0c23a325 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/TxtFileUsability.cs @@ -0,0 +1,13 @@ +namespace SIL.Machine.QualityEstimation +{ + public class TxtFileUsability : UsabilityBase + { + public TxtFileUsability(string targetDraftFile, UsabilityLabel label, double projectedChrF3, double usability) + : base(label, projectedChrF3, usability) + { + TargetDraftFile = targetDraftFile; + } + + public string TargetDraftFile { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/UsabilityBase.cs b/src/SIL.Machine/QualityEstimation/UsabilityBase.cs new file mode 100644 index 00000000..da7ae05c --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/UsabilityBase.cs @@ -0,0 +1,18 @@ +namespace SIL.Machine.QualityEstimation +{ + public abstract class UsabilityBase + { + protected UsabilityBase(UsabilityLabel label, double projectedChrF3, double usability) + { + Label = label; + ProjectedChrF3 = projectedChrF3; + Usability = usability; + } + + public UsabilityLabel Label { get; } + + public double ProjectedChrF3 { get; } + + public double Usability { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/UsabilityLabel.cs b/src/SIL.Machine/QualityEstimation/UsabilityLabel.cs new file mode 100644 index 00000000..0b207384 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/UsabilityLabel.cs @@ -0,0 +1,9 @@ +namespace SIL.Machine.QualityEstimation +{ + public enum UsabilityLabel + { + Red, + Yellow, + Green, + } +} diff --git a/src/SIL.Machine/QualityEstimation/UsabilityParameters.cs b/src/SIL.Machine/QualityEstimation/UsabilityParameters.cs new file mode 100644 index 00000000..0181af76 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/UsabilityParameters.cs @@ -0,0 +1,30 @@ +namespace SIL.Machine.QualityEstimation +{ + public class UsabilityParameters + { + public static readonly UsabilityParameters Unusable = new UsabilityParameters( + count: 97, + mean: 45.85, + variance: 99.91 + ); + + public static readonly UsabilityParameters Usable = new UsabilityParameters( + count: 263, + mean: 51.4, + variance: 95.19 + ); + + public UsabilityParameters(double count, double mean, double variance) + { + Count = count; + Mean = mean; + Variance = variance; + } + + public double Count { get; } + + public double Mean { get; } + + public double Variance { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/VerseScore.cs b/src/SIL.Machine/QualityEstimation/VerseScore.cs new file mode 100644 index 00000000..baa5d0a0 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/VerseScore.cs @@ -0,0 +1,15 @@ +using SIL.Machine.Corpora; + +namespace SIL.Machine.QualityEstimation +{ + internal class VerseScore : Score + { + public VerseScore(double slope, double confidence, double intercept, ScriptureRef scriptureRef) + : base(slope, confidence, intercept) + { + ScriptureRef = scriptureRef; + } + + public ScriptureRef ScriptureRef { get; } + } +} diff --git a/src/SIL.Machine/QualityEstimation/VerseUsability.cs b/src/SIL.Machine/QualityEstimation/VerseUsability.cs new file mode 100644 index 00000000..77c9c271 --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/VerseUsability.cs @@ -0,0 +1,20 @@ +namespace SIL.Machine.QualityEstimation +{ + public class VerseUsability : ChapterUsability + { + public VerseUsability( + string book, + int chapter, + string verse, + UsabilityLabel label, + double projectedChrF3, + double usability + ) + : base(book, chapter, label, projectedChrF3, usability) + { + Verse = verse; + } + + public string Verse { get; } + } +} diff --git a/tests/SIL.Machine.Tests/QualityEstimation/ChrF3QualityEstimationTests.cs b/tests/SIL.Machine.Tests/QualityEstimation/ChrF3QualityEstimationTests.cs new file mode 100644 index 00000000..da09a074 --- /dev/null +++ b/tests/SIL.Machine.Tests/QualityEstimation/ChrF3QualityEstimationTests.cs @@ -0,0 +1,62 @@ +using NUnit.Framework; +using SIL.Machine.Corpora; +using SIL.Scripture; + +namespace SIL.Machine.QualityEstimation; + +[TestFixture] +public class ChrF3QualityEstimationTests +{ + [Test] + public void ChrF3QualityEstimation_TxtFiles() + { + var qualityEstimation = new ChrF3QualityEstimation(slope: 0.6, intercept: 1.0); + List<(MultiKeyRef Key, double Confidence)> confidences = + [ + (new MultiKeyRef("MAT.txt", 1), 85.0), + (new MultiKeyRef("MAT.txt", 2), 80.0), + (new MultiKeyRef("MRK.txt", 1), 60.0), + ]; + (List usabilitySequences, List usabilityTxtFiles) = + qualityEstimation.EstimateQuality(confidences); + using (Assert.EnterMultipleScope()) + { + Assert.That(usabilitySequences, Has.Count.EqualTo(3)); + Assert.That(usabilitySequences[0].Label, Is.EqualTo(UsabilityLabel.Green)); + Assert.That(usabilitySequences[1].Label, Is.EqualTo(UsabilityLabel.Yellow)); + Assert.That(usabilitySequences[2].Label, Is.EqualTo(UsabilityLabel.Red)); + Assert.That(usabilityTxtFiles, Has.Count.EqualTo(2)); + Assert.That(usabilityTxtFiles[0].Label, Is.EqualTo(UsabilityLabel.Green)); + Assert.That(usabilityTxtFiles[1].Label, Is.EqualTo(UsabilityLabel.Red)); + } + } + + [Test] + public void ChrF3QualityEstimation_Verses() + { + var qualityEstimation = new ChrF3QualityEstimation(slope: 0.6, intercept: 1.0); + List<(ScriptureRef key, double confidence)> confidences = + [ + (new ScriptureRef(new VerseRef(1, 1, 1)), 85.0), + (new ScriptureRef(new VerseRef(1, 1, 2)), 80.0), + (new ScriptureRef(new VerseRef(1, 2, 1)), 60.0), + ]; + ( + List usabilityVerses, + List usabilityChapters, + List usabilityBooks + ) = qualityEstimation.EstimateQuality(confidences); + using (Assert.EnterMultipleScope()) + { + Assert.That(usabilityVerses, Has.Count.EqualTo(3)); + Assert.That(usabilityVerses[0].Label, Is.EqualTo(UsabilityLabel.Green)); + Assert.That(usabilityVerses[1].Label, Is.EqualTo(UsabilityLabel.Yellow)); + Assert.That(usabilityVerses[2].Label, Is.EqualTo(UsabilityLabel.Red)); + Assert.That(usabilityChapters, Has.Count.EqualTo(2)); + Assert.That(usabilityChapters[0].Label, Is.EqualTo(UsabilityLabel.Green)); + Assert.That(usabilityChapters[1].Label, Is.EqualTo(UsabilityLabel.Red)); + Assert.That(usabilityBooks, Has.Count.EqualTo(1)); + Assert.That(usabilityBooks[0].Label, Is.EqualTo(UsabilityLabel.Yellow)); + } + } +}