Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 70 additions & 7 deletions src/TextMateSharp.Benchmarks/BigFileTokenizationBenchmark.cs
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
using Perfolizer.Metrology;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;

using BenchmarkDotNet.Attributes;

using TextMateSharp.Grammars;

namespace TextMateSharp.Benchmarks
{
[Config(typeof(CustomBenchmarksConfig))]
[MemoryDiagnoser]
public class BigFileTokenizationBenchmark
{
Expand All @@ -20,10 +25,10 @@ public void Setup()
// Walk up directories to find the solution root
string? dir = AppDomain.CurrentDomain.BaseDirectory;
string bigFilePath = "";

while (dir != null)
{
string candidate = Path.Combine(dir, "src", "TextMateSharp.Demo",
string candidate = Path.Combine(dir, "src", "TextMateSharp.Demo",
"testdata", "samplefiles", "bigfile.cs");
if (File.Exists(candidate))
{
Expand All @@ -42,13 +47,13 @@ public void Setup()

// Load the file into memory
_content = File.ReadAllText(bigFilePath);
Console.WriteLine($"Loaded bigfile.cs");
Console.WriteLine("Loaded bigfile.cs");

// Load the C# grammar
RegistryOptions options = new RegistryOptions(ThemeName.DarkPlus);
Registry.Registry registry = new Registry.Registry(options);
_grammar = registry.LoadGrammar("source.cs");

if (_grammar == null)
{
throw new InvalidOperationException("Failed to load C# grammar");
Expand Down Expand Up @@ -94,5 +99,63 @@ public int TokenizeAllLines()
yield return (lineStart, content.Length - lineStart);
}
}

#region helper classes for benchmarks

public sealed class CustomBenchmarksConfig : ManualConfig
{
public CustomBenchmarksConfig()
{
// Use the default summary style with size unit in kilobytes.
// We have a separate column to measure in bytes so we can measure even small differences in memory usage.
SummaryStyle = SummaryStyle.Default
.WithSizeUnit(SizeUnit.KB)
.WithCultureInfo(CultureInfo.CurrentCulture);

AddColumn(new AllocatedBytesColumn());
}
}

public sealed class AllocatedBytesColumn : IColumn
{
public string Id => nameof(AllocatedBytesColumn);

public string ColumnName => "Allocated B";

public bool AlwaysShow => true;

public ColumnCategory Category => ColumnCategory.Custom;

public int PriorityInCategory => 0;

public bool IsNumeric => true;

public UnitType UnitType => UnitType.Dimensionless;

public string Legend => "Bytes allocated per operation";

public bool IsAvailable(Summary summary) => true;

public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false;

public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
BenchmarkReport? report = summary[benchmarkCase];
long? bytesAllocatedPerOperation = report?.GcStats.GetBytesAllocatedPerOperation(benchmarkCase);
if (!bytesAllocatedPerOperation.HasValue)
{
return "NA";
}

return bytesAllocatedPerOperation.Value.ToString("N0", style.CultureInfo);
}

public string GetValue(Summary summary, BenchmarkCase benchmarkCase)
=> GetValue(summary, benchmarkCase, summary.Style);

public override string ToString() => ColumnName;
}

#endregion helper classes for benchmarks
}
}
4 changes: 2 additions & 2 deletions src/TextMateSharp.Grammars/GrammarNames.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
namespace TextMateSharp.Grammars
{
internal class GrammarNames
internal static class GrammarNames
{
internal static string[] SupportedGrammars = new string[] {
internal static readonly string[] SupportedGrammars = new string[] {
"Asciidoc",
"Bat",
"Clojure",
Expand Down
3 changes: 2 additions & 1 deletion src/TextMateSharp.Grammars/Properties/AssemblyInfo.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using System.Runtime.CompilerServices;

[assembly: InternalsVisibleTo("TextMateSharp.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001000db16f8de24159e7ee94e32addce2a9b60f3ea5be200ae7b5abbf8676705064a1b5a5a44d570a884bd86bd2d3e83411fb88914e00028bc7d4b5be1ba8fd8db4335e3ad911d0ef7e694cf433f3314e991100c72c7473641a9e3437deeab402c8f4a03fdf9c174cbae00142a28ce43475ca61f0016ede73dc778b5ed5a0344cfc2")]
[assembly: InternalsVisibleTo("TextMateSharp.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001000db16f8de24159e7ee94e32addce2a9b60f3ea5be200ae7b5abbf8676705064a1b5a5a44d570a884bd86bd2d3e83411fb88914e00028bc7d4b5be1ba8fd8db4335e3ad911d0ef7e694cf433f3314e991100c72c7473641a9e3437deeab402c8f4a03fdf9c174cbae00142a28ce43475ca61f0016ede73dc778b5ed5a0344cfc2")]
[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]
11 changes: 5 additions & 6 deletions src/TextMateSharp/Internal/Grammars/AttributedScopeStack.cs
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
using System;
using System.Collections.Generic;

using TextMateSharp.Themes;

namespace TextMateSharp.Internal.Grammars
{
public class AttributedScopeStack : IEquatable<AttributedScopeStack>
internal sealed class AttributedScopeStack : IEquatable<AttributedScopeStack>
{
public AttributedScopeStack Parent { get; private set; }
public string ScopePath { get; private set; }
public int TokenAttributes { get; private set; }
internal AttributedScopeStack Parent { get; private set; }
internal string ScopePath { get; private set; }
internal int TokenAttributes { get; private set; }
private List<string> _cachedScopeNames;

// Precomputed, per-node hash code (persistent structure => safe as long as instances are immutable)
private readonly int _hashCode;

public AttributedScopeStack(AttributedScopeStack parent, string scopePath, int tokenAttributes)
internal AttributedScopeStack(AttributedScopeStack parent, string scopePath, int tokenAttributes)
{
Parent = parent;
ScopePath = scopePath;
Expand Down
12 changes: 6 additions & 6 deletions src/TextMateSharp/Internal/Grammars/BalancedBracketSelectors.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@

namespace TextMateSharp.Internal.Grammars
{
public class BalancedBracketSelectors
public sealed class BalancedBracketSelectors
{
private Predicate<List<string>>[] _balancedBracketScopes;
private Predicate<List<string>>[] _unbalancedBracketScopes;
private readonly Predicate<List<string>>[] _balancedBracketScopes;
private readonly Predicate<List<string>>[] _unbalancedBracketScopes;

private bool _allowAny = false;

Expand All @@ -18,17 +18,17 @@ public BalancedBracketSelectors(
_unbalancedBracketScopes = CreateUnbalancedBracketScopes(unbalancedBracketScopes);
}

public bool MatchesAlways()
internal bool MatchesAlways()
{
return _allowAny && _unbalancedBracketScopes.Length == 0;
}

public bool MatchesNever()
internal bool MatchesNever()
{
return !_allowAny && _balancedBracketScopes.Length == 0;
}

public bool Match(List<string> scopes)
internal bool Match(List<string> scopes)
{
foreach (var excluder in _unbalancedBracketScopes)
{
Expand Down
10 changes: 5 additions & 5 deletions src/TextMateSharp/Internal/Grammars/BasicScopeAttributes.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@

namespace TextMateSharp.Internal.Grammars
{
public class BasicScopeAttributes
internal sealed class BasicScopeAttributes
{
public int LanguageId { get; private set; }
public int TokenType { get; private set; } /* OptionalStandardTokenType */
public List<ThemeTrieElementRule> ThemeData { get; private set; }
internal int LanguageId { get; private set; }
internal int TokenType { get; private set; } /* OptionalStandardTokenType */
internal List<ThemeTrieElementRule> ThemeData { get; private set; }

public BasicScopeAttributes(
internal BasicScopeAttributes(
int languageId,
int tokenType,
List<ThemeTrieElementRule> themeData)
Expand Down
20 changes: 10 additions & 10 deletions src/TextMateSharp/Internal/Grammars/BasicScopeAttributesProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,21 @@

namespace TextMateSharp.Internal.Grammars
{
public class BasicScopeAttributesProvider
internal sealed class BasicScopeAttributesProvider
{

private static BasicScopeAttributes _NULL_SCOPE_METADATA = new BasicScopeAttributes(0, 0, null);

private static Regex STANDARD_TOKEN_TYPE_REGEXP = new Regex("\\b(comment|string|regex|meta\\.embedded)\\b");

private int _initialLanguage;
private IThemeProvider _themeProvider;
private Dictionary<string, BasicScopeAttributes> _cache = new Dictionary<string, BasicScopeAttributes>();
private readonly int _initialLanguage;
private readonly IThemeProvider _themeProvider;
private readonly Dictionary<string, BasicScopeAttributes> _cache = new Dictionary<string, BasicScopeAttributes>();
private BasicScopeAttributes _defaultAttributes;
private Dictionary<string, int> _embeddedLanguages;
private Regex _embeddedLanguagesRegex;
private readonly Dictionary<string, int> _embeddedLanguages;
private readonly Regex _embeddedLanguagesRegex;

public BasicScopeAttributesProvider(int initialLanguage, IThemeProvider themeProvider,
internal BasicScopeAttributesProvider(int initialLanguage, IThemeProvider themeProvider,
Dictionary<string, int> embeddedLanguages)
{
this._initialLanguage = initialLanguage;
Expand Down Expand Up @@ -58,7 +58,7 @@ public BasicScopeAttributesProvider(int initialLanguage, IThemeProvider themePro
reversedScopes.Reverse();
this._embeddedLanguagesRegex = new Regex(
"^((" +
string.Join(")|(", escapedScopes) +
string.Join(")|(", reversedScopes) +
"))($|\\.)");
}
}
Expand All @@ -72,12 +72,12 @@ public void OnDidChangeTheme()
new List<ThemeTrieElementRule>() { this._themeProvider.GetDefaults() });
}

public BasicScopeAttributes GetDefaultAttributes()
internal BasicScopeAttributes GetDefaultAttributes()
{
return this._defaultAttributes;
}

public BasicScopeAttributes GetBasicScopeAttributes(string scopeName)
internal BasicScopeAttributes GetBasicScopeAttributes(string scopeName)
{
if (scopeName == null)
{
Expand Down
2 changes: 1 addition & 1 deletion src/TextMateSharp/Internal/Grammars/LineTokenizer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ private void ScanNext()
if (beginWhileRule.WhileHasBackReferences)
{
_stack = _stack.WithEndRule(
beginWhileRule.getWhileWithResolvedBackReferences(_lineText, captureIndices));
beginWhileRule.GetWhileWithResolvedBackReferences(_lineText, captureIndices));
}

if (!hasAdvanced && beforePush.HasSameRuleAs(_stack))
Expand Down
2 changes: 1 addition & 1 deletion src/TextMateSharp/Internal/Matcher/IMatchesName.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public interface IMatchesName<T>
bool Match(ICollection<string> names, T scopes);
}

public class NameMatcher : IMatchesName<List<string>>
public sealed class NameMatcher : IMatchesName<List<string>>
{
public static IMatchesName<List<string>> Default = new NameMatcher();

Expand Down
25 changes: 9 additions & 16 deletions src/TextMateSharp/Internal/Parser/PList.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
using System;
using System.Collections.Generic;
using System.Text;

Expand Down Expand Up @@ -55,15 +54,15 @@ private PListObject Create(PListObject parent, bool valueAsArray)
public void EndElement(string tagName)
{
object value = null;
string text = this.text.ToString();
string t = this.text.ToString();
if ("key".Equals(tagName))
{
if (currObject == null || currObject.IsValueAsArray())
{
errors.Add("key can only be used inside an open dict element");
return;
}
currObject.SetLastKey(text);
currObject.SetLastKey(t);
return;
}
else if ("dict".Equals(tagName) || "array".Equals(tagName))
Expand All @@ -78,35 +77,29 @@ public void EndElement(string tagName)
}
else if ("string".Equals(tagName) || "data".Equals(tagName))
{
value = text;
value = t;
}
else if ("date".Equals(tagName))
{
// TODO : parse date
}
else if ("integer".Equals(tagName))
{
try
if (!int.TryParse(t, out int i))
{
value = int.Parse(text);
}
catch (Exception)
{
errors.Add(text + " is not a integer");
errors.Add(t + " is not an integer");
return;
}
value = i;
}
else if ("real".Equals(tagName))
{
try
{
value = float.Parse(text);
}
catch (Exception)
if (!float.TryParse(t, out float f))
{
errors.Add(text + " is not a float");
errors.Add(t + " is not a float");
return;
}
value = f;
Comment on lines 95 to +102
}
else if ("true".Equals(tagName))
{
Expand Down
16 changes: 8 additions & 8 deletions src/TextMateSharp/Internal/Rules/BeginEndRule.cs
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
using Onigwrap;
using System;
using System.Collections.Generic;
using Onigwrap;

namespace TextMateSharp.Internal.Rules
{
public class BeginEndRule : Rule
public sealed class BeginEndRule : Rule
{
public List<CaptureRule> BeginCaptures { get; private set; }
public bool EndHasBackReferences { get; private set; }
public List<CaptureRule> EndCaptures { get; private set; }
public bool ApplyEndPatternLast { get; private set; }
public bool HasMissingPatterns { get; private set; }
public IList<RuleId> Patterns { get; private set; }
private bool ApplyEndPatternLast { get; }
internal bool HasMissingPatterns { get; private set; }
internal IList<RuleId> Patterns { get; private set; }

private RegExpSource _begin;
private RegExpSource _end;
private readonly RegExpSource _begin;
private readonly RegExpSource _end;
private RegExpSourceList _cachedCompiledPatterns;

public BeginEndRule(RuleId id, string name, string contentName, string begin, List<CaptureRule> beginCaptures,
internal BeginEndRule(RuleId id, string name, string contentName, string begin, List<CaptureRule> beginCaptures,
string end, List<CaptureRule> endCaptures, bool applyEndPatternLast, CompilePatternsResult patterns)
: base(id, name, contentName)
{
Expand Down
Loading
Loading