support regexp lookaround assertions in syntax files

go.mod

@@ -24,6 +24,7 @@ require (
 require (
 	github.com/creack/pty v1.1.18 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dlclark/regexp2 v1.11.5 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
+github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=

pkg/highlight/highlight_test.go

@@ -0,0 +1,229 @@
+package highlight
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// resetGroups clears the global group registry so tests don't interfere with each other.
+func resetGroups() {
+	Groups = make(map[string]Group)
+	numGroups = 0
+}
+
+// makeHighlighter parses a YAML syntax definition and returns a Highlighter.
+func makeHighlighter(t *testing.T, yaml string) *Highlighter {
+	t.Helper()
+	data := []byte(yaml)
+
+	header, err := MakeHeaderYaml(data)
+	if err != nil {
+		t.Fatalf("MakeHeaderYaml: %v", err)
+	}
+
+	f, err := ParseFile(data)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+
+	def, err := ParseDef(f, header)
+	if err != nil {
+		t.Fatalf("ParseDef: %v", err)
+	}
+
+	return NewHighlighter(def)
+}
+
+// groupAt returns the Group name at a given character position in a LineMatch.
+// It walks backwards from pos to find the most recent color change.
+func groupAt(lm LineMatch, pos int) string {
+	best := -1
+	for k := range lm {
+		if k <= pos && k > best {
+			best = k
+		}
+	}
+	if best < 0 {
+		return ""
+	}
+	return lm[best].String()
+}
+
+func TestLookahead(t *testing.T) {
+	resetGroups()
+	assert := assert.New(t)
+
+	h := makeHighlighter(t, `
+filetype: test-lookahead
+detect:
+    filename: "\\.test$"
+rules:
+    - identifier.function: "\\w+(?=\\()"
+`)
+
+	matches := h.HighlightString("foo(bar)")
+	assert.Len(matches, 1)
+
+	lm := matches[0]
+
+	// "foo" (positions 0-2) should be highlighted as identifier.function
+	assert.Equal("identifier.function", groupAt(lm, 0))
+	assert.Equal("identifier.function", groupAt(lm, 1))
+	assert.Equal("identifier.function", groupAt(lm, 2))
+
+	// "(" at position 3 should NOT be identifier.function (lookahead doesn't consume)
+	assert.NotEqual("identifier.function", groupAt(lm, 3))
+
+	// "bar" should not match (not followed by "(")
+	assert.NotEqual("identifier.function", groupAt(lm, 4))
+}
+
+func TestNegativeLookahead(t *testing.T) {
+	resetGroups()
+	assert := assert.New(t)
+
+	h := makeHighlighter(t, `
+filetype: test-neg-lookahead
+detect:
+    filename: "\\.test$"
+rules:
+    - identifier: "foo(?!bar)"
+`)
+
+	matches := h.HighlightString("foobar foobaz")
+	assert.Len(matches, 1)
+
+	lm := matches[0]
+
+	// "foobar": "foo" is followed by "bar", so negative lookahead fails — no match
+	assert.NotEqual("identifier", groupAt(lm, 0))
+
+	// "foobaz": "foo" at position 7 is NOT followed by "bar", so it matches
+	assert.Equal("identifier", groupAt(lm, 7))
+	assert.Equal("identifier", groupAt(lm, 8))
+	assert.Equal("identifier", groupAt(lm, 9))
+
+	// "baz" at position 10 should not be highlighted
+	assert.NotEqual("identifier", groupAt(lm, 10))
+}
+
+func TestLookbehind(t *testing.T) {
+	resetGroups()
+	assert := assert.New(t)
+
+	h := makeHighlighter(t, `
+filetype: test-lookbehind
+detect:
+    filename: "\\.test$"
+rules:
+    - identifier.field: "(?<=\\.)\\w+"
+`)
+
+	matches := h.HighlightString("obj.field")
+	assert.Len(matches, 1)
+
+	lm := matches[0]
+
+	// "obj" (positions 0-2) should NOT be highlighted
+	assert.NotEqual("identifier.field", groupAt(lm, 0))
+
+	// "." at position 3 should NOT be highlighted (lookbehind doesn't consume)
+	assert.NotEqual("identifier.field", groupAt(lm, 3))
+
+	// "field" (positions 4-8) should be highlighted
+	assert.Equal("identifier.field", groupAt(lm, 4))
+	assert.Equal("identifier.field", groupAt(lm, 5))
+	assert.Equal("identifier.field", groupAt(lm, 8))
+}
+
+func TestNegativeLookbehind(t *testing.T) {
+	resetGroups()
+	assert := assert.New(t)
+
+	h := makeHighlighter(t, `
+filetype: test-neg-lookbehind
+detect:
+    filename: "\\.test$"
+rules:
+    - identifier: "(?<!\\.)\\b\\w+\\b"
+`)
+
+	matches := h.HighlightString("obj.field")
+	assert.Len(matches, 1)
+
+	lm := matches[0]
+
+	// "obj" (positions 0-2) should be highlighted — not preceded by "."
+	assert.Equal("identifier", groupAt(lm, 0))
+	assert.Equal("identifier", groupAt(lm, 2))
+
+	// "field" (positions 4-8) should NOT be highlighted — preceded by "."
+	assert.NotEqual("identifier", groupAt(lm, 4))
+}
+
+func TestLookaheadInRegion(t *testing.T) {
+	resetGroups()
+	assert := assert.New(t)
+
+	h := makeHighlighter(t, `
+filetype: test-region-lookahead
+detect:
+    filename: "\\.test$"
+rules:
+    - constant.string:
+        start: "\""
+        end: "\""
+        rules:
+            - special: "\\w+(?=!)"
+`)
+
+	matches := h.HighlightString(`"hello!"`)
+	assert.Len(matches, 1)
+
+	lm := matches[0]
+
+	// Position 0 is the opening `"` — should be constant.string (the region delimiter)
+	assert.Equal("constant.string", groupAt(lm, 0))
+
+	// "hello" at positions 1-5 inside the string should get "special" via lookahead
+	assert.Equal("special", groupAt(lm, 1))
+	assert.Equal("special", groupAt(lm, 5))
+
+	// "!" at position 6 should revert to constant.string (not consumed by lookahead)
+	assert.Equal("constant.string", groupAt(lm, 6))
+}
+
+func TestBasicHighlighting(t *testing.T) {
+	resetGroups()
+	assert := assert.New(t)
+
+	h := makeHighlighter(t, `
+filetype: test-basic
+detect:
+    filename: "\\.test$"
+rules:
+    - keyword: "\\b(if|else|while)\\b"
+    - constant.number: "\\b\\d+\\b"
+`)
+
+	matches := h.HighlightString("if x 42 else")
+	assert.Len(matches, 1)
+
+	lm := matches[0]
+
+	// "if" at positions 0-1
+	assert.Equal("keyword", groupAt(lm, 0))
+	assert.Equal("keyword", groupAt(lm, 1))
+
+	// "x" at position 3 — no highlight
+	assert.Equal("", groupAt(lm, 3))
+
+	// "42" at positions 5-6
+	assert.Equal("constant.number", groupAt(lm, 5))
+	assert.Equal("constant.number", groupAt(lm, 6))
+
+	// "else" at positions 8-11
+	assert.Equal("keyword", groupAt(lm, 8))
+	assert.Equal("keyword", groupAt(lm, 11))
+}

pkg/highlight/highlighter.go

@@ -1,8 +1,10 @@
 package highlight
 
 import (
-	"regexp"
+	"log"
 	"strings"
+
+	"github.com/dlclark/regexp2"
 )
 
 func sliceStart(slc []byte, index int) []byte {
@@ -39,18 +41,6 @@ func sliceEnd(slc []byte, index int) []byte {
 	return slc[:totalSize]
 }
 
-// RunePos returns the rune index of a given byte index
-// This could cause problems if the byte index is between code points
-func runePos(p int, str []byte) int {
-	if p < 0 {
-		return 0
-	}
-	if p >= len(str) {
-		return CharacterCount(str)
-	}
-	return CharacterCount(str[:p])
-}
-
 // A State represents the region at the end of a line
 type State *region
 
@@ -82,30 +72,47 @@ func NewHighlighter(def *Def) *Highlighter {
 // color's group (represented as one byte)
 type LineMatch map[int]Group
 
-func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) []int {
-	var strbytes []byte
+func findIndex(regex *regexp2.Regexp, skip *regexp2.Regexp, str []byte) []int {
+	searchStr := string(str)
 	if skip != nil {
-		strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte {
-			res := make([]byte, CharacterCount(match))
-			return res
-		})
-	} else {
-		strbytes = str
+		replaced, err := skip.ReplaceFunc(searchStr, func(m regexp2.Match) string {
+			return strings.Repeat(" ", m.Length)
+		}, 0, -1)
+		if err != nil {
+			log.Printf("highlight: regex timeout in skip replace for pattern %q: %v", skip.String(), err)
+		} else {
+			searchStr = replaced
+		}
 	}
 
-	match := regex.FindIndex(strbytes)
-	if match == nil {
+	m, err := regex.FindStringMatch(searchStr)
+	if err != nil {
+		log.Printf("highlight: regex timeout finding match for pattern %q: %v", regex.String(), err)
+		return nil
+	}
+	if m == nil {
 		return nil
 	}
-	// return []int{match.Index, match.Index + match.Length}
-	return []int{runePos(match[0], str), runePos(match[1], str)}
+	return []int{charPosFromRunePos(m.Index, str), charPosFromRunePos(m.Index+m.Length, str)}
 }
 
-func findAllIndex(regex *regexp.Regexp, str []byte) [][]int {
-	matches := regex.FindAllIndex(str, -1)
-	for i, m := range matches {
-		matches[i][0] = runePos(m[0], str)
-		matches[i][1] = runePos(m[1], str)
+func findAllIndex(regex *regexp2.Regexp, str []byte) [][]int {
+	var matches [][]int
+	m, err := regex.FindStringMatch(string(str))
+	if err != nil {
+		log.Printf("highlight: regex timeout finding matches for pattern %q: %v", regex.String(), err)
+		return nil
+	}
+	for m != nil {
+		matches = append(matches, []int{
+			charPosFromRunePos(m.Index, str),
+			charPosFromRunePos(m.Index+m.Length, str),
+		})
+		m, err = regex.FindNextMatch(m)
+		if err != nil {
+			log.Printf("highlight: regex timeout finding next match for pattern %q: %v", regex.String(), err)
+			break
+		}
 	}
 	return matches
 }