Refactor lexer (#653)

Author: antonmedv

expr_test.go

@@ -1623,7 +1623,10 @@ func TestCompile_exposed_error(t *testing.T) {
 
 	b, err := json.Marshal(err)
 	require.NoError(t, err)
-	require.Equal(t, `{"Line":1,"Column":2,"Message":"invalid operation: == (mismatched types int and bool)","Snippet":"\n | 1 == true\n | ..^","Prev":null}`, string(b))
+	require.Equal(t,
+		`{"from":2,"to":4,"line":1,"column":2,"message":"invalid operation: == (mismatched types int and bool)","snippet":"\n | 1 == true\n | ..^","prev":null}`,
+		string(b),
+	)
 }
 
 func TestAsBool_exposed_error(t *testing.T) {
@@ -2667,3 +2670,11 @@ func TestIssue_integer_truncated_by_compiler(t *testing.T) {
 	_, err = expr.Compile("fn(256)", expr.Env(env))
 	require.Error(t, err)
 }
+
+func TestExpr_crash(t *testing.T) {
+	content, err := os.ReadFile("testdata/crash.txt")
+	require.NoError(t, err)
+
+	_, err = expr.Compile(string(content))
+	require.Error(t, err)
+}

file/error.go

@@ -8,22 +8,36 @@ import (
 
 type Error struct {
 	Location
-	Message string
-	Snippet string
-	Prev    error
+	Line    int    `json:"line"`
+	Column  int    `json:"column"`
+	Message string `json:"message"`
+	Snippet string `json:"snippet"`
+	Prev    error  `json:"prev"`
 }
 
 func (e *Error) Error() string {
 	return e.format()
 }
 
-func (e *Error) Bind(source *Source) *Error {
-	if snippet, found := source.Snippet(e.Location.Line); found {
+func (e *Error) Bind(source Source) *Error {
+	e.Line = 1
+	for i, r := range source {
+		if i == e.From {
+			break
+		}
+		if r == '\n' {
+			e.Line++
+			e.Column = 0
+		} else {
+			e.Column++
+		}
+	}
+	if snippet, found := source.Snippet(e.Line); found {
 		snippet := strings.Replace(snippet, "\t", " ", -1)
 		srcLine := "\n | " + snippet
 		var bytes = []byte(snippet)
 		var indLine = "\n | "
-		for i := 0; i < e.Location.Column && len(bytes) > 0; i++ {
+		for i := 0; i < e.Column && len(bytes) > 0; i++ {
 			_, sz := utf8.DecodeRune(bytes)
 			bytes = bytes[sz:]
 			if sz > 1 {
@@ -54,7 +68,7 @@ func (e *Error) Wrap(err error) {
 }
 
 func (e *Error) format() string {
-	if e.Location.Empty() {
+	if e.Snippet == "" {
 		return e.Message
 	}
 	return fmt.Sprintf(

file/location.go

@@ -1,10 +1,6 @@
 package file
 
 type Location struct {
-	Line   int // The 1-based line of the location.
-	Column int // The 0-based column number of the location.
-}
-
-func (l Location) Empty() bool {
-	return l.Column == 0 && l.Line == 0
+	From int `json:"from"`
+	To   int `json:"to"`
 }

file/source.go

@@ -1,78 +1,47 @@
 package file
 
 import (
-	"encoding/json"
 	"strings"
 	"unicode/utf8"
 )
 
-type Source struct {
-	contents    []rune
-	lineOffsets []int32
-}
-
-func NewSource(contents string) *Source {
-	s := &Source{
-		contents: []rune(contents),
-	}
-	s.updateOffsets()
-	return s
-}
-
-func (s *Source) MarshalJSON() ([]byte, error) {
-	return json.Marshal(s.contents)
-}
-
-func (s *Source) UnmarshalJSON(b []byte) error {
-	contents := make([]rune, 0)
-	err := json.Unmarshal(b, &contents)
-	if err != nil {
-		return err
-	}
+type Source []rune
 
-	s.contents = contents
-	s.updateOffsets()
-	return nil
+func NewSource(contents string) Source {
+	return []rune(contents)
 }
 
-func (s *Source) Content() string {
-	return string(s.contents)
+func (s Source) String() string {
+	return string(s)
 }
 
-func (s *Source) Snippet(line int) (string, bool) {
+func (s Source) Snippet(line int) (string, bool) {
 	if s == nil {
 		return "", false
 	}
-	charStart, found := s.findLineOffset(line)
-	if !found || len(s.contents) == 0 {
+	lines := strings.Split(string(s), "\n")
+	lineOffsets := make([]int, len(lines))
+	var offset int
+	for i, line := range lines {
+		offset = offset + utf8.RuneCountInString(line) + 1
+		lineOffsets[i] = offset
+	}
+	charStart, found := getLineOffset(lineOffsets, line)
+	if !found || len(s) == 0 {
 		return "", false
 	}
-	charEnd, found := s.findLineOffset(line + 1)
+	charEnd, found := getLineOffset(lineOffsets, line+1)
 	if found {
-		return string(s.contents[charStart : charEnd-1]), true
-	}
-	return string(s.contents[charStart:]), true
-}
-
-// updateOffsets compute line offsets up front as they are referred to frequently.
-func (s *Source) updateOffsets() {
-	lines := strings.Split(string(s.contents), "\n")
-	offsets := make([]int32, len(lines))
-	var offset int32
-	for i, line := range lines {
-		offset = offset + int32(utf8.RuneCountInString(line)) + 1
-		offsets[int32(i)] = offset
+		return string(s[charStart : charEnd-1]), true
 	}
-	s.lineOffsets = offsets
+	return string(s[charStart:]), true
 }
 
-// findLineOffset returns the offset where the (1-indexed) line begins,
-// or false if line doesn't exist.
-func (s *Source) findLineOffset(line int) (int32, bool) {
+func getLineOffset(lineOffsets []int, line int) (int, bool) {
 	if line == 1 {
 		return 0, true
-	} else if line > 1 && line <= len(s.lineOffsets) {
-		offset := s.lineOffsets[line-2]
+	} else if line > 1 && line <= len(lineOffsets) {
+		offset := lineOffsets[line-2]
 		return offset, true
 	}
 	return -1, false

file/source_test.go

@@ -1,10 +1,7 @@
 package file
 
 import (
-	"encoding/json"
 	"testing"
-
-	"github.com/expr-lang/expr/internal/testify/assert"
 )
 
 const (
@@ -55,15 +52,3 @@ func TestStringSource_SnippetSingleLine(t *testing.T) {
 		t.Errorf(unexpectedSnippet, t.Name(), str2, "")
 	}
 }
-
-func TestStringSource_MarshalJSON(t *testing.T) {
-	source := NewSource("hello, world")
-	encoded, err := json.Marshal(source)
-	assert.NoError(t, err)
-	assert.Equal(t, `[104,101,108,108,111,44,32,119,111,114,108,100]`, string(encoded))
-
-	decoded := &Source{}
-	err = json.Unmarshal(encoded, decoded)
-	assert.NoError(t, err)
-	assert.Equal(t, source.Content(), decoded.Content())
-}

parser/lexer/lexer.go

@@ -3,20 +3,18 @@ package lexer
 import (
 	"fmt"
 	"strings"
-	"unicode/utf8"
 
 	"github.com/expr-lang/expr/file"
 )
 
-func Lex(source *file.Source) ([]Token, error) {
+func Lex(source file.Source) ([]Token, error) {
 	l := &lexer{
-		input:  source.Content(),
+		source: source,
 		tokens: make([]Token, 0),
+		start:  0,
+		end:    0,
 	}
-
-	l.loc = file.Location{Line: 1, Column: 0}
-	l.prev = l.loc
-	l.startLoc = l.loc
+	l.commit()
 
 	for state := root; state != nil; {
 		state = state(l)
@@ -30,34 +28,25 @@ func Lex(source *file.Source) ([]Token, error) {
 }
 
 type lexer struct {
-	input      string
+	source     file.Source
 	tokens     []Token
-	start, end int           // current position in input
-	width      int           // last rune width
-	startLoc   file.Location // start location
-	prev, loc  file.Location // prev location of end location, end location
+	start, end int
 	err        *file.Error
 }
 
 const eof rune = -1
 
+func (l *lexer) commit() {
+	l.start = l.end
+}
+
 func (l *lexer) next() rune {
-	if l.end >= len(l.input) {
-		l.width = 0
+	if l.end >= len(l.source) {
+		l.end++
 		return eof
 	}
-	r, w := utf8.DecodeRuneInString(l.input[l.end:])
-	l.width = w
-	l.end += w
-
-	l.prev = l.loc
-	if r == '\n' {
-		l.loc.Line++
-		l.loc.Column = 0
-	} else {
-		l.loc.Column++
-	}
-
+	r := l.source[l.end]
+	l.end++
 	return r
 }
 
@@ -68,8 +57,7 @@ func (l *lexer) peek() rune {
 }
 
 func (l *lexer) backup() {
-	l.end -= l.width
-	l.loc = l.prev
+	l.end--
 }
 
 func (l *lexer) emit(t Kind) {
@@ -78,35 +66,39 @@ func (l *lexer) emit(t Kind) {
 
 func (l *lexer) emitValue(t Kind, value string) {
 	l.tokens = append(l.tokens, Token{
-		Location: l.startLoc,
+		Location: file.Location{From: l.start, To: l.end},
 		Kind:     t,
 		Value:    value,
 	})
-	l.start = l.end
-	l.startLoc = l.loc
+	l.commit()
 }
 
 func (l *lexer) emitEOF() {
+	from := l.end - 2
+	if from < 0 {
+		from = 0
+	}
+	to := l.end - 1
+	if to < 0 {
+		to = 0
+	}
 	l.tokens = append(l.tokens, Token{
-		Location: l.prev, // Point to previous position for better error messages.
+		Location: file.Location{From: from, To: to},
 		Kind:     EOF,
 	})
-	l.start = l.end
-	l.startLoc = l.loc
+	l.commit()
 }
 
 func (l *lexer) skip() {
-	l.start = l.end
-	l.startLoc = l.loc
+	l.commit()
 }
 
 func (l *lexer) word() string {
-	return l.input[l.start:l.end]
-}
-
-func (l *lexer) ignore() {
-	l.start = l.end
-	l.startLoc = l.loc
+	// TODO: boundary check is NOT needed here, but for some reason CI fuzz tests are failing.
+	if l.start > len(l.source) || l.end > len(l.source) {
+		return "__invalid__"
+	}
+	return string(l.source[l.start:l.end])
 }
 
 func (l *lexer) accept(valid string) bool {
@@ -132,18 +124,18 @@ func (l *lexer) skipSpaces() {
 }
 
 func (l *lexer) acceptWord(word string) bool {
-	pos, loc, prev := l.end, l.loc, l.prev
+	pos := l.end
 
 	l.skipSpaces()
 
 	for _, ch := range word {
 		if l.next() != ch {
-			l.end, l.loc, l.prev = pos, loc, prev
+			l.end = pos
 			return false
 		}
 	}
 	if r := l.peek(); r != ' ' && r != eof {
-		l.end, l.loc, l.prev = pos, loc, prev
+		l.end = pos
 		return false
 	}
 
@@ -153,8 +145,11 @@ func (l *lexer) acceptWord(word string) bool {
 func (l *lexer) error(format string, args ...any) stateFn {
 	if l.err == nil { // show first error
 		l.err = &file.Error{
-			Location: l.loc,
-			Message:  fmt.Sprintf(format, args...),
+			Location: file.Location{
+				From: l.end - 1,
+				To:   l.end,
+			},
+			Message: fmt.Sprintf(format, args...),
 		}
 	}
 	return nil
@@ -230,6 +225,6 @@ func (l *lexer) scanRawString(quote rune) (n int) {
 		ch = l.next()
 		n++
 	}
-	l.emitValue(String, l.input[l.start+1:l.end-1])
+	l.emitValue(String, string(l.source[l.start+1:l.end-1]))
 	return
 }