10 files changed,
16 insertions(+),
16 deletions(-)
Author:
Oleksandr Smirnov
olexsmir@gmail.com
Committed at:
2026-08-12 16:49:20 +0300
Authored at:
2026-08-12 15:31:44 +0300
Change ID:
pyttqlkqqomsptqrvsmznpvstvyykxrm
Parent:
1efa5a1
jump to
M
internal/lsp/textdocument_completion.go
··· 430 430 431 431 func lexLine(content string, cursor int) []token.Token { 432 432 lineStart, lineEnd := lineBounds(content, cursor) 433 - l := lexer.New("", []byte(content[lineStart:lineEnd])) 433 + l := lexer.New("", content[lineStart:lineEnd]) 434 434 var out []token.Token 435 435 for { 436 436 t := l.Next()
M
internal/lsp/textdocument_format.go
··· 18 18 } 19 19 20 20 text := state.text 21 - lex := lexer.New(params.TextDocument.URI.Path(), []byte(text)) 21 + lex := lexer.New(params.TextDocument.URI.Path(), text) 22 22 jnrl := parser.New(lex).ParseJournal() 23 23 if len(jnrl.Errors) > 0 { 24 24 return nil, fmt.Errorf("can't format file with errors: %v", jnrl.Errors[0].Message) // TODO: report all errors
M
internal/lsp/textdocument_semantic.go
··· 522 522 } 523 523 524 524 func semLexerFallback(content string, base []rawSpan, emit semEmitFunc) { 525 - l := lexer.New("", []byte(content)) 525 + l := lexer.New("", content) 526 526 527 527 var commentStart, commentEnd int // 0 = not inside a comment line 528 528 lineStart := true // the next significant token starts a line
M
journal/lexer/lexer.go
··· 41 41 42 42 type Lexer struct { 43 43 file string 44 - input []byte 44 + input string 45 45 mode mode 46 46 47 47 ch rune // current rune (0 = EOF/sentinel) ··· 62 62 subdirective bool 63 63 } 64 64 65 -func New(file string, input []byte) *Lexer { 65 +func New(file string, input string) *Lexer { 66 66 l := &Lexer{ 67 67 file: file, 68 68 input: input, ··· 690 690 l.ch = 0 691 691 l.chSize = 0 692 692 } else { 693 - r, size := utf8.DecodeRune(l.input[l.rpos:]) 693 + r, size := utf8.DecodeRuneInString(l.input[l.rpos:]) 694 694 l.ch = r 695 695 l.chSize = size 696 696 } ··· 705 705 } 706 706 707 707 func (l *Lexer) peek() rune { 708 - r, _ := utf8.DecodeRune(l.input[l.rpos:]) 708 + r, _ := utf8.DecodeRuneInString(l.input[l.rpos:]) 709 709 return r 710 710 } 711 711
M
journal/lexer/lexer_test.go
··· 35 35 for _, tt := range tests { 36 36 t.Run(tt, func(t *testing.T) { 37 37 a := golden.Read(t, tt) 38 - golden.Assert(t, a, New("j", a.Get("input")).dump()) 38 + golden.Assert(t, a, New("j", string(a.Get("input"))).dump()) 39 39 }) 40 40 } 41 41 } ··· 75 75 76 76 f.Fuzz(func(t *testing.T, data []byte) { 77 77 // Pass 1: lex and validate token stream 78 - l := New("j", data) 78 + l := New("j", string(data)) 79 79 var tokens []token.Token 80 80 maxTokens := max(len(data)*2, 16) 81 81 prevEnd := -1 ··· 125 125 } 126 126 127 127 // Pass 2: re-lex the same input — token stream must be identical 128 - l2 := New("j", data) 128 + l2 := New("j", string(data)) 129 129 for _, expected := range tokens { 130 130 tok := l2.Next() 131 131 if tok.Type != expected.Type || tok.Literal != expected.Literal {
M
journal/parser/parser_test.go
··· 101 101 for _, tt := range tests { 102 102 t.Run(tt, func(t *testing.T) { 103 103 a := golden.Read(t, tt) 104 - l := lexer.New("j", a.Get("input")) 104 + l := lexer.New("j", string(a.Get("input"))) 105 105 f := New(l).ParseJournal() 106 106 golden.Assert(t, a, ast.Dump(f)) 107 107 }) ··· 144 144 f.Add([]byte{0xff, 0xfe, 0x00}) 145 145 146 146 f.Fuzz(func(t *testing.T, data []byte) { 147 - l := lexer.New("j", data) 147 + l := lexer.New("j", string(data)) 148 148 j := New(l).ParseJournal() 149 149 if j == nil { 150 150 t.Fatalf("nil journal for input %q", string(data))