2 files changed,
29 insertions(+),
28 deletions(-)
Author:
Oleksandr Smirnov
olexsmir@gmail.com
Committed at:
2026-08-24 00:13:47 +0300
Authored at:
2026-08-21 20:47:49 +0300
Change ID:
xlrtrwstxwuqlpvzntkovtsspzovwklt
Parent:
c508b13
jump to
| M | journal/lexer/lexer.go |
| M | journal/loader.go |
M
journal/lexer/lexer.go
··· 4 4 "strings" 5 5 "unicode" 6 6 "unicode/utf8" 7 + "unsafe" 7 8 8 9 "olexsmir.xyz/clerk/journal/token" 9 10 ) ··· 163 164 default: 164 165 s := l.save() 165 166 l.advance() 166 - return token.Token{Type: token.ILLEGAL, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 167 + return token.Token{Type: token.ILLEGAL, Literal: l.lit(s), Span: l.span(s)} 167 168 } 168 169 } 169 170 ··· 194 195 for l.ch != '\n' && l.ch != '\r' && l.ch != 0 { 195 196 l.advance() 196 197 } 197 - return token.Token{Type: token.TEXT, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 198 + return token.Token{Type: token.TEXT, Literal: l.lit(s), Span: l.span(s)} 198 199 } 199 200 200 201 func (l *Lexer) lexTransaction() token.Token { ··· 261 262 for l.ch != 0 && l.ch != '\n' && l.ch != '\r' && l.ch != ';' { 262 263 l.advance() 263 264 } 264 - lit := string(l.input[s.offset:l.pos]) 265 + lit := l.lit(s) 265 266 for len(lit) > 0 && (lit[len(lit)-1] == ' ' || lit[len(lit)-1] == '\t') { 266 267 lit = lit[:len(lit)-1] 267 268 } ··· 428 429 } 429 430 return token.Token{ 430 431 Type: token.TEXT, 431 - Literal: string(l.input[s.offset:l.pos]), 432 + Literal: l.lit(s), 432 433 Span: l.span(s), 433 434 } 434 435 } ··· 438 439 l.advance() 439 440 return token.Token{ 440 441 Type: kind, 441 - Literal: string(l.input[s.offset:l.pos]), 442 + Literal: l.lit(s), 442 443 Span: l.span(s), 443 444 } 444 445 } ··· 455 456 for l.ch == ' ' || l.ch == '\t' { 456 457 l.advance() 457 458 } 458 - lit := string(l.input[s.offset:l.pos]) 459 - return token.Token{Type: token.WHITESPACE, Literal: lit, Span: l.span(s)} 459 + return token.Token{Type: token.WHITESPACE, Literal: l.lit(s), Span: l.span(s)} 460 460 } 461 461 462 462 func (l *Lexer) lexIndent() token.Token { ··· 464 464 for l.ch == ' ' || l.ch == '\t' { 465 465 l.advance() 466 466 } 467 - lit := string(l.input[s.offset:l.pos]) 468 - return token.Token{Type: token.INDENT, Literal: lit, Span: l.span(s)} 467 + return token.Token{Type: token.INDENT, Literal: l.lit(s), Span: l.span(s)} 469 468 } 470 469 471 470 func (l *Lexer) lexEquals() token.Token { ··· 518 517 for l.ch != '\n' && l.ch != '\r' && l.ch != ';' && l.ch != 0 && l.ch != ')' && l.ch != ']' && l.ch != ':' && l.ch != ' ' && l.ch != '\t' { 519 518 l.advance() 520 519 } 521 - lit := string(l.input[s.offset:l.pos]) 522 - return token.Token{Type: token.TEXT, Literal: lit, Span: l.span(s)} 520 + return token.Token{Type: token.TEXT, Literal: l.lit(s), Span: l.span(s)} 523 521 } 524 522 525 523 // lexAccountNamePosting reads an account name in posting context. ··· 535 533 if l.ch != ':' { 536 534 l.postingExpectAccount = false 537 535 } 538 - lit := string(l.input[s.offset:l.pos]) 539 - return token.Token{Type: token.TEXT, Literal: lit, Span: l.span(s)} 536 + return token.Token{Type: token.TEXT, Literal: l.lit(s), Span: l.span(s)} 540 537 } 541 538 542 539 func (l *Lexer) lexParenExpr() token.Token { ··· 554 551 } 555 552 l.advance() 556 553 } 557 - lit := string(l.input[s.offset:l.pos]) 558 - return token.Token{Type: token.PARENEXPR, Literal: lit, Span: l.span(s)} 554 + return token.Token{Type: token.PARENEXPR, Literal: l.lit(s), Span: l.span(s)} 559 555 } 560 556 561 557 func (l *Lexer) lexNumber() token.Token { ··· 585 581 break 586 582 } 587 583 } 588 - lit := string(l.input[s.offset:l.pos]) 584 + lit := l.lit(s) 589 585 kind := token.INT 590 586 if strings.ContainsAny(lit, "., eE") { 591 587 kind = token.DECIMAL ··· 598 594 for l.ch != 0 && l.ch != '\n' && l.ch != '\r' && l.ch != ' ' && l.ch != '\t' && l.ch != ';' { 599 595 l.advance() 600 596 } 601 - lit := string(l.input[s.offset:l.pos]) 597 + lit := l.lit(s) 602 598 kind := l.keyword(lit) 603 599 if kind == token.ILLEGAL { // todo: report an error ?? 604 600 kind = token.TEXT ··· 615 611 for l.isDigit() || (l.isDateSep() && l.peekIsDigit()) { 616 612 l.advance() 617 613 } 618 - return token.Token{Type: token.DATE, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 614 + return token.Token{Type: token.DATE, Literal: l.lit(s), Span: l.span(s)} 619 615 } 620 616 621 617 func isSymbolChar(r rune) bool { ··· 632 628 if l.ch == quote { 633 629 l.advance() 634 630 } 635 - return token.Token{Type: token.STRING, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 631 + return token.Token{Type: token.STRING, Literal: l.lit(s), Span: l.span(s)} 636 632 } 637 633 638 634 func (l *Lexer) lexCommodityMark() token.Token { ··· 646 642 if l.ch == '"' { 647 643 l.advance() 648 644 } 649 - return token.Token{Type: token.COMMODITYMARK, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 645 + return token.Token{Type: token.COMMODITYMARK, Literal: l.lit(s), Span: l.span(s)} 650 646 } 651 647 652 648 if unicode.IsLetter(l.ch) { 653 649 for unicode.IsLetter(l.ch) { 654 650 l.advance() 655 651 } 656 - return token.Token{Type: token.COMMODITYMARK, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 652 + return token.Token{Type: token.COMMODITYMARK, Literal: l.lit(s), Span: l.span(s)} 657 653 } 658 654 659 655 if isSymbolChar(l.ch) { 660 656 for isSymbolChar(l.ch) { 661 657 l.advance() 662 658 } 663 - return token.Token{Type: token.COMMODITYMARK, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 659 + return token.Token{Type: token.COMMODITYMARK, Literal: l.lit(s), Span: l.span(s)} 664 660 } 665 661 666 662 l.advance() 667 - return token.Token{Type: token.COMMODITYMARK, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 663 + return token.Token{Type: token.COMMODITYMARK, Literal: l.lit(s), Span: l.span(s)} 668 664 } 669 665 670 666 func (l *Lexer) lexLBrace() token.Token { ··· 691 687 if l.rpos >= len(l.input) { 692 688 l.ch = 0 693 689 l.chSize = 0 690 + } else if b := l.input[l.rpos]; b < utf8.RuneSelf { // ASCII fast path 691 + l.ch = rune(b) 692 + l.chSize = 1 694 693 } else { 695 - r, size := utf8.DecodeRune(l.input[l.rpos:]) 696 - l.ch = r 697 - l.chSize = size 694 + l.ch, l.chSize = utf8.DecodeRune(l.input[l.rpos:]) 698 695 } 699 696 l.pos = l.rpos 700 697 l.rpos += l.chSize ··· 798 795 for l.isDigit() || l.ch == ':' { 799 796 l.advance() 800 797 } 801 - return token.Token{Type: token.TIME, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} 798 + return token.Token{Type: token.TIME, Literal: l.lit(s), Span: l.span(s)} 802 799 } 803 800 804 801 type savedPos struct{ offset, line, col int } ··· 817 814 func (l *Lexer) token(kind token.Type, literal string) token.Token { 818 815 s := savedPos{l.pos, l.line, l.col} 819 816 return token.Token{Type: kind, Literal: literal, Span: l.span(s)} 817 +} 818 + 819 +func (l *Lexer) lit(s savedPos) string { 820 + return unsafe.String(&l.input[s.offset], l.pos-s.offset) 820 821 } 821 822 822 823 func (l *Lexer) keyword(s string) token.Type {