clerk/journal/lexer/lexer_test.go (view raw)
| 1 | package lexer |
| 2 | |
| 3 | import ( |
| 4 | "testing" |
| 5 | |
| 6 | "olexsmir.xyz/clerk/internal/testutil/golden" |
| 7 | "olexsmir.xyz/clerk/journal/token" |
| 8 | ) |
| 9 | |
| 10 | func TestLexer(t *testing.T) { |
| 11 | tests := []string{ |
| 12 | "account directive", |
| 13 | "automated transaction", |
| 14 | "better date", |
| 15 | "blank lines", |
| 16 | "cleared transaction", |
| 17 | "comment block directive without end", |
| 18 | "comment block directive", |
| 19 | "comment line", |
| 20 | "commodity directive", |
| 21 | "date with secondary", |
| 22 | "empty", |
| 23 | "hash comment", |
| 24 | "inline comment", |
| 25 | "market price directive with time", |
| 26 | "market price directive", |
| 27 | "simple transaction", |
| 28 | "special chars in description", |
| 29 | "star comment", |
| 30 | "transaction with code", |
| 31 | "transaction with unicode commodity symbols", |
| 32 | "transaction with virtual accounts", |
| 33 | "transaction, accounts with uppercase letters", |
| 34 | } |
| 35 | for _, tt := range tests { |
| 36 | t.Run(tt, func(t *testing.T) { |
| 37 | a := golden.Read(t, tt) |
| 38 | golden.Assert(t, a, New("j", a.Get("input")).dump()) |
| 39 | }) |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | func FuzzLexer(f *testing.F) { |
| 44 | const maxKnownTokenType = token.C // ensures fuzzer never sees out-of-range token types |
| 45 | |
| 46 | f.Add([]byte("2024/01/01 groceries\n expenses:food $10.00\n assets:checking\n")) |
| 47 | f.Add([]byte("2024/01/01 * groceries\n expenses:food $10.00\n assets:checking\n")) |
| 48 | f.Add([]byte("2024/01/01 ! groceries\n expenses:food $10.00\n assets:checking\n")) |
| 49 | f.Add([]byte("2024/01/01 t ; inline comment\n a $10\n")) |
| 50 | f.Add([]byte("2024/01/01 t\n (a) 10 @@ $20\n [b] 30\n")) |
| 51 | f.Add([]byte("2008/06/03 * eat & shop\n expenses:food $1\n expenses:supplies $1\n assets:cash\n")) |
| 52 | f.Add([]byte("2015-01-03 * Money exchange office\n Assets:Cash -20 EUR @ 7.53 HRK\n Assets:Cash 150.60 HRK\n")) |
| 53 | f.Add([]byte("2024/01/01 ß\n (ß) 10 ß\n")) |
| 54 | f.Add([]byte("2024/01/01 t\n (! a) 10\n")) |
| 55 | f.Add([]byte("comment\nbody\nend\n")) |
| 56 | f.Add([]byte("apply tag foo\nend\n")) |
| 57 | f.Add([]byte("; a comment\n")) |
| 58 | f.Add([]byte("# a comment\n")) |
| 59 | f.Add([]byte("* a comment\n")) |
| 60 | f.Add([]byte("account expenses:food\n")) |
| 61 | f.Add([]byte("commodity 1,000.00 UAH\n")) |
| 62 | f.Add([]byte("N $\n")) |
| 63 | f.Add([]byte("P 2024-01-01 USD 41.50 UAH\n")) |
| 64 | f.Add([]byte("P 2024-01-01 12:00:00 USD 41.50 UAH\n")) |
| 65 | f.Add([]byte("P 2024-01-01 12:00 USD 41.50 UAH\n")) |
| 66 | f.Add([]byte("~ monthly\n a $10\n b\n")) |
| 67 | f.Add([]byte("= /^Income/\n expenses:food $10\n")) |
| 68 | f.Add([]byte("перевірка\n")) |
| 69 | f.Add([]byte("")) |
| 70 | f.Add([]byte("\n\n\n")) |
| 71 | f.Add([]byte("@@@\n")) |
| 72 | f.Add([]byte(" \n")) |
| 73 | f.Add([]byte("0\n")) |
| 74 | f.Add([]byte{0xff, 0xfe, 0x00}) |
| 75 | |
| 76 | f.Fuzz(func(t *testing.T, data []byte) { |
| 77 | // Pass 1: lex and validate token stream |
| 78 | l := New("j", data) |
| 79 | var tokens []token.Token |
| 80 | maxTokens := max(len(data)*2, 16) |
| 81 | prevEnd := -1 |
| 82 | for range maxTokens { |
| 83 | tok := l.Next() |
| 84 | |
| 85 | if tok.Span.Start.Offset < prevEnd { |
| 86 | t.Fatalf("non-monotonic span: prevEnd=%d current=%s %d", |
| 87 | prevEnd, tok.Type, tok.Span.Start.Offset) |
| 88 | } |
| 89 | |
| 90 | if tok.Type < 0 || tok.Type > maxKnownTokenType { |
| 91 | t.Fatalf("token type out of range: %d", tok.Type) |
| 92 | } |
| 93 | |
| 94 | // EOF and NEWLINE sentinels may extend one past the input |
| 95 | maxEnd := len(data) |
| 96 | if tok.Type == token.NEWLINE || tok.Type == token.EOF { |
| 97 | maxEnd = len(data) + 1 |
| 98 | } |
| 99 | if tok.Span.Start.Offset < 0 || tok.Span.End.Offset > maxEnd || |
| 100 | tok.Span.Start.Offset > tok.Span.End.Offset { |
| 101 | t.Fatalf("span out of bounds: [%d,%d] for len=%d type=%s", |
| 102 | tok.Span.Start.Offset, tok.Span.End.Offset, len(data), tok.Type) |
| 103 | } |
| 104 | |
| 105 | if tok.Type == token.EOF { |
| 106 | break |
| 107 | } |
| 108 | |
| 109 | // Non-zero-length for non-EOF tokens (NEWLINE sentinel is exempt) |
| 110 | if tok.Type != token.NEWLINE && tok.Span.End.Offset <= tok.Span.Start.Offset { |
| 111 | t.Fatalf("non-progressing token: %s %q at %d:%d-%d:%d", |
| 112 | tok.Type, tok.Literal, |
| 113 | tok.Span.Start.Line, tok.Span.Start.Col, |
| 114 | tok.Span.End.Line, tok.Span.End.Col) |
| 115 | } |
| 116 | |
| 117 | tokens = append(tokens, tok) |
| 118 | prevEnd = tok.Span.End.Offset |
| 119 | } |
| 120 | |
| 121 | if prevEnd > len(data)+1 { |
| 122 | t.Fatalf("token consumed beyond input: end=%d len=%d", prevEnd, len(data)) |
| 123 | } |
| 124 | |
| 125 | // pass 2: re-lex the same input, the token stream must be identical |
| 126 | l2 := New("j", data) |
| 127 | for _, expected := range tokens { |
| 128 | tok := l2.Next() |
| 129 | if tok.Type != expected.Type || tok.Literal != expected.Literal { |
| 130 | t.Fatalf("re-lex mismatch at offset %d: expected (%s %q), got (%s %q)", |
| 131 | expected.Span.Start.Offset, expected.Type, expected.Literal, tok.Type, tok.Literal) |
| 132 | } |
| 133 | } |
| 134 | }) |
| 135 | } |