19 files changed,
1207 insertions(+),
106 deletions(-)
Author:
Oleksandr Smirnov
olexsmir@gmail.com
Committed at:
2026-08-03 11:44:08 +0300
Authored at:
2026-08-02 18:00:46 +0300
Change ID:
lpmvqpsmmqpnyloyynlnpnpkvoyktrzy
Parent:
64f0a9d
jump to
M
internal/analyzer/build.go
··· 217 217 } 218 218 } 219 219 sort.Slice(a.Dates, func(i, j int) bool { 220 - return dateLess(a.Dates[i], a.Dates[j]) 220 + return a.Dates[i].Compare(a.Dates[j]) < 0 221 221 }) 222 222 sort.Strings(a.DateStrings) 223 223 } ··· 241 241 return "" 242 242 } 243 243 return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day) 244 -} 245 - 246 -func dateLess(a, b ast.Date) bool { 247 - if a.Year != b.Year { 248 - return a.Year < b.Year 249 - } 250 - if a.Month != b.Month { 251 - return a.Month < b.Month 252 - } 253 - return a.Day < b.Day 254 244 } 255 245 256 246 func (a *Analysis) addCommodityUsage(fileIndex int, am *ast.Amount, date *ast.Date) {
M
internal/linter/rule_orderdate.go
··· 21 21 if !ok { 22 22 continue 23 23 } 24 - if anchor != nil && o.compareDate(txn.Date, *anchor) < 0 { 24 + if anchor != nil && txn.Date.Compare(*anchor) < 0 { 25 25 finds = append(finds, Find{ 26 26 Code: o.ID(), 27 27 Severity: o.Severity(), ··· 34 34 } 35 35 } 36 36 return finds 37 -} 38 - 39 -// compareDate returns -1 if a < b, 0 if equal, 1 if a > b. 40 -func (o OrderDate) compareDate(a, b ast.Date) int { 41 - if a.Year != b.Year { 42 - if a.Year < b.Year { 43 - return -1 44 - } 45 - return 1 46 - } 47 - if a.Month != b.Month { 48 - if a.Month < b.Month { 49 - return -1 50 - } 51 - return 1 52 - } 53 - if a.Day != b.Day { 54 - if a.Day < b.Day { 55 - return -1 56 - } 57 - return 1 58 - } 59 - return 0 60 37 } 61 38 62 39 func (o OrderDate) dateString(d ast.Date) string {
M
internal/lsp/diagnostics.go
··· 9 9 "go.lsp.dev/protocol" 10 10 "go.lsp.dev/uri" 11 11 12 - "olexsmir.xyz/clerk/internal/analyzer" 13 12 "olexsmir.xyz/clerk/internal/linter" 14 13 "olexsmir.xyz/clerk/journal/token" 15 14 ) ··· 38 37 func (s *server) publishDiagnostics(ctx context.Context) { 39 38 s.log.Debug("publishing diagnostics") 40 39 41 - s.mu.Lock() 42 - openDocs := make(map[uri.URI]docState, len(s.openDocs)) 43 - for u, st := range s.openDocs { 44 - openDocs[u] = st 45 - } 46 - s.mu.Unlock() 47 - 48 40 if ctx.Err() != nil { 49 41 return 50 42 } 51 43 52 - if len(openDocs) == 0 { 53 - return 54 - } 55 - 56 - activePaths := make(map[string]bool) 57 - 58 - var a *analyzer.Analysis 59 - for duri, state := range openDocs { 60 - path := duri.Path() 61 - rj := s.loader.ResolveBytes(path, []byte(state.text)) 62 - if a == nil { 63 - a = analyzer.Build(rj) 64 - } else { 65 - // Merge: only one root in practice for now. 66 - } 67 - for _, pf := range rj.Occurrences { 68 - activePaths[pf.Path] = true 69 - } 70 - } 71 - 44 + a := s.buildAnalysis() 72 45 if a == nil { 73 46 s.log.Debug("no files in workspace") 74 47 return 48 + } 49 + 50 + activePaths := make(map[string]bool, len(a.Files)) 51 + for _, pf := range a.Files { 52 + activePaths[pf.Path] = true 75 53 } 76 54 77 55 if ctx.Err() != nil {
A
internal/lsp/fuzzy/fuzzy.go
··· 1 +package fuzzy 2 + 3 +import "strings" 4 + 5 +// Score scores how well pattern matches text as a subsequence. 6 +// It returns 0 if pattern is not a case-insensitive subsequence of text, 7 +// otherwise a score in (0, 1], where 1 is a perfect contiguous match at a 8 +// segment boundary. An empty pattern matches everything with score 1. 9 +// 10 +// Matching is greedy and leftmost; each matched rune scores base 1, plus 3 if 11 +// it starts a segment (text start or after a separator), plus 2 if it is 12 +// contiguous with the previous matched rune, plus 1 if the case matches 13 +// exactly. The total is normalized by 4L+1, the maximum score of a perfect 14 +// match of length L, so exact segment matches score 1 regardless of length 15 +// ("food" and "expenses" both match "expenses:food" at 1.0). 16 +func Score(pattern, text string) float64 { 17 + if pattern == "" { 18 + return 1 19 + } 20 + p := []rune(strings.ToLower(pattern)) 21 + t := []rune(text) 22 + tl := []rune(strings.ToLower(text)) 23 + 24 + total := 0 25 + prev := -1 26 + for i, pr := range p { 27 + j := prev + 1 28 + for ; j < len(t); j++ { 29 + if tl[j] == pr { 30 + break 31 + } 32 + } 33 + if j == len(t) { 34 + return 0 35 + } 36 + w := 1 37 + if j == 0 || isFuzzySep(t[j-1]) { 38 + w += 3 39 + } 40 + if i > 0 && j == prev+1 { 41 + w += 2 42 + } 43 + if t[j] == p[i] { 44 + w++ 45 + } 46 + total += w 47 + prev = j 48 + } 49 + score := float64(total) / float64(4*len(p)+1) 50 + if score > 1 { 51 + return 1 52 + } 53 + return score 54 +} 55 + 56 +// isFuzzySep reports whether r is a segment boundary for fuzzy matching: 57 +// account-name separators and word boundaries. 58 +func isFuzzySep(r rune) bool { 59 + switch r { 60 + case ':', '.', '-', '_', '/', ' ', '\t': 61 + return true 62 + } 63 + return false 64 +}
A
internal/lsp/fuzzy/fuzzy_test.go
··· 1 +package fuzzy 2 + 3 +import ( 4 + "math" 5 + "testing" 6 +) 7 + 8 +func TestFuzzyScore(t *testing.T) { 9 + tests := []struct { 10 + name string 11 + pattern string 12 + text string 13 + want float64 // -1 means: expect 0 (no match) 14 + }{ 15 + {"empty pattern matches everything", "", "expenses:food", 1}, 16 + {"exact boundary match", "food", "expenses:food", 1}, 17 + {"long exact match also perfect", "expenses", "expenses:food", 1}, 18 + {"acronym across segments", "expf", "expenses:food", 1}, 19 + {"colon need not be typed", "expensesfood", "expenses:food", 1}, 20 + {"all-caps pattern still perfect", "FOOD", "expenses:food", 1}, 21 + {"gapped mid-word match", "od", "expenses:food", 0.444}, 22 + {"boundary beats mid-word", "xf", "expenses:food", 0.778}, 23 + {"exact prefix", "ex", "expenses", 1}, 24 + {"gap penalty", "esf", "expenses:food", 0.923}, 25 + {"no match", "xyz", "abc", 0}, 26 + {"cjk perfect match", "食物", "支出:食物", 1}, 27 + {"ukrainian perfect match", "продукти", "витрати:продукти", 1}, 28 + {"cjk gap", "食物", "支出费物", 0}, 29 + } 30 + for _, tt := range tests { 31 + t.Run(tt.name, func(t *testing.T) { 32 + got := Score(tt.pattern, tt.text) 33 + if tt.want == 0 { 34 + if got != 0 { 35 + t.Errorf("Score(%q, %q) = %v, want 0", tt.pattern, tt.text, got) 36 + } 37 + return 38 + } 39 + if math.Abs(got-tt.want) > 1e-3 { 40 + t.Errorf("Score(%q, %q) = %v, want %v", tt.pattern, tt.text, got, tt.want) 41 + } 42 + }) 43 + } 44 +} 45 + 46 +func TestFuzzyScoreDeterministic(t *testing.T) { 47 + for _, text := range []string{"expenses:food", "支出:食物", "витрати:продукти"} { 48 + for _, pattern := range []string{"", "ex", "expf", "o", "食物", "п"} { 49 + if a, b := Score(pattern, text), Score(pattern, text); a != b { 50 + t.Errorf("Score(%q, %q) not deterministic: %v != %v", pattern, text, a, b) 51 + } 52 + } 53 + } 54 +}
M
internal/lsp/lsputil/utf16.go
··· 1 1 package lsputil 2 2 3 -import "unicode/utf8" 3 +import ( 4 + "unicode/utf8" 5 + 6 + "go.lsp.dev/protocol" 7 +) 4 8 5 9 // Utf16Col returns the UTF-16 code unit column (0-based) for a byte offset 6 10 // within the given line content (without newline). ··· 46 50 return n 47 51 } 48 52 53 +// Offset converts a 0-based line and UTF-16 code unit column to a byte offset 54 +// in content, clamped to the content bounds. The inverse of LineCol. 55 +func Offset(content string, line, col int) int { 56 + if line < 0 { 57 + line = 0 58 + } 59 + off := 0 60 + for curLine := 0; curLine < line && off < len(content); { 61 + switch content[off] { 62 + case '\n': 63 + off++ 64 + curLine++ 65 + case '\r': 66 + off++ 67 + if off < len(content) && content[off] == '\n' { 68 + off++ 69 + } 70 + curLine++ 71 + default: 72 + for off < len(content) && content[off] != '\n' && content[off] != '\r' { 73 + off++ 74 + } 75 + } 76 + } 77 + lineEnd := off 78 + for lineEnd < len(content) && content[lineEnd] != '\n' && content[lineEnd] != '\r' { 79 + lineEnd++ 80 + } 81 + units := 0 82 + for off < lineEnd && units < col { 83 + r, size := utf8.DecodeRuneInString(content[off:]) 84 + off += size 85 + units += utf16Len(r) 86 + } 87 + return off 88 +} 89 + 49 90 // LineCol converts a byte offset in content to 0-based line number and 50 91 // UTF-16 code unit column. 51 92 func LineCol(content string, offset int) (line int, col int) { ··· 94 135 } 95 136 col = Utf16Col(content[lineStart:], len(content)-lineStart) 96 137 return lineNum, col 138 +} 139 + 140 +// Position converts a byte offset to an LSP position (0-based line, UTF-16 141 +// code unit column). The inverse of Offset at the protocol.Position level. 142 +func Position(content string, offset int) protocol.Position { 143 + line, col := LineCol(content, offset) 144 + return protocol.Position{Line: uint32(line), Character: uint32(col)} 97 145 } 98 146 99 147 // utf16Len returns the number of UTF-16 code units for a rune.
M
internal/lsp/lsputil/utf16_test.go
··· 1 1 package lsputil 2 2 3 -import "testing" 3 +import ( 4 + "testing" 5 + "unicode/utf8" 6 +) 4 7 5 8 func TestUtf16Col_Basic(t *testing.T) { 6 9 col := Utf16Col("hello", 3) ··· 82 85 t.Errorf("LineCol(2) = (%d,%d), want (0,2)", line, col) 83 86 } 84 87 } 88 + 89 +func TestOffset_Basic(t *testing.T) { 90 + if got := Offset("hello\nworld", 1, 5); got != 11 { 91 + t.Errorf("Offset(1,5) = %d, want 11", got) 92 + } 93 +} 94 + 95 +func TestOffset_RoundTrip(t *testing.T) { 96 + content := "first\nПривет мир\r\nlast\n" 97 + // only rune-boundary offsets round-trip (LineCol clamps mid-rune offsets) 98 + var boundaries []int 99 + for i := 0; i < len(content); { 100 + _, size := utf8.DecodeRuneInString(content[i:]) 101 + if !(content[i] == '\n' && i > 0 && content[i-1] == '\r') { 102 + boundaries = append(boundaries, i) 103 + } 104 + i += size 105 + } 106 + for _, off := range boundaries { 107 + line, col := LineCol(content, off) 108 + if got := Offset(content, line, col); got != off { 109 + t.Errorf("round trip at %d: Offset(LineCol(%d)) = %d", off, off, got) 110 + } 111 + } 112 +} 113 + 114 +func TestOffset_Clamps(t *testing.T) { 115 + content := "ab\ncd" 116 + if got := Offset(content, 5, 0); got != len(content) { 117 + t.Errorf("past EOF line: Offset = %d, want %d", got, len(content)) 118 + } 119 + // column beyond line end clamps to line end 120 + if got := Offset(content, 0, 99); got != 2 { 121 + t.Errorf("past EOL col: Offset = %d, want 2", got) 122 + } 123 +} 124 + 125 +func TestOffset_UTF16(t *testing.T) { 126 + // Cyrillic chars are 1 UTF-16 unit each; emoji are 2 127 + content := "Привет😀" 128 + if got := Offset(content, 0, 6); got != 12 { 129 + t.Errorf("Offset after Cyrillic = %d, want 12", got) 130 + } 131 + if got := Offset(content, 0, 7); got != 16 { 132 + t.Errorf("Offset after emoji (2 UTF-16 units) = %d, want 16", got) 133 + } 134 +} 135 + 136 +func TestPosition_RoundTrip(t *testing.T) { 137 + content := "2024-01-15 Супермаркет\r\n Витрати:Продукти ¥50\n" 138 + for _, off := range []int{0, 10, len("2024-01-15 Супермаркет"), len(content)} { 139 + p := Position(content, off) 140 + if got := Offset(content, int(p.Line), int(p.Character)); got != off { 141 + t.Errorf("round trip of %d = %d (pos %d:%d), want identity", off, got, p.Line, p.Character) 142 + } 143 + } 144 +}
M
internal/lsp/server.go
··· 3 3 import ( 4 4 "context" 5 5 "log/slog" 6 + "maps" 6 7 "sync" 7 8 8 9 "go.lsp.dev/protocol" ··· 11 12 "olexsmir.xyz/clerk/internal/analyzer" 12 13 "olexsmir.xyz/clerk/internal/linter" 13 14 "olexsmir.xyz/clerk/journal" 14 - "olexsmir.xyz/clerk/journal/ast" 15 15 "olexsmir.xyz/clerk/journal/printer" 16 16 ) 17 17 ··· 38 38 diagCancel context.CancelFunc 39 39 } 40 40 41 +func (s *server) analysis() *analyzer.Analysis { 42 + s.mu.Lock() 43 + a := s.current 44 + s.mu.Unlock() 45 + if a != nil { 46 + return a 47 + } 48 + return s.buildAnalysis() 49 +} 50 + 51 +func (s *server) buildAnalysis() *analyzer.Analysis { 52 + s.mu.Lock() 53 + docs := make(map[uri.URI]docState, len(s.openDocs)) 54 + maps.Copy(docs, s.openDocs) 55 + s.mu.Unlock() 56 + 57 + var a *analyzer.Analysis 58 + for duri, state := range docs { 59 + rj := s.loader.ResolveBytes(duri.Path(), []byte(state.text)) 60 + if a == nil { 61 + a = analyzer.Build(rj) 62 + } 63 + } 64 + return a 65 +} 66 + 41 67 type docState struct { 42 68 text string 43 69 version int32 44 70 languageID protocol.LanguageKind 45 - journal *ast.Journal // cached parse; nil on parse failure 71 + semTokens []semanticToken // cached semantic tokens 46 72 } 47 73 48 74 func (s *server) Initialize(ctx context.Context, params *protocol.InitializeParams) (*protocol.InitializeResult, error) { ··· 54 80 }, 55 81 Capabilities: protocol.ServerCapabilities{ 56 82 DocumentFormattingProvider: &protocol.DocumentFormattingOptions{}, 83 + CompletionProvider: &protocol.CompletionOptions{ 84 + TriggerCharacters: []string{":", "@"}, 85 + }, 57 86 TextDocumentSync: &protocol.TextDocumentSyncOptions{ 58 87 OpenClose: new(true), 59 88 Change: new(protocol.TextDocumentSyncKindFull),
A
internal/lsp/testdata/completion-crlf.txtar
··· 1 +-- in.journal -- 2 +2024-01-15 Grocery 3 + expenses:food $50 4 + expenses:taxi $30 5 + assets:cash 6 + 7 +2024-01-16 ^ 8 + expenses:f^ood $5 9 + assets:cash 10 +-- expect -- 11 +5:11 payee "" 12 + Grocery 13 +6:14 account "expenses:f" 14 + expenses:food
A
internal/lsp/testdata/completion-journal.txtar
··· 1 +-- in.journal -- 2 +2024-01-15 Grocery 3 + expenses:food $50 4 + expenses:taxi $30 5 + assets:cash 6 + 7 +2024-01-15 Grocery 8 + expenses:food $10 9 + assets:cash 10 + 11 +account assets:cash 12 + 13 +commodity USD 14 + 15 +payee Grocery 16 + 17 +tag client 18 + 19 +; client:acme 20 +; project:alpha 21 + 22 +2024-01-16 ^ 23 + expenses:f^ood $5 24 + assets:cash 25 + 26 +2024-01-16 Store | note 27 + 28 +account assets:c^ash 29 + 30 +commodity U^SD 31 + 32 +payee Gr^ocery 33 + 34 +tag cli^ent 35 + 36 +; cli^ent:x 37 +; proj^ect 38 + 39 +acc^ount 40 + 41 +-- expect -- 42 +20:11 payee "" 43 + Grocery 44 + Store 45 +21:12 account "expenses:f" 46 + expenses:food 47 +26:16 account "assets:c" 48 + assets:cash 49 +28:11 commodity "U" 50 + USD 51 +30:8 payee "Gr" 52 + Grocery 53 +32:7 tag "cli" 54 + client 55 +34:5 tag "cli" 56 + client 57 +35:6 tag "proj" 58 +37:3 directive "acc" 59 + account
A
internal/lsp/testdata/completion-unicode.txtar
··· 1 +-- in.journal -- 2 +2024-01-15 Супермаркет 3 + Витрати:Продукти ¥50 4 + Витрати:Транспорт ¥30 5 + Активи:Готівка 6 + 7 +2024-01-15 Продукти 8 + Витрати:Продукти €10 9 + Активи:Готівка 10 + 11 +2024-01-16 Супер^маркет 12 + Витрати:Про^дукти ¥20 13 + Активи:Готівка 14 + 15 +2024-01-16 Прод^укти 16 + Витрати:Тр^анспорт €5 17 + Активи:Готівка 18 + 19 +commodity ¥^元 20 + 21 +tag 客^户 22 + 23 +-- expect -- 24 +9:16 payee "Супер" 25 + Супермаркет 26 +10:13 account "Витрати:Про" 27 + Витрати:Продукти 28 +13:15 payee "Прод" 29 + Продукти 30 +14:12 account "Витрати:Тр" 31 + Витрати:Транспорт 32 +17:11 commodity "¥" 33 + ¥ 34 +19:5 tag "客" 35 + 客户
A
internal/lsp/textdocument_completion.go
··· 1 +package lsp 2 + 3 +import ( 4 + "context" 5 + "fmt" 6 + "math" 7 + "sort" 8 + "strings" 9 + "time" 10 + 11 + "go.lsp.dev/protocol" 12 + 13 + "olexsmir.xyz/clerk/internal/analyzer" 14 + "olexsmir.xyz/clerk/internal/lsp/fuzzy" 15 + "olexsmir.xyz/clerk/internal/lsp/lsputil" 16 + "olexsmir.xyz/clerk/journal/ast" 17 + "olexsmir.xyz/clerk/journal/lexer" 18 + "olexsmir.xyz/clerk/journal/token" 19 +) 20 + 21 +func (s *server) Completion(ctx context.Context, params *protocol.CompletionParams) (protocol.CompletionResult, error) { 22 + state, ok := s.getDocState(params.TextDocument.URI) 23 + if !ok { 24 + return &protocol.CompletionList{}, nil 25 + } 26 + cursor := lsputil.Offset(state.text, int(params.Position.Line), int(params.Position.Character)) 27 + if cursor > len(state.text) { 28 + return &protocol.CompletionList{}, nil 29 + } 30 + detectedCtx, start := detectCompletionCtx(state.text, cursor) 31 + if detectedCtx == cmplNone { 32 + return &protocol.CompletionList{}, nil 33 + } 34 + a := s.analysis() 35 + if a == nil { 36 + return &protocol.CompletionList{}, nil 37 + } 38 + return &protocol.CompletionList{ 39 + IsIncomplete: true, 40 + Items: cmplItems(a, detectedCtx, state.text, start, cursor), 41 + }, nil 42 +} 43 + 44 +const maxCompletionItems = 50 45 + 46 +type cmplCtx int 47 + 48 +const ( 49 + cmplNone cmplCtx = iota 50 + cmplAccount 51 + cmplPayee 52 + cmplCommodity 53 + cmplTagName 54 + cmplDirective 55 +) 56 + 57 +var directiveKeywords = []string{ 58 + "account", "include", "commodity", "payee", "decimal-mark", "alias", 59 + "apply", "end", "tag", "year", "D", "P", "N", "C", "Y", 60 +} 61 + 62 +func detectCompletionCtx(content string, cursor int) (cmplCtx, int) { 63 + toks := lexLine(content, cursor) 64 + lineStart, _ := lineBounds(content, cursor) 65 + 66 + if m := commentMarker(toks, cursor); m != -1 { 67 + return cmplTagContext(content, toks[m].Span.End.Offset, cursor) 68 + } 69 + if len(toks) == 0 { 70 + return cmplDirective, lineStart 71 + } 72 + 73 + switch toks[0].Type { 74 + case token.INDENT: 75 + return cmplPostingCtx(content, cursor, toks) 76 + case token.DATE: 77 + return cmplHeaderCtx(content, cursor, toks) 78 + case token.ACCOUNT, token.COMMODITY, token.PAYEE, token.TAG: 79 + return cmplDirectiveContext(cursor, lineStart, toks) 80 + case token.TEXT: 81 + return cmplDirective, lineStart // half-typed keyword or unparseable line 82 + } 83 + return cmplNone, cursor 84 +} 85 + 86 +func cmplPostingCtx(content string, cursor int, toks []token.Token) (cmplCtx, int) { 87 + if inDirectiveBody(content, cursor) { 88 + return cmplNone, cursor 89 + } 90 + fieldStart := toks[0].Span.End.Offset 91 + i := 1 92 + for i < len(toks) { 93 + switch toks[i].Type { 94 + case token.STAR, token.BANG, token.LPAREN, token.LBRACKET, token.WHITESPACE: 95 + fieldStart = toks[i].Span.End.Offset 96 + i++ 97 + default: 98 + goto run 99 + } 100 + } 101 +run: 102 + // account run: consecutive account-name segments and colons 103 + fieldEnd := fieldStart 104 + for ; i < len(toks); i++ { 105 + if toks[i].Type != token.TEXT && toks[i].Type != token.COLON { 106 + break 107 + } 108 + fieldEnd = toks[i].Span.End.Offset 109 + } 110 + if cursor <= fieldEnd && cursor >= fieldStart { 111 + return cmplAccount, fieldStart 112 + } 113 + if cursor > fieldEnd { 114 + if t := tokenUnder(toks, cursor); t != nil && (t.Type == token.COMMODITYMARK || t.Type == token.STRING) { 115 + start := t.Span.Start.Offset 116 + if t.Type == token.STRING { 117 + start++ // skip opening quote 118 + } 119 + return cmplCommodity, start 120 + } 121 + if strings.TrimSpace(content[fieldEnd:cursor]) == "" { 122 + return cmplCommodity, cursor 123 + } 124 + } 125 + return cmplNone, cursor 126 +} 127 + 128 +func cmplHeaderCtx(content string, cursor int, toks []token.Token) (cmplCtx, int) { 129 + // skip date, status, code, and whitespace - where the payee beginds 130 + fieldStart := toks[0].Span.End.Offset 131 + fieldEnd := fieldStart 132 + seen := false 133 + for i := 1; i < len(toks); i++ { 134 + t := toks[i] 135 + switch t.Type { 136 + case token.WHITESPACE, token.STAR, token.BANG, token.DATE, token.TIME, 137 + token.EQ, token.EQEQ, token.EQEQEQ: 138 + fieldStart = t.Span.End.Offset 139 + fieldEnd = t.Span.End.Offset 140 + case token.TEXT: 141 + lit := content[t.Span.Start.Offset:t.Span.End.Offset] 142 + if !seen && len(lit) >= 2 && lit[0] == '(' && lit[len(lit)-1] == ')' { 143 + fieldStart = t.Span.End.Offset // parenthesized code 144 + fieldEnd = t.Span.End.Offset 145 + continue 146 + } 147 + if !seen { 148 + fieldStart = t.Span.Start.Offset 149 + seen = true 150 + } 151 + fieldEnd = t.Span.End.Offset 152 + if p := strings.IndexByte(lit, '|'); p >= 0 { 153 + fieldEnd = t.Span.Start.Offset + p // "payee|note" keeps the pipe in the token 154 + return payeeAt(cursor, fieldStart, fieldEnd) 155 + } 156 + case token.STRING: 157 + if !seen { 158 + fieldStart = t.Span.Start.Offset + 1 // skip opening quote 159 + seen = true 160 + } 161 + fieldEnd = t.Span.End.Offset 162 + default: 163 + // PIPE, SEMICOLON, ... 164 + return payeeAt(cursor, fieldStart, fieldEnd) 165 + } 166 + } 167 + if seen { 168 + return payeeAt(cursor, fieldStart, fieldEnd) 169 + } 170 + // no payee yet: the payee field is the whitespace after the header meta 171 + if cursor >= fieldStart && strings.TrimSpace(content[fieldStart:cursor]) == "" { 172 + return cmplPayee, cursor 173 + } 174 + return cmplNone, cursor 175 +} 176 + 177 +func payeeAt(cursor, start, end int) (cmplCtx, int) { 178 + if cursor >= start && cursor <= end { 179 + return cmplPayee, start 180 + } 181 + return cmplNone, cursor 182 +} 183 + 184 +// cmplDirectiveContext classifies a directive line. keyword completion before the keyword ends, symbol completion in the value field after 185 +func cmplDirectiveContext(cursor, lineStart int, toks []token.Token) (cmplCtx, int) { 186 + kwEnd := toks[0].Span.End.Offset 187 + if cursor <= kwEnd { 188 + return cmplDirective, lineStart 189 + } 190 + start := kwEnd 191 + for i := 1; i < len(toks); i++ { 192 + if toks[i].Type == token.WHITESPACE || toks[i].Span.End.Offset <= kwEnd { 193 + continue 194 + } 195 + start = toks[i].Span.Start.Offset 196 + if toks[i].Type == token.STRING { 197 + start++ // skip opening quote 198 + } 199 + break 200 + } 201 + if start > cursor { 202 + start = cursor 203 + } 204 + switch toks[0].Type { 205 + case token.ACCOUNT: 206 + return cmplAccount, start 207 + case token.COMMODITY: 208 + return cmplCommodity, start 209 + case token.PAYEE: 210 + return cmplPayee, start 211 + case token.TAG: 212 + return cmplTagName, start 213 + } 214 + return cmplNone, cursor 215 +} 216 + 217 +// cmplTagContext completes tag names at the start of a comment, stopping at first ':'. 218 +func cmplTagContext(content string, commentStart, cursor int) (cmplCtx, int) { 219 + if strings.ContainsRune(content[commentStart:cursor], ':') { 220 + return cmplNone, cursor 221 + } 222 + keyStart := commentStart + lastSeparator(content[commentStart:cursor]) + 1 223 + return cmplTagName, keyStart 224 +} 225 + 226 +// commentMarker returns index of the first comment marker token at or before the cursor, or -1. 227 +func commentMarker(toks []token.Token, cursor int) int { 228 + for i, t := range toks { 229 + switch t.Type { 230 + case token.SEMICOLON, token.HASH, token.PERCENT: 231 + if t.Span.Start.Offset <= cursor { 232 + return i 233 + } 234 + case token.STAR: 235 + if i == 0 && t.Span.Start.Offset <= cursor { 236 + return i 237 + } 238 + } 239 + } 240 + return -1 241 +} 242 + 243 +func inDirectiveBody(content string, cursor int) bool { 244 + lineStart, _ := lineBounds(content, cursor) 245 + if toks := lexLine(content, lineStart); len(toks) == 0 || toks[0].Type != token.INDENT { 246 + return false 247 + } 248 + for lineStart > 0 { 249 + lineStart, _ = lineBounds(content, lineStart-1) 250 + toks := lexLine(content, lineStart) 251 + if len(toks) == 0 || toks[0].Type != token.INDENT { 252 + return len(toks) > 0 && (toks[0].Type == token.ACCOUNT || toks[0].Type == token.COMMODITY) 253 + } 254 + } 255 + return false 256 +} 257 + 258 +type cmplCand struct { 259 + label string 260 + score float64 261 + count int 262 + lastUsed ast.Date 263 +} 264 + 265 +// cmplItems ranks candidates for the content against typed pattern 266 +func cmplItems(a *analyzer.Analysis, ctx cmplCtx, content string, start, cursor int) []protocol.CompletionItem { 267 + pattern := content[start:cursor] 268 + 269 + var kind protocol.CompletionItemKind 270 + var cands []cmplCand 271 + switch ctx { 272 + case cmplAccount: 273 + kind = protocol.CompletionItemKindClass 274 + for name, info := range a.Accounts { 275 + cands = append(cands, cmplCand{label: name, count: info.UsedCount, lastUsed: info.LastUsed}) 276 + } 277 + case cmplPayee: 278 + kind = protocol.CompletionItemKindVariable 279 + for name, info := range a.Payees { 280 + cands = append(cands, cmplCand{label: name, count: info.UsedCount, lastUsed: info.LastUsed}) 281 + } 282 + case cmplCommodity: 283 + kind = protocol.CompletionItemKindValue 284 + for name, info := range a.Commodities { 285 + cands = append(cands, cmplCand{label: name, count: info.UsedCount, lastUsed: info.LastUsed}) 286 + } 287 + case cmplTagName: 288 + kind = protocol.CompletionItemKindProperty 289 + for _, name := range a.TagNames { 290 + cands = append(cands, cmplCand{label: name}) 291 + } 292 + case cmplDirective: 293 + kind = protocol.CompletionItemKindKeyword 294 + for _, name := range directiveKeywords { 295 + cands = append(cands, cmplCand{label: name}) 296 + } 297 + default: 298 + return nil 299 + } 300 + 301 + var newest ast.Date 302 + if n := len(a.Dates); n > 0 { 303 + newest = a.Dates[n-1] 304 + } 305 + ranked := cands[:0] 306 + for i := range cands { 307 + sc := fuzzy.Score(pattern, cands[i].label) 308 + if sc != 0 { 309 + sc *= 1 + math.Log1p(float64(cands[i].count)) 310 + if cands[i].count > 0 && cands[i].lastUsed.Year != 0 && newest.Year != 0 { 311 + days := daysBetween(cands[i].lastUsed, newest) 312 + sc *= 1 + 0.5*max(0, 1-float64(days)/365) 313 + } 314 + } 315 + cands[i].score = sc 316 + if sc != 0 { 317 + ranked = append(ranked, cands[i]) 318 + } 319 + } 320 + sort.Slice(ranked, func(i, j int) bool { 321 + if ranked[i].score != ranked[j].score { 322 + return ranked[i].score > ranked[j].score 323 + } 324 + if ranked[i].lastUsed != ranked[j].lastUsed { 325 + return ranked[i].lastUsed.Compare(ranked[j].lastUsed) > 0 326 + } 327 + return ranked[i].label < ranked[j].label 328 + }) 329 + if len(ranked) > maxCompletionItems { 330 + ranked = ranked[:maxCompletionItems] 331 + } 332 + 333 + replace := protocol.Range{ 334 + Start: lsputil.Position(content, start), 335 + End: lsputil.Position(content, cursor), 336 + } 337 + items := make([]protocol.CompletionItem, len(ranked)) 338 + for i, r := range ranked { 339 + it := protocol.CompletionItem{ 340 + Label: r.label, 341 + Kind: kind, 342 + SortText: protocol.NewOptional(fmt.Sprintf("%04d", i)), 343 + FilterText: protocol.NewOptional(r.label), 344 + TextEdit: &protocol.TextEdit{ 345 + Range: replace, 346 + NewText: r.label, 347 + }, 348 + } 349 + if r.count > 0 { 350 + it.Detail = protocol.NewOptional(fmt.Sprintf("%d uses", r.count)) 351 + } 352 + items[i] = it 353 + } 354 + return items 355 +} 356 + 357 +func daysBetween(a, b ast.Date) int { 358 + return int(time.Date(b.Year, time.Month(b.Month), b.Day, 0, 0, 0, 0, time.UTC). 359 + Sub(time.Date(a.Year, time.Month(a.Month), a.Day, 0, 0, 0, 0, time.UTC)). 360 + Hours() / 24) 361 +} 362 + 363 +// lineBounds returns the byte offsets of the line containing cursor 364 +func lineBounds(content string, cursor int) (start, end int) { 365 + start = cursor 366 + for start > 0 && content[start-1] != '\n' && content[start-1] != '\r' { 367 + start-- 368 + } 369 + end = start 370 + for end < len(content) && content[end] != '\n' && content[end] != '\r' { 371 + end++ 372 + } 373 + return start, end 374 +} 375 + 376 +func lastSeparator(s string) int { 377 + for i := len(s) - 1; i >= 0; i-- { 378 + switch s[i] { 379 + case ' ', '\t', ',': 380 + return i 381 + } 382 + } 383 + return -1 384 +} 385 + 386 +func lexLine(content string, cursor int) []token.Token { 387 + lineStart, lineEnd := lineBounds(content, cursor) 388 + l := lexer.New("", []byte(content[lineStart:lineEnd])) 389 + var out []token.Token 390 + for { 391 + t := l.Next() 392 + if t.Type == token.EOF || t.Type == token.NEWLINE { 393 + break 394 + } 395 + t.Span.Start.Offset += lineStart 396 + t.Span.End.Offset += lineStart 397 + out = append(out, t) 398 + } 399 + return out 400 +} 401 + 402 +func tokenUnder(toks []token.Token, cursor int) *token.Token { 403 + for i := range toks { 404 + t := &toks[i] 405 + if t.Span.Start.Offset <= cursor && cursor <= t.Span.End.Offset { 406 + return t 407 + } 408 + } 409 + return nil 410 +}
A
internal/lsp/textdocument_completion_test.go
··· 1 +package lsp 2 + 3 +import ( 4 + "fmt" 5 + "strings" 6 + "testing" 7 + "time" 8 + 9 + "go.lsp.dev/protocol" 10 + "go.lsp.dev/uri" 11 + 12 + "olexsmir.xyz/clerk/internal/analyzer" 13 + "olexsmir.xyz/clerk/internal/lsp/lsputil" 14 + "olexsmir.xyz/clerk/internal/testutil/golden" 15 + "olexsmir.xyz/clerk/journal" 16 +) 17 + 18 +func TestDetectCompletionCtx(t *testing.T) { 19 + tests := []struct { 20 + name string 21 + ctx cmplCtx 22 + in string 23 + want string 24 + }{ 25 + {"posting account", cmplAccount, " expenses:f^ood $50\n", "expenses:f"}, 26 + {"posting account empty", cmplAccount, " ^\n", ""}, 27 + {"posting account after colon", cmplAccount, " expenses:^food $50\n", "expenses:"}, 28 + {"posting amount commodity", cmplCommodity, " expenses:food $^50\n", "$"}, 29 + {"posting empty amount region", cmplCommodity, " expenses:food ^\n", ""}, 30 + {"posting commodity word", cmplCommodity, " expenses:food U^SD\n", "U"}, 31 + {"posting amount number", cmplNone, " expenses:food $5^0\n", ""}, 32 + {"posting cost quantity", cmplNone, " expenses:food $50 @^ 1.5\n", ""}, 33 + {"posting status", cmplAccount, " * expenses:f^ood $50\n", "expenses:f"}, 34 + {"posting virtual", cmplAccount, " (expenses:f^ood) $50\n", "expenses:f"}, 35 + {"posting comment tag", cmplTagName, " expenses:food ; clie^nt:x\n", "clie"}, 36 + {"posting comment tag value", cmplNone, " expenses:food ; client:^x\n", ""}, 37 + {"header payee", cmplPayee, "2024-01-15 acm^e\n assets:cash\n", "acm"}, 38 + {"header payee empty", cmplPayee, "2024-01-15 ^\n", ""}, 39 + {"header payee right after date", cmplPayee, "2024-01-15^\n", ""}, 40 + {"header status and code", cmplPayee, "2024-01-15 * (123) gro^cer\n", "gro"}, 41 + {"header second date", cmplPayee, "2024-01-15=2024-01-16 acm^e\n", "acm"}, 42 + {"header quoted payee", cmplPayee, "2024-01-15 \"ac^me\"\n", "ac"}, 43 + {"header pipe note", cmplNone, "2024-01-15 acme | no^te\n", ""}, 44 + {"header pipe inline", cmplNone, "2024-01-15 acme|note x^y\n", ""}, 45 + {"header inline comment", cmplTagName, "2024-01-15 ; foo^", "foo"}, 46 + {"directive keyword partial", cmplDirective, "acc^ount expenses\n", "acc"}, 47 + {"directive keyword empty line", cmplDirective, "\n^", ""}, 48 + {"account directive value", cmplAccount, "account exp^enses\n", "exp"}, 49 + {"commodity directive value", cmplCommodity, "commodity U^SD\n", "U"}, 50 + {"payee directive value", cmplPayee, "payee ac^me\n", "ac"}, 51 + {"tag directive value", cmplTagName, "tag pro^ject\n", "pro"}, 52 + {"comment tag", cmplTagName, "; clie^nt:x\n", "clie"}, 53 + {"comment value", cmplNone, "; client:x^yz\n", ""}, 54 + {"comment plain text", cmplTagName, "; groc^eries\n", "groc"}, 55 + {"subdirective ignored", cmplNone, "account expenses\n no^te ignore\n", ""}, 56 + {"periodic header", cmplNone, "~ monthly^ budget\n", ""}, 57 + {"cjk posting", cmplAccount, " 支出:食^物 50\n", "支出:食"}, 58 + {"ukrainian payee", cmplPayee, "2024-01-15 прод^укти\n", "прод"}, 59 + {"crlf posting", cmplAccount, "2024-01-15 x\r\n expenses:f^ood $50\r\n", "expenses:f"}, 60 + {"crlf header", cmplPayee, "2024-01-15 acm^e\r\n assets:cash\r\n", "acm"}, 61 + } 62 + for _, tt := range tests { 63 + t.Run(tt.name, func(t *testing.T) { 64 + cont := tt.in 65 + i := strings.Index(cont, "^") 66 + if i < 0 { 67 + t.Fatal("no cursor marker '^' in content") 68 + } 69 + 70 + cont = cont[:i] + cont[i+1:] 71 + ctx, start := detectCompletionCtx(cont, i) 72 + prefix := cont[start:i] 73 + 74 + if ctx != tt.ctx { 75 + t.Errorf("ctx = %v, want %v", ctx, tt.ctx) 76 + } 77 + if prefix != tt.want { 78 + t.Errorf("prefix = %q, want %q", prefix, tt.want) 79 + } 80 + }) 81 + } 82 +} 83 + 84 +func (c cmplCtx) String() string { 85 + switch c { 86 + case cmplNone: 87 + return "none" 88 + case cmplAccount: 89 + return "account" 90 + case cmplPayee: 91 + return "payee" 92 + case cmplCommodity: 93 + return "commodity" 94 + case cmplTagName: 95 + return "tag" 96 + case cmplDirective: 97 + return "directive" 98 + default: 99 + return "?" 100 + } 101 +} 102 + 103 +func TestDetectCompletion_NoJournal(t *testing.T) { 104 + // detection must not depend on a parse 105 + i := strings.Index(" no^te foo\n", "^") 106 + content := " note foo\n" 107 + ctx, _ := detectCompletionCtx(content, i) 108 + if ctx != cmplAccount { 109 + t.Errorf("ctx = %v, want account", ctx) 110 + } 111 +} 112 + 113 +func TestDetectCompletion_Subdirective(t *testing.T) { 114 + // the lexical rule suppresses completion on subdirective lines without a parse 115 + i := strings.Index("account expenses\n no^te ignore\n", "^") 116 + content := "account expenses\n note ignore\n" 117 + if ctx, _ := detectCompletionCtx(content, i); ctx != cmplNone { 118 + t.Errorf("subdirective: ctx = %v, want none", ctx) 119 + } 120 + // a posting line after a transaction header is still an account context 121 + i = strings.Index("2024-01-15 acme\n expe^nses:food $50\n", "^") 122 + content = "2024-01-15 acme\n expenses:food $50\n" 123 + if ctx, _ := detectCompletionCtx(content, i); ctx != cmplAccount { 124 + t.Errorf("posting: ctx = %v, want account", ctx) 125 + } 126 + // a blank line ends the directive body; a whitespace-only line does not 127 + i = strings.Index("account expenses\n note: x\n\n no^te\n", "^") 128 + content = "account expenses\n note: x\n\n note\n" 129 + if ctx, _ := detectCompletionCtx(content, i); ctx != cmplAccount { 130 + t.Errorf("after blank line: ctx = %v, want account", ctx) 131 + } 132 + i = strings.Index("account expenses\n note: x\n \n no^te\n", "^") 133 + content = "account expenses\n note: x\n \n note\n" 134 + if ctx, _ := detectCompletionCtx(content, i); ctx != cmplNone { 135 + t.Errorf("whitespace-only line keeps body: ctx = %v, want none", ctx) 136 + } 137 +} 138 + 139 +func TestCompleteItems_NoTransactions(t *testing.T) { 140 + // directives-only doc: empty a.Dates must not panic the ranking 141 + content := "account expenses:food\n\n^" 142 + i := strings.Index(content, "^") 143 + content = content[:i] 144 + a := analyzer.Build(journal.NewLoader().ResolveBytes("", []byte(content))) 145 + if len(a.Dates) != 0 { 146 + t.Fatalf("setup: want 0 dates, got %d", len(a.Dates)) 147 + } 148 + items := cmplItems(a, cmplAccount, content, len(content), len(content)) 149 + if len(items) == 0 { 150 + t.Fatal("expected the directive-defined account to complete") 151 + } 152 + if items[0].Label != "expenses:food" { 153 + t.Errorf("label = %q, want expenses:food", items[0].Label) 154 + } 155 +} 156 + 157 +// Golden 158 + 159 +func TestCompletionTxtar(t *testing.T) { 160 + tests := []string{ 161 + "completion-journal", 162 + "completion-unicode", 163 + "completion-crlf", 164 + } 165 + 166 + for _, tt := range tests { 167 + ar := golden.Read(t, tt) 168 + 169 + t.Run(tt, func(t *testing.T) { 170 + content := string(ar.Get("in.journal")) 171 + 172 + var cursors []int 173 + for { 174 + m := strings.Index(content, "^") 175 + if m < 0 { 176 + break 177 + } 178 + cursors = append(cursors, m) 179 + content = content[:m] + content[m+1:] 180 + } 181 + if len(cursors) == 0 { 182 + t.Fatal("no '^' markers in in.journal") 183 + } 184 + 185 + srv := NewServer("test") 186 + srv.server.openDoc(uri.URI("file:///test.journal"), content, 1, "journal") 187 + srv.server.current = analyzer.Build(srv.server.loader.ResolveBytes("", []byte(content))) 188 + 189 + var b strings.Builder 190 + for _, c := range cursors { 191 + line, col := lsputil.LineCol(content, c) 192 + res, err := srv.server.Completion(t.Context(), &protocol.CompletionParams{ 193 + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ 194 + TextDocument: protocol.TextDocumentIdentifier{URI: uri.URI("file:///test.journal")}, 195 + Position: protocol.Position{Line: uint32(line), Character: uint32(col)}, 196 + }, 197 + }) 198 + if err != nil { 199 + t.Fatal(err) 200 + } 201 + ctx, start := detectCompletionCtx(content, c) 202 + fmt.Fprintf(&b, "%d:%d %s %q\n", line, col, ctx, content[start:c]) 203 + list, ok := res.(*protocol.CompletionList) 204 + if !ok { 205 + t.Fatalf("Completion returned %T, want *protocol.CompletionList", res) 206 + } 207 + for _, item := range list.Items { 208 + fmt.Fprintf(&b, " %s\n", item.Label) 209 + } 210 + } 211 + golden.Assert(t, ar, b.String()) 212 + }) 213 + } 214 +} 215 + 216 +// Benchmark 217 + 218 +func BenchmarkCompletion(b *testing.B) { 219 + path := "../../journal/testdata/journals/actual-1ktxns-100accts.journal" 220 + rj, err := journal.NewLoader().Resolve(path) 221 + if err != nil { 222 + b.Fatal(err) 223 + } 224 + a := analyzer.Build(rj) 225 + content := string(rj.Occurrences[0].Src) 226 + 227 + srv := NewServer("test") 228 + srv.server.openDoc(uri.URI("file:///test.journal"), content, 1, "journal") 229 + srv.server.current = a 230 + 231 + for _, tc := range []struct { 232 + name string 233 + pos int 234 + }{ 235 + {"1k txns, account", strings.Index(content, "\n 1:2:3 ") + len("\n ") + 2}, 236 + {"1k txns, empty payee", strings.Index(content, "transaction 1") + len("transaction ")}, 237 + {"1k txns, commodity", strings.Index(content, "2 B @@") + len("2 B")}, 238 + } { 239 + b.Run(tc.name, func(b *testing.B) { 240 + line, col := lsputil.LineCol(content, tc.pos) 241 + params := &protocol.CompletionParams{ 242 + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ 243 + TextDocument: protocol.TextDocumentIdentifier{URI: uri.URI("file:///test.journal")}, 244 + Position: protocol.Position{Line: uint32(line), Character: uint32(col)}, 245 + }, 246 + } 247 + // warm up: first request parses the journal lazily 248 + if _, err := srv.server.Completion(b.Context(), params); err != nil { 249 + b.Fatal(err) 250 + } 251 + b.ReportAllocs() 252 + b.ResetTimer() 253 + for b.Loop() { 254 + if _, err := srv.server.Completion(b.Context(), params); err != nil { 255 + b.Fatal(err) 256 + } 257 + } 258 + // guard: line-local lexing must stay far below the old whole-file 259 + // relex (~3ms on this file); the measured target is sub-ms 260 + if avg := b.Elapsed() / time.Duration(b.N); avg > 2*time.Millisecond { 261 + b.Fatalf("completion %v/op: whole-file relex regression", avg) 262 + } 263 + }) 264 + } 265 +}
M
internal/lsp/textdocument_semantic.go
··· 3 3 import ( 4 4 "context" 5 5 "slices" 6 + "unicode/utf8" 6 7 7 8 "go.lsp.dev/protocol" 8 9 "go.lsp.dev/uri" ··· 46 47 } 47 48 48 49 func (s *server) tokensForDoc(doc uri.URI) ([]semanticToken, bool) { 49 - state, ok := s.getDocState(doc) 50 + s.mu.Lock() 51 + defer s.mu.Unlock() 52 + st, ok := s.openDocs[doc] 50 53 if !ok { 51 54 return nil, false 52 55 } 53 - return tokenizeForSemantics(state.text, state.journal), true 56 + if st.semTokens != nil { 57 + return st.semTokens, true 58 + } 59 + st.semTokens = tokenizeForSemantics(st.text, parseJournalStr(st.text)) 60 + s.openDocs[doc] = st 61 + return st.semTokens, true 54 62 } 55 63 56 64 // Implementation ··· 109 117 } 110 118 111 119 func tokenizeForSemantics(content string, j *ast.Journal) []semanticToken { 120 + var raw []rawSpan 121 + emit := func(s token.Span, tokType, mods uint32) { 122 + if s.Start.Offset >= s.End.Offset { 123 + return 124 + } 125 + raw = append(raw, rawSpan{s, tokType, mods}) 126 + } 112 127 if j == nil || len(j.Errors) > 0 { 113 - return semLexerFallback(content) 128 + semLexerFallback(content, emit) 129 + } else { 130 + for _, e := range j.Entries { 131 + visitEntry(content, e, emit) 132 + } 114 133 } 115 - return visitEntries(content, j) 134 + return rawToSemanticTokens(content, raw) 116 135 } 117 136 118 -// visitEntries walks the AST and emits semantic tokens. 119 -func visitEntries(content string, j *ast.Journal) []semanticToken { 120 - var out []semanticToken 121 - emit := func(s token.Span, tokType uint32, mods uint32) { 122 - if s.Start.Offset >= s.End.Offset { 123 - return 137 +// rawSpan is a source span tagged with semantic token 138 +type rawSpan struct { 139 + span token.Span 140 + tok, mods uint32 141 +} 142 + 143 +func rawToSemanticTokens(content string, raw []rawSpan) []semanticToken { 144 + if len(raw) == 0 { 145 + return nil 146 + } 147 + slices.SortFunc(raw, func(a, b rawSpan) int { return a.span.Start.Offset - b.span.Start.Offset }) 148 + out := make([]semanticToken, len(raw)) 149 + line, col, cursor := 0, 0, 0 150 + advance := func(end int) { 151 + for cursor < end { 152 + r, size := utf8.DecodeRuneInString(content[cursor:]) 153 + if r == utf8.RuneError && size <= 1 { 154 + break 155 + } 156 + if r == '\r' { 157 + cursor += size 158 + if cursor < len(content) && content[cursor] == '\n' { 159 + cursor++ 160 + } 161 + line++ 162 + col = 0 163 + continue 164 + } 165 + if r == '\n' { 166 + cursor += size 167 + line++ 168 + col = 0 169 + continue 170 + } 171 + cursor += size 172 + col += utf16Units(r) 124 173 } 125 - out = append(out, semanticTokenAt(content, s, tokType, mods)) 126 174 } 127 - for _, e := range j.Entries { 128 - visitEntry(content, e, emit) 175 + for i, t := range raw { 176 + if cursor < t.span.Start.Offset { 177 + advance(t.span.Start.Offset) 178 + } 179 + out[i] = semanticToken{ 180 + line: uint32(line), 181 + col: uint32(col), 182 + length: uint32(lsputil.Utf16Len(content, t.span.Start.Offset, t.span.End.Offset)), 183 + tokenType: t.tok, 184 + modifiers: t.mods, 185 + } 186 + advance(t.span.End.Offset) 129 187 } 130 188 return out 131 189 } 132 190 133 -// semanticTokenAt converts a source span into a semantic token. 134 -func semanticTokenAt(content string, s token.Span, tokType, mods uint32) semanticToken { 135 - line, col := lsputil.LineCol(content, s.Start.Offset) 136 - length := lsputil.Utf16Len(content, s.Start.Offset, s.End.Offset) 137 - return semanticToken{ 138 - line: uint32(line), 139 - col: uint32(col), 140 - length: uint32(length), 141 - tokenType: tokType, 142 - modifiers: mods, 191 +func utf16Units(r rune) int { 192 + if r >= 0x10000 && r <= 0x10FFFF { 193 + return 2 143 194 } 195 + return 1 144 196 } 145 197 146 198 func visitEntry(content string, e ast.Entry, emit semEmitFn) { ··· 474 526 } 475 527 476 528 // semLexerFallback produces semantic tokens using only the lexer (for unparseable documents). 477 -func semLexerFallback(content string) []semanticToken { 529 +func semLexerFallback(content string, emit semEmitFn) { 478 530 l := lexer.New("", []byte(content)) 479 - var out []semanticToken 480 531 481 532 var commentStart, commentEnd int // 0 = not inside a comment line 482 533 take := func(span token.Span, tokType uint32, mods uint32) { 483 - out = append(out, semanticTokenAt(content, span, tokType, mods)) 534 + emit(span, tokType, mods) 484 535 } 485 536 for { 486 537 tok := l.Next() ··· 536 587 } 537 588 take(tok.Span, tokType, 0) 538 589 } 539 - return out 540 590 } 541 591 542 592 func encodeSemTokens(tokens []semanticToken) []uint32 {
M
internal/lsp/textdocument_sync.go
··· 36 36 // Document management 37 37 38 38 func (s *server) openDoc(u uri.URI, text string, version int32, langID protocol.LanguageKind) { 39 - j := parseJournalStr(text) 40 39 s.mu.Lock() 41 40 s.openDocs[u] = docState{ 42 41 text: text, 43 42 version: version, 44 43 languageID: langID, 45 - journal: j, 46 44 } 47 45 s.mu.Unlock() 48 46 } ··· 59 57 switch ev := ch.(type) { 60 58 case *protocol.TextDocumentContentChangeWholeDocument: 61 59 state.text = ev.Text 62 - state.journal = parseJournalStr(ev.Text) 60 + state.semTokens = nil 63 61 case *protocol.TextDocumentContentChangePartial: 64 62 _ = ev // TODO: incremental edit support 65 63 }
M
journal/ast/ast.go
··· 32 32 Span token.Span 33 33 } 34 34 35 +// Compare returns -1 if d is before other, 0 if equal, 1 if after. 36 +func (d Date) Compare(other Date) int { 37 + if d.Year != other.Year { 38 + if d.Year < other.Year { 39 + return -1 40 + } 41 + return 1 42 + } 43 + if d.Month != other.Month { 44 + if d.Month < other.Month { 45 + return -1 46 + } 47 + return 1 48 + } 49 + if d.Day != other.Day { 50 + if d.Day < other.Day { 51 + return -1 52 + } 53 + return 1 54 + } 55 + return 0 56 +} 57 + 35 58 type Time struct { 36 59 Hour, Minute, Second int 37 60 Span token.Span
A
journal/ast/ast_test.go
··· 1 +package ast 2 + 3 +import "testing" 4 + 5 +func TestDateCompare(t *testing.T) { 6 + jan1 := Date{Year: 2024, Month: 1, Day: 1} 7 + jan2 := Date{Year: 2024, Month: 1, Day: 2} 8 + feb1 := Date{Year: 2024, Month: 2, Day: 1} 9 + nextYear := Date{Year: 2025, Month: 1, Day: 1} 10 + 11 + tests := []struct { 12 + a, b Date 13 + want int 14 + }{ 15 + {jan1, jan1, 0}, 16 + {jan2, jan1, 1}, 17 + {jan1, jan2, -1}, 18 + {feb1, jan2, 1}, 19 + {jan2, feb1, -1}, 20 + {nextYear, feb1, 1}, 21 + {feb1, nextYear, -1}, 22 + } 23 + for _, c := range tests { 24 + if got := c.a.Compare(c.b); got != c.want { 25 + t.Errorf("Compare(%v, %v) = %d, want %d", c.a, c.b, got, c.want) 26 + } 27 + } 28 +}
M
journal/lexer/lexer.go
··· 161 161 } 162 162 163 163 func (l *Lexer) lexComment() token.Token { 164 - if l.ch == '\n' || l.ch == 0 { 164 + if l.ch == '\n' || l.ch == '\r' || l.ch == 0 { 165 165 l.mode = ModeDefault 166 + if l.ch == '\r' { 167 + l.col = 0 168 + l.advance() 169 + } 166 170 return l.lexNewline() 167 171 } 168 172 ··· 170 174 l.lexWhitespace() 171 175 } 172 176 173 - if l.ch == '\n' || l.ch == 0 { 177 + if l.ch == '\n' || l.ch == '\r' || l.ch == 0 { 174 178 l.mode = ModeDefault 179 + if l.ch == '\r' { 180 + l.col = 0 181 + l.advance() 182 + } 175 183 return l.lexNewline() 176 184 } 177 185 178 186 s := l.save() 179 - for l.ch != '\n' && l.ch != 0 { 187 + for l.ch != '\n' && l.ch != '\r' && l.ch != 0 { 180 188 l.advance() 181 189 } 182 190 return token.Token{Type: token.TEXT, Literal: string(l.input[s.offset:l.pos]), Span: l.span(s)} ··· 243 251 return l.Next() 244 252 } 245 253 s := l.save() 246 - for l.ch != 0 && l.ch != '\n' && l.ch != ';' { 254 + for l.ch != 0 && l.ch != '\n' && l.ch != '\r' && l.ch != ';' { 247 255 l.advance() 248 256 } 249 257 lit := string(l.input[s.offset:l.pos]) ··· 304 312 l.postingExpectAccount = false 305 313 l.mode = ModeDefault 306 314 return l.lexNewline() 315 + case l.ch == '\r': 316 + l.postingExpectAccount = false 317 + l.col = 0 318 + l.advance() 319 + l.mode = ModeDefault 320 + return l.lexNewline() 307 321 case l.ch == ';': 308 322 l.postingExpectAccount = false 309 323 l.mode = ModeComment ··· 360 374 switch l.ch { 361 375 case '\n', 0: 362 376 l.mode = ModeDefault 377 + return l.lexNewline() 378 + case '\r': 379 + l.mode = ModeDefault 380 + l.col = 0 381 + l.advance() 363 382 return l.lexNewline() 364 383 case ';': 365 384 l.mode = ModeComment ··· 468 487 func (l *Lexer) lexText() token.Token { 469 488 s := l.save() 470 489 l.advance() 471 - for l.ch != '\n' && l.ch != ';' && l.ch != 0 && l.ch != ' ' && l.ch != '\t' { 490 + for l.ch != '\n' && l.ch != '\r' && l.ch != ';' && l.ch != 0 && l.ch != ' ' && l.ch != '\t' { 472 491 l.advance() 473 492 } 474 493 lit := string(l.input[s.offset:l.pos]) ··· 479 498 // stops at any whitespace, supports multi-word names("Taxi Fare"). 480 499 func (l *Lexer) lexAccountNameDirective() token.Token { 481 500 s := l.save() 482 - for l.ch != '\n' && l.ch != ';' && l.ch != 0 && l.ch != ')' && l.ch != ']' && l.ch != ':' && l.ch != ' ' && l.ch != '\t' { 501 + for l.ch != '\n' && l.ch != '\r' && l.ch != ';' && l.ch != 0 && l.ch != ')' && l.ch != ']' && l.ch != ':' && l.ch != ' ' && l.ch != '\t' { 483 502 l.advance() 484 503 } 485 504 lit := string(l.input[s.offset:l.pos]) ··· 490 509 // stops at two consecutive spaces. 491 510 func (l *Lexer) lexAccountNamePosting() token.Token { 492 511 s := l.save() 493 - for l.ch != '\n' && l.ch != ';' && l.ch != 0 && l.ch != ')' && l.ch != ']' && l.ch != ':' { 512 + for l.ch != '\n' && l.ch != '\r' && l.ch != ';' && l.ch != 0 && l.ch != ')' && l.ch != ']' && l.ch != ':' { 494 513 if l.isTwoSpaces() { 495 514 break 496 515 } ··· 506 525 func (l *Lexer) lexParenExpr() token.Token { 507 526 s := l.save() 508 527 depth := 0 509 - for l.ch != '\n' && l.ch != 0 { 528 + for l.ch != '\n' && l.ch != '\r' && l.ch != 0 { 510 529 if l.ch == '(' { 511 530 depth++ 512 531 } else if l.ch == ')' {