10 files changed,
584 insertions(+),
32 deletions(-)
Author:
Oleksandr Smirnov
olexsmir@gmail.com
Committed at:
2026-09-03 16:19:46 +0300
Authored at:
2026-09-03 14:18:10 +0300
Change ID:
oovkszqtyqslormwwtornzqoykkmolkl
Parent:
7b8e468
jump to
M
internal/lsp/server.go
··· 122 122 }, 123 123 TextDocumentSync: &protocol.TextDocumentSyncOptions{ 124 124 OpenClose: new(true), 125 - Change: new(protocol.TextDocumentSyncKindFull), 125 + Change: new(protocol.TextDocumentSyncKindIncremental), 126 126 }, 127 127 SemanticTokensProvider: &protocol.SemanticTokensOptions{ 128 128 Legend: getSemanticTokensLegend(),
A
internal/lsp/testdata/semantic-incremental-append.txtar
··· 1 +-- in.journal -- 2 +2024-01-01 groceries $10 3 + expenses:food $10 4 + assets:cash 5 + 6 +2024-01-02 salary $100 7 + assets:bank $100 8 + income:salary 9 + 10 +2024-01-03 rent $50 11 + expenses:rent $50 12 + assets:cash 13 + 14 +-- edited.journal -- 15 +2024-01-01 groceries $10 16 + expenses:food $10 17 + assets:cash 18 + 19 +2024-01-02 salary $100 20 + assets:bank $100 21 + income:salary 22 + 23 +2024-01-03 rent $50 24 + expenses:rent $50 25 + assets:cash 26 + 27 +2024-01-04 taxi $5 28 + expenses:taxi $5 29 + assets:cash 30 + 31 +-- expect -- 32 +0:0+10 class 33 +0:11+14 property 34 +1:4+13 namespace 35 +1:19+1 type 36 +1:20+2 number 37 +2:4+11 namespace 38 +4:0+10 class 39 +4:11+12 property 40 +5:4+11 namespace 41 +5:17+1 type 42 +5:18+3 number 43 +6:4+13 namespace 44 +8:0+10 class 45 +8:11+9 property 46 +9:4+13 namespace 47 +9:19+1 type 48 +9:20+2 number 49 +10:4+11 namespace 50 +12:0+10 class 51 +12:11+8 property 52 +13:4+13 namespace 53 +13:19+1 type 54 +13:20+1 number 55 +14:4+11 namespace
A
internal/lsp/testdata/semantic-incremental-delete.txtar
··· 1 +-- in.journal -- 2 +2024-01-01 groceries $10 3 + expenses:food $10 4 + assets:cash 5 + 6 +2024-01-02 salary $100 7 + assets:bank $100 8 + income:salary 9 + 10 +2024-01-03 rent $50 11 + expenses:rent $50 12 + assets:cash 13 + 14 +-- edited.journal -- 15 +2024-01-02 salary $100 16 + assets:bank $100 17 + income:salary 18 + 19 +2024-01-03 rent $50 20 + expenses:rent $50 21 + assets:cash 22 + 23 +-- expect -- 24 +0:0+10 class 25 +0:11+12 property 26 +1:4+11 namespace 27 +1:17+1 type 28 +1:18+3 number 29 +2:4+13 namespace 30 +4:0+10 class 31 +4:11+9 property 32 +5:4+13 namespace 33 +5:19+1 type 34 +5:20+2 number 35 +6:4+11 namespace
A
internal/lsp/testdata/semantic-incremental-header.txtar
··· 1 +-- in.journal -- 2 +2024-01-01 groceries $10 3 + expenses:food $10 4 + assets:cash 5 + 6 +2024-01-02 salary $100 7 + assets:bank $100 8 + income:salary 9 + 10 +2024-01-03 rent $50 11 + expenses:rent $50 12 + assets:cash 13 + 14 +-- edited.journal -- 15 +3024-01-01 groceries $10 16 + expenses:food $10 17 + assets:cash 18 + 19 +2024-01-02 salary $100 20 + assets:bank $100 21 + income:salary 22 + 23 +2024-01-03 rent $50 24 + expenses:rent $50 25 + assets:cash 26 + 27 +-- expect -- 28 +0:0+10 class 29 +0:11+14 property 30 +1:4+13 namespace 31 +1:19+1 type 32 +1:20+2 number 33 +2:4+11 namespace 34 +4:0+10 class 35 +4:11+12 property 36 +5:4+11 namespace 37 +5:17+1 type 38 +5:18+3 number 39 +6:4+13 namespace 40 +8:0+10 class 41 +8:11+9 property 42 +9:4+13 namespace 43 +9:19+1 type 44 +9:20+2 number 45 +10:4+11 namespace
A
internal/lsp/testdata/semantic-incremental-inline.txtar
··· 1 +-- in.journal -- 2 +2024-01-01 groceries $10 3 + expenses:food $10 4 + assets:cash 5 + 6 +2024-01-02 salary $100 7 + assets:bank $100 8 + income:salary 9 + 10 +2024-01-03 rent $50 11 + expenses:rent $50 12 + assets:cash 13 + 14 +-- edited.journal -- 15 +2024-01-01 groceries $10 16 + expenses:food $10 17 + assets:bank 18 + 19 +2024-01-02 salary $100 20 + assets:bank $100 21 + income:salary 22 + 23 +2024-01-03 rent $50 24 + expenses:rent $50 25 + assets:cash 26 + 27 +-- expect -- 28 +0:0+10 class 29 +0:11+14 property 30 +1:4+13 namespace 31 +1:19+1 type 32 +1:20+2 number 33 +2:4+11 namespace 34 +4:0+10 class 35 +4:11+12 property 36 +5:4+11 namespace 37 +5:17+1 type 38 +5:18+3 number 39 +6:4+13 namespace 40 +8:0+10 class 41 +8:11+9 property 42 +9:4+13 namespace 43 +9:19+1 type 44 +9:20+2 number 45 +10:4+11 namespace
A
internal/lsp/testdata/semantic-incremental-insert.txtar
··· 1 +-- in.journal -- 2 +2024-01-01 groceries $10 3 + expenses:food $10 4 + assets:cash 5 + 6 +2024-01-02 salary $100 7 + assets:bank $100 8 + income:salary 9 + 10 +2024-01-03 rent $50 11 + expenses:rent $50 12 + assets:cash 13 + 14 +-- edited.journal -- 15 +2024-01-01 groceries $10 16 + expenses:food $10 17 + assets:cash 18 + 19 +2024-01-05 new $20 20 + expenses:new $20 21 + assets:cash 22 + 23 +2024-01-02 salary $100 24 + assets:bank $100 25 + income:salary 26 + 27 +2024-01-03 rent $50 28 + expenses:rent $50 29 + assets:cash 30 + 31 +-- expect -- 32 +0:0+10 class 33 +0:11+14 property 34 +1:4+13 namespace 35 +1:19+1 type 36 +1:20+2 number 37 +2:4+11 namespace 38 +4:0+10 class 39 +4:11+8 property 40 +5:4+12 namespace 41 +5:18+1 type 42 +5:19+2 number 43 +6:4+11 namespace 44 +8:0+10 class 45 +8:11+12 property 46 +9:4+11 namespace 47 +9:17+1 type 48 +9:18+3 number 49 +10:4+13 namespace 50 +12:0+10 class 51 +12:11+9 property 52 +13:4+13 namespace 53 +13:19+1 type 54 +13:20+2 number 55 +14:4+11 namespace
A
internal/lsp/testdata/semantic-incremental-recover.txtar
··· 1 +-- in.journal -- 2 +2024-01-01 groceries $10 3 + expenses:food $10 4 + assets:cash 5 + 6 +2024-01-02 salary $100 7 + assets:bank $100 8 + income:salary 9 + 10 +2024-01-03 rent $50 11 + expenses:rent $50 12 + assets:cash 13 +not a valid transaction !!! 14 + 15 +-- edited.journal -- 16 +2024-01-01 groceries $10 17 + expenses:food $10 18 + assets:cash 19 + 20 +2024-01-02 salary $100 21 + assets:bank $100 22 + income:salary 23 + 24 +2024-01-03 rent $50 25 + expenses:rent $50 26 + assets:cash 27 + 28 +-- expect -- 29 +0:0+10 class 30 +0:11+14 property 31 +1:4+13 namespace 32 +1:19+1 type 33 +1:20+2 number 34 +2:4+11 namespace 35 +4:0+10 class 36 +4:11+12 property 37 +5:4+11 namespace 38 +5:17+1 type 39 +5:18+3 number 40 +6:4+13 namespace 41 +8:0+10 class 42 +8:11+9 property 43 +9:4+13 namespace 44 +9:19+1 type 45 +9:20+2 number 46 +10:4+11 namespace
M
internal/lsp/textdocument_semantic_tokens.go
··· 3 3 import ( 4 4 "context" 5 5 "slices" 6 + "strconv" 6 7 "unicode/utf8" 7 8 8 9 "go.lsp.dev/protocol" ··· 30 31 s.mu.RLock() 31 32 st, ok := s.openDocs[u] 32 33 s.mu.RUnlock() 33 - if !ok || st.semGen == 0 || params.PreviousResultID != st.resultID() { 34 + if !ok || st.sem == nil || st.sem.gen == 0 || params.PreviousResultID != st.sem.resultID() { 34 35 return s.semanticTokensFullResult(u), nil 35 36 } 36 37 ··· 38 39 if !ok { 39 40 return &protocol.SemanticTokens{}, nil 40 41 } 41 - edits := semanticTokensEdits(st.semBaseline, data) 42 + edits := semanticTokensEdits(st.sem.baseline, data) 42 43 if len(edits) == 0 { 43 - return &protocol.SemanticTokensDelta{ResultID: new(st.resultID()), Edits: []protocol.SemanticTokensEdit{}}, nil 44 + return &protocol.SemanticTokensDelta{ResultID: new(st.sem.resultID()), Edits: []protocol.SemanticTokensEdit{}}, nil 44 45 } 45 46 46 47 rid, ok := s.storeSemResult(u, data) ··· 90 91 return encodeSemTokens(tokens), true 91 92 } 92 93 94 +type semCache struct { 95 + entries []semEntry // tokens per entry of the current text 96 + baseline []uint32 // encoded data of the last response; diff baseline for the next delta request 97 + gen uint64 // increments per response; 0 before the first 98 + pending *semEdit // single edit to apply incrementally, nil when none 99 +} 100 + 101 +type semEdit struct { 102 + start, oldEnd, newEnd int 103 + deltaLine int 104 +} 105 + 106 +func (c *semCache) resultID() string { return strconv.FormatUint(c.gen, 10) } 107 +func (c *semCache) nextResultID() string { 108 + c.gen++ 109 + return c.resultID() 110 +} 111 + 93 112 func (s *server) storeSemResult(u uri.URI, data []uint32) (string, bool) { 94 113 s.mu.Lock() 95 114 defer s.mu.Unlock() ··· 97 116 if !ok { 98 117 return "", false 99 118 } 100 - rid := st.nextResultID() 101 - st.semBaseline = data 119 + if st.sem == nil { 120 + st.sem = &semCache{} 121 + } 122 + rid := st.sem.nextResultID() 123 + st.sem.baseline = data 102 124 s.openDocs[u] = st 103 125 return rid, true 104 126 } ··· 110 132 if !ok { 111 133 return nil, false 112 134 } 113 - if st.semTokens != nil { 114 - return st.semTokens, true 135 + if st.sem != nil && st.sem.pending == nil { 136 + return flattenSemTokens(st.sem.entries), true 115 137 } 116 - // Tokenize outside the lock: a full tokenization of a large journal is 117 - // milliseconds, during which didChange/didOpen would otherwise stall. 118 138 rj := s.loader.ResolveBytes(doc.Path(), []byte(st.text)) 119 - tokens := tokenizeForSemantics(st.text, rj.Occurrences[0].Ast) 139 + j := rj.Occurrences[0].Ast 140 + 141 + var tokens []semanticToken 142 + var entries []semEntry 143 + good := st.sem != nil && st.sem.pending != nil 144 + if good { 145 + entries, good = incrementalTokens(st.text, j, st.sem.entries, *st.sem.pending) 146 + } 147 + if !good { 148 + tokens, entries = computeSemTokens(st.text, j) 149 + } else { 150 + tokens = flattenSemTokens(entries) 151 + } 120 152 s.mu.Lock() 121 153 defer s.mu.Unlock() 122 154 cur, ok := s.openDocs[doc] ··· 124 156 return nil, false 125 157 } 126 158 if cur.text == st.text { // unchanged during tokenization 127 - cur.semTokens = tokens 159 + if cur.sem == nil { 160 + cur.sem = &semCache{} 161 + } 162 + cur.sem.entries = entries 163 + cur.sem.pending = nil 128 164 s.openDocs[doc] = cur 129 165 } 130 166 return tokens, true ··· 180 216 modifiers uint32 181 217 } 182 218 219 +type semEntry struct { 220 + start, end int 221 + tokens []semanticToken 222 +} 223 + 183 224 func tokenizeForSemantics(content string, j *ast.Journal) []semanticToken { 184 225 var raw []rawSpan 185 226 emit := func(s token.Span, tokType, mods uint32) { ··· 204 245 tok, mods uint32 205 246 } 206 247 248 +func computeSemTokens(content string, j *ast.Journal) ([]semanticToken, []semEntry) { 249 + if len(j.Errors) > 0 { 250 + return tokenizeForSemantics(content, j), nil 251 + } 252 + entries := buildSemTokens(content, j) 253 + return flattenSemTokens(entries), entries 254 +} 255 + 256 +func buildSemTokens(content string, j *ast.Journal) []semEntry { 257 + entries := make([]semEntry, 0, len(j.Entries)) 258 + for _, e := range j.Entries { 259 + if _, isBlank := e.(*ast.BlankLine); isBlank { 260 + continue 261 + } 262 + sp := entrySpan(e) 263 + entries = append(entries, semEntry{start: sp.Start.Offset, end: sp.End.Offset, tokens: tokensForEntry(content, e)}) 264 + } 265 + return entries 266 +} 267 + 268 +func flattenSemTokens(entries []semEntry) []semanticToken { 269 + var flat []semanticToken 270 + for _, en := range entries { 271 + flat = append(flat, en.tokens...) 272 + } 273 + return flat 274 +} 275 + 276 +func tokensForEntry(content string, e ast.Entry) []semanticToken { 277 + sp := entrySpan(e) 278 + var eraw []rawSpan 279 + visitEntry(content, e, func(tok token.Span, tokKind, modifier uint32) { 280 + if tok.Start.Offset >= tok.End.Offset { 281 + return 282 + } 283 + eraw = append(eraw, rawSpan{tok, tokKind, modifier}) 284 + }) 285 + if len(eraw) == 0 { 286 + return nil 287 + } 288 + return rawToSemanticTokensFrom(content, eraw, sp.Start.Line-1, 0, sp.Start.Offset) 289 +} 290 + 291 +func shiftLines(toks []semanticToken, delta int) []semanticToken { 292 + if delta == 0 { 293 + return toks 294 + } 295 + out := make([]semanticToken, len(toks)) 296 + for i, t := range toks { 297 + t.line = uint32(int(t.line) + delta) 298 + out[i] = t 299 + } 300 + return out 301 +} 302 + 303 +func incrementalTokens(content string, j *ast.Journal, old []semEntry, e semEdit) ([]semEntry, bool) { 304 + if len(j.Errors) > 0 { 305 + return nil, false 306 + } 307 + var nb, na int 308 + for _, en := range j.Entries { 309 + if _, isBlank := en.(*ast.BlankLine); isBlank { 310 + continue 311 + } 312 + switch sp := entrySpan(en); { 313 + case sp.End.Offset <= e.start: 314 + nb++ 315 + case sp.Start.Offset >= e.newEnd: 316 + na++ 317 + } 318 + } 319 + var ob, oa int 320 + for _, en := range old { 321 + switch { 322 + case en.end <= e.start: 323 + ob++ 324 + case en.start >= e.oldEnd: 325 + oa++ 326 + } 327 + } 328 + if nb != ob || na != oa { 329 + return nil, false 330 + } 331 + 332 + out := make([]semEntry, 0, len(old)+1) 333 + iBefore, iTail := 0, len(old)-oa 334 + for _, en := range j.Entries { 335 + if _, isBlank := en.(*ast.BlankLine); isBlank { 336 + continue 337 + } 338 + sp := entrySpan(en) 339 + var toks []semanticToken 340 + switch { 341 + case sp.End.Offset <= e.start: 342 + toks = old[iBefore].tokens 343 + iBefore++ 344 + case sp.Start.Offset >= e.newEnd: 345 + toks = shiftLines(old[iTail].tokens, e.deltaLine) 346 + iTail++ 347 + default: 348 + toks = tokensForEntry(content, en) 349 + } 350 + out = append(out, semEntry{start: sp.Start.Offset, end: sp.End.Offset, tokens: toks}) 351 + } 352 + return out, true 353 +} 354 + 207 355 func rawToSemanticTokens(content string, raw []rawSpan) []semanticToken { 356 + return rawToSemanticTokensFrom(content, raw, 0, 0, 0) 357 +} 358 + 359 +func rawToSemanticTokensFrom(content string, raw []rawSpan, line, col, cursor int) []semanticToken { 208 360 if len(raw) == 0 { 209 361 return nil 210 362 } 211 363 slices.SortFunc(raw, func(a, b rawSpan) int { return a.span.Start.Offset - b.span.Start.Offset }) 212 364 out := make([]semanticToken, len(raw)) 213 - line, col, cursor := 0, 0, 0 214 365 advance := func(end int) { 215 366 for cursor < end { 216 367 r, size := utf8.DecodeRuneInString(content[cursor:])
M
internal/lsp/textdocument_semantic_tokens_test.go
··· 10 10 "go.lsp.dev/protocol" 11 11 "go.lsp.dev/uri" 12 12 13 + "olexsmir.xyz/clerk/internal/lsp/lsputil" 13 14 "olexsmir.xyz/clerk/internal/testutil/golden" 14 15 ) 15 16 ··· 204 205 205 206 finalText := in 206 207 if ed := ar.Get("edited.journal"); ed != nil { 207 - srv.server.updateDoc(u, 2, []protocol.TextDocumentContentChangeEvent{ 208 - &protocol.TextDocumentContentChangeWholeDocument{Text: string(ed)}, 209 - }) 208 + srv.server.updateDoc(u, 2, partialChangeFromDiff(string(in), string(ed))) 210 209 finalText = ed 211 210 } 212 211 prev := *full.ResultID ··· 324 323 return tokenizeForSemantics(c, parseJournalStr(c)) 325 324 } 326 325 326 +func TestSemanticTokensIncremental(t *testing.T) { 327 + for _, tt := range []string{"semantic-incremental-append", "semantic-incremental-insert", "semantic-incremental-delete", "semantic-incremental-inline", "semantic-incremental-header"} { 328 + t.Run(tt, func(t *testing.T) { 329 + ar := golden.Read(t, tt) 330 + in, edited := ar.Get("in.journal"), ar.Get("edited.journal") 331 + 332 + // golden: the server's incremental result equals a full tokenization 333 + golden.Assert(t, ar, renderSemanticTokens(serverSemTokensAfterEdit(t, newServer(t).server, string(in), string(edited)))) 334 + 335 + // engage: the incremental path actually runs, not a silent full rebuild 336 + ev := partialChangeFromDiff(string(in), string(edited))[0].(*protocol.TextDocumentContentChangePartial) 337 + start, oldEnd, newEnd, delta := editRegion(string(in), ev) 338 + _, entries := computeSemTokens(string(in), parseJournalStr(string(in))) 339 + if _, ok := incrementalTokens(string(edited), parseJournalStr(string(edited)), entries, semEdit{start: start, oldEnd: oldEnd, newEnd: newEnd, deltaLine: delta}); !ok { 340 + t.Error("incremental path not taken") 341 + } 342 + }) 343 + } 344 + 345 + t.Run("recover", func(t *testing.T) { 346 + ar := golden.Read(t, "semantic-incremental-recover") 347 + bad, good := ar.Get("in.journal"), ar.Get("edited.journal") 348 + golden.Assert(t, ar, renderSemanticTokens(serverSemTokensAfterEdit(t, newServer(t).server, string(bad), string(good)))) 349 + }) 350 +} 351 + 327 352 func BenchmarkSemanticTokens(b *testing.B) { 328 353 content := openJournal(b, "../../journal/testdata/journals/actual-1ktxns-100accts.journal") 329 354 ··· 353 378 func BenchmarkSemanticTokensEdits(b *testing.B) { 354 379 content := openJournal(b, "../../journal/testdata/journals/actual-1ktxns-100accts.journal") 355 380 old := encodeSemTokens(tokenizeForSemantics(content, parseJournalStr(content))) 356 - new := encodeSemTokens(tokenizeForSemantics( 381 + new_ := encodeSemTokens(tokenizeForSemantics( 357 382 content+"\n2000-06-15 transaction 2501\n expenses:new 1 C\n assets:cash\n", 358 383 parseJournalStr(content+"\n2000-06-15 transaction 2501\n expenses:new 1 C\n assets:cash\n"), 359 384 )) ··· 361 386 b.ReportAllocs() 362 387 b.ResetTimer() 363 388 for b.Loop() { 364 - _ = semanticTokensEdits(old, new) 389 + _ = semanticTokensEdits(old, new_) 390 + } 391 +} 392 + 393 +func BenchmarkIncrementalTokens(b *testing.B) { 394 + content := openJournal(b, "../../journal/testdata/journals/actual-1ktxns-100accts.journal") 395 + _, entries := computeSemTokens(content, parseJournalStr(content)) 396 + 397 + newTx := "\n2024-01-01 new $1\n expenses:new $1\n assets:cash" 398 + appendEdit := content + newTx + "\n" 399 + first := strings.Index(content, "\n\n") 400 + mid := strings.Index(content[first+2:], "\n\n") + first + 2 // blank line between the first two transactions 401 + insertEdit := content[:mid] + newTx + content[mid:] 402 + 403 + b.ReportAllocs() 404 + for _, tc := range []struct{ name, edited string }{ 405 + {"append", appendEdit}, 406 + {"insert", insertEdit}, 407 + } { 408 + ev := partialChangeFromDiff(content, tc.edited)[0].(*protocol.TextDocumentContentChangePartial) 409 + start, oldEnd, newEnd, delta := editRegion(content, ev) 410 + j := parseJournalStr(tc.edited) 411 + 412 + b.Run(tc.name+"/full", func(b *testing.B) { 413 + for b.Loop() { 414 + _ = tokenizeForSemantics(tc.edited, j) 415 + } 416 + }) 417 + b.Run(tc.name+"/incremental", func(b *testing.B) { 418 + for b.Loop() { 419 + if _, ok := incrementalTokens(tc.edited, j, entries, semEdit{start: start, oldEnd: oldEnd, newEnd: newEnd, deltaLine: delta}); !ok { 420 + b.Fatal("fell back to full") 421 + } 422 + } 423 + }) 365 424 } 366 425 } 367 426 ··· 371 430 out = append(append(out[:e.Start], e.Data...), out[e.Start+e.DeleteCount:]...) 372 431 } 373 432 return out 433 +} 434 + 435 +func partialChangeFromDiff(from, to string) []protocol.TextDocumentContentChangeEvent { 436 + p := 0 437 + for p < len(from) && p < len(to) && from[p] == to[p] { 438 + p++ 439 + } 440 + s := 0 441 + for s < len(from)-p && s < len(to)-p && from[len(from)-1-s] == to[len(to)-1-s] { 442 + s++ 443 + } 444 + return []protocol.TextDocumentContentChangeEvent{ 445 + &protocol.TextDocumentContentChangePartial{ 446 + Range: protocol.Range{ 447 + Start: lsputil.Position(from, p), 448 + End: lsputil.Position(from, len(from)-s), 449 + }, 450 + Text: to[p : len(to)-s], 451 + }, 452 + } 453 +} 454 + 455 +func editRegion(from string, ev *protocol.TextDocumentContentChangePartial) (start, oldEnd, newEnd, deltaLine int) { 456 + li := lsputil.NewLineIndex(from) 457 + start = li.Offset(int(ev.Range.Start.Line), int(ev.Range.Start.Character)) 458 + oldEnd = li.Offset(int(ev.Range.End.Line), int(ev.Range.End.Character)) 459 + newEnd = start + len(ev.Text) 460 + deltaLine = strings.Count(ev.Text, "\n") - strings.Count(from[start:oldEnd], "\n") 461 + return 462 +} 463 + 464 +func serverSemTokensAfterEdit(t *testing.T, srv *server, in, edited string) []semanticToken { 465 + t.Helper() 466 + u := uri.URI("file:///test.journal") 467 + srv.openDoc(u, in, 1, "journal") 468 + if _, ok := srv.tokensForDoc(u); !ok { 469 + t.Fatal("priming failed") 470 + } 471 + srv.updateDoc(u, 2, partialChangeFromDiff(in, edited)) 472 + toks, ok := srv.tokensForDoc(u) 473 + if !ok { 474 + t.Fatal("tokensForDoc failed after edit") 475 + } 476 + return toks 374 477 } 375 478 376 479 func openJournal(t testing.TB, path string) string {
M
internal/lsp/textdocument_sync.go
··· 2 2 3 3 import ( 4 4 "context" 5 - "strconv" 5 + "strings" 6 6 7 7 "go.lsp.dev/protocol" 8 8 "go.lsp.dev/uri" ··· 59 59 lineIdx *lsputil.LineIndex // cached line index for the text 60 60 61 61 analysis *analyzer.Analysis // cached analysis, nil until first build 62 + sem *semCache // semantic token cache, nil until first tokenize 62 63 dirty bool // true while the cached analysis may not reflect the current text 63 - 64 - semTokens []semanticToken // cached tokens for the current text 65 - semBaseline []uint32 // encoded data of the last response; diff baseline for the next delta request 66 - semGen uint64 // semantic token generation, increments per response; 0 before the first 67 -} 68 - 69 -func (d *docState) resultID() string { return strconv.FormatUint(d.semGen, 10) } 70 -func (d *docState) nextResultID() string { 71 - d.semGen++ 72 - return d.resultID() 73 64 } 74 65 75 66 func (s *server) openDoc(u uri.URI, text string, version int32, langID protocol.LanguageKind) { ··· 92 83 return 93 84 } 94 85 state.version = version 86 + 87 + incremental := len(changes) == 1 88 + var edit semEdit 95 89 for _, ch := range changes { 96 90 switch ev := ch.(type) { 97 91 case *protocol.TextDocumentContentChangeWholeDocument: 92 + incremental = false 98 93 state.text = ev.Text 99 - state.semTokens = nil 100 - state.lineIdx = lsputil.NewLineIndex(ev.Text) 101 94 case *protocol.TextDocumentContentChangePartial: 102 - _ = ev // TODO: incremental edit support 95 + if state.sem == nil || state.sem.pending != nil { 96 + incremental = false 97 + } 98 + if state.lineIdx == nil { 99 + state.lineIdx = lsputil.NewLineIndex(state.text) 100 + } 101 + start := state.lineIdx.Offset(int(ev.Range.Start.Line), int(ev.Range.Start.Character)) 102 + end := state.lineIdx.Offset(int(ev.Range.End.Line), int(ev.Range.End.Character)) 103 + removed := state.text[start:end] 104 + state.text = state.text[:start] + ev.Text + state.text[end:] 105 + if incremental { 106 + edit = semEdit{ 107 + start: start, 108 + oldEnd: end, 109 + newEnd: start + len(ev.Text), 110 + deltaLine: strings.Count(ev.Text, "\n") - strings.Count(removed, "\n"), 111 + } 112 + } 103 113 } 114 + state.lineIdx = lsputil.NewLineIndex(state.text) 104 115 } 116 + 105 117 state.analysis = nil 106 118 state.dirty = true 119 + if incremental { 120 + state.sem.pending = &edit 121 + } else { 122 + state.sem = nil 123 + } 107 124 s.openDocs[u] = state 108 125 s.mu.Unlock() 109 126