7 files changed,
351 insertions(+),
56 deletions(-)
Author:
Oleksandr Smirnov
olexsmir@gmail.com
Committed at:
2026-08-17 14:46:05 +0300
Authored at:
2026-08-16 19:01:31 +0300
Change ID:
nnzyltrzyryputvowqywtmnwoqyvkqwo
Parent:
7be0936
jump to
M
internal/lsp/server.go
··· 78 78 } 79 79 80 80 s.applySettings(params.InitializationOptions) 81 + full := protocol.SemanticTokensOptionsFull(protocol.Boolean(true)) 82 + if td := params.Capabilities.TextDocument; td != nil { 83 + if fd, ok := td.SemanticTokens.Requests.Full.(*protocol.ClientSemanticTokensRequestFullDelta); ok && fd.Delta != nil && *fd.Delta { 84 + full = &protocol.SemanticTokensFullDelta{Delta: new(true)} 85 + } 86 + } 87 + 81 88 return &protocol.InitializeResult{ 82 89 ServerInfo: protocol.ServerInfo{ 83 90 Name: s.name, ··· 103 110 SemanticTokensProvider: &protocol.SemanticTokensOptions{ 104 111 Legend: getSemanticTokensLegend(), 105 112 Range: protocol.Boolean(true), 106 - Full: protocol.Boolean(true), 113 + Full: full, 107 114 }, 108 115 }, 109 116 }, nil
A
internal/lsp/testdata/semantic-delta-edit.txtar
··· 1 +-- in.journal -- 2 +2024-01-01 groceries 3 + expenses:food $10 4 + assets:cash 5 + 6 +2024-01-02 fuel 7 + expenses:car $40 8 + assets:cash 9 + 10 +2024-01-03 books 11 + expenses:books $20 12 + assets:cash 13 + 14 +-- edited.journal -- 15 +2024-01-01 groceries 16 + expenses:food $10 17 + assets:cash 18 + 19 +2024-01-02 fuel 20 + expenses:car $40 21 + assets:cash 22 + 23 +2024-01-05 coffee 24 + expenses:meals $5 25 + assets:cash 26 + 27 +2024-01-03 books 28 + expenses:books $20 29 + assets:cash 30 + 31 +-- expect -- 32 +delta resultId 2 33 +0:0+10 class 34 +0:11+9 property 35 +1:4+13 namespace 36 +1:19+1 type 37 +1:20+2 number 38 +2:4+11 namespace 39 +4:0+10 class 40 +4:11+4 property 41 +5:4+12 namespace 42 +5:18+1 type 43 +5:19+2 number 44 +6:4+11 namespace 45 +8:0+10 class 46 +8:11+6 property 47 +9:4+14 namespace 48 +9:20+1 type 49 +9:21+1 number 50 +10:4+11 namespace 51 +12:0+10 class 52 +12:11+5 property 53 +13:4+14 namespace 54 +13:20+1 type 55 +13:21+2 number 56 +14:4+11 namespace
A
internal/lsp/testdata/semantic-delta-stale.txtar
··· 1 +-- in.journal -- 2 +2024-01-15 test 3 + expenses:food $50 4 + assets:cash 5 + 6 +-- prev-id -- 7 +bogus 8 + 9 +-- expect -- 10 +full resultId 2 11 +0:0+10 class 12 +0:11+4 property 13 +1:4+13 namespace 14 +1:19+1 type 15 +1:20+2 number 16 +2:4+11 namespace
M
internal/lsp/textdocument_semantic_tokens.go
··· 18 18 if !s.semanticHighlightingEnabled() { 19 19 return &protocol.SemanticTokens{}, nil 20 20 } 21 + return s.semanticTokensFullResult(params.TextDocument.URI), nil 22 +} 21 23 22 - tokens, ok := s.tokensForDoc(params.TextDocument.URI) 24 +func (s *server) SemanticTokensFullDelta(ctx context.Context, params *protocol.SemanticTokensDeltaParams) (protocol.SemanticTokensDeltaResult, error) { 25 + if !s.semanticHighlightingEnabled() { 26 + return &protocol.SemanticTokens{}, nil 27 + } 28 + 29 + u := params.TextDocument.URI 30 + s.mu.RLock() 31 + st, ok := s.openDocs[u] 32 + s.mu.RUnlock() 33 + if !ok || st.semGen == 0 || params.PreviousResultID != st.resultID() { 34 + return s.semanticTokensFullResult(u), nil 35 + } 36 + 37 + data, ok := s.semanticTokensData(u) 23 38 if !ok { 24 39 return &protocol.SemanticTokens{}, nil 25 40 } 26 - return &protocol.SemanticTokens{Data: encodeSemTokens(tokens)}, nil 41 + edits := semanticTokensEdits(st.semBaseline, data) 42 + if len(edits) == 0 { 43 + return &protocol.SemanticTokensDelta{ResultID: new(st.resultID()), Edits: []protocol.SemanticTokensEdit{}}, nil 44 + } 45 + 46 + rid, ok := s.storeSemResult(u, data) 47 + if !ok { 48 + return &protocol.SemanticTokens{Data: data}, nil 49 + } 50 + return &protocol.SemanticTokensDelta{ResultID: &rid, Edits: edits}, nil 27 51 } 28 52 29 53 func (s *server) SemanticTokensRange(ctx context.Context, params *protocol.SemanticTokensRangeParams) (*protocol.SemanticTokens, error) { ··· 46 70 return &protocol.SemanticTokens{Data: encodeSemTokens(filtered)}, nil 47 71 } 48 72 73 +func (s *server) semanticTokensFullResult(u uri.URI) *protocol.SemanticTokens { 74 + data, ok := s.semanticTokensData(u) 75 + if !ok { 76 + return &protocol.SemanticTokens{} 77 + } 78 + res := &protocol.SemanticTokens{Data: data} 79 + if rid, ok := s.storeSemResult(u, data); ok { 80 + res.ResultID = &rid 81 + } 82 + return res 83 +} 84 + 85 +func (s *server) semanticTokensData(u uri.URI) ([]uint32, bool) { 86 + tokens, ok := s.tokensForDoc(u) 87 + if !ok { 88 + return nil, false 89 + } 90 + return encodeSemTokens(tokens), true 91 +} 92 + 93 +func (s *server) storeSemResult(u uri.URI, data []uint32) (string, bool) { 94 + s.mu.Lock() 95 + defer s.mu.Unlock() 96 + st, ok := s.openDocs[u] 97 + if !ok { 98 + return "", false 99 + } 100 + rid := st.nextResultID() 101 + st.semBaseline = data 102 + s.openDocs[u] = st 103 + return rid, true 104 +} 105 + 49 106 func (s *server) tokensForDoc(doc uri.URI) ([]semanticToken, bool) { 50 107 s.mu.RLock() 51 108 st, ok := s.openDocs[doc] ··· 71 128 } 72 129 return tokens, true 73 130 } 74 - 75 -// Implementation 76 131 77 132 const ( 78 133 semDirective uint32 = iota ··· 649 704 return true 650 705 } 651 706 return false 707 +} 708 + 709 +// semanticTokensEdits returns the single edit turning old into new, or nil when 710 +// identical. Relative delta encoding keeps the common prefix and suffix unchanged. 711 +func semanticTokensEdits(old, new []uint32) []protocol.SemanticTokensEdit { 712 + p := 0 713 + for p < len(old) && p < len(new) && old[p] == new[p] { 714 + p++ 715 + } 716 + s := 0 717 + for s < len(old)-p && s < len(new)-p && old[len(old)-1-s] == new[len(new)-1-s] { 718 + s++ 719 + } 720 + delCount := len(old) - p - s 721 + ins := new[p : len(new)-s] 722 + if delCount == 0 && len(ins) == 0 { 723 + return nil 724 + } 725 + return []protocol.SemanticTokensEdit{{ 726 + Start: uint32(p), 727 + DeleteCount: uint32(delCount), 728 + Data: ins, 729 + }} 652 730 } 653 731 654 732 // encodeSemTokens encodes tokens into LSP delta form. Input must be sorted by
M
internal/lsp/textdocument_semantic_tokens_test.go
··· 59 59 } 60 60 } 61 61 62 -func TestServer_Semantic_SimpleTransaction(t *testing.T) { 63 - content := `2024-01-15 test 64 - expenses:food $50 65 - assets:cash 66 -` 67 - 68 - srv := NewServer("test") 69 - srv.server.openDoc(uri.URI("file:///test.journal"), content, 1, "journal") 70 - 71 - result, err := srv.server.SemanticTokensFull(t.Context(), &protocol.SemanticTokensParams{ 72 - TextDocument: protocol.TextDocumentIdentifier{URI: uri.URI("file:///test.journal")}, 73 - }) 74 - if err != nil { 75 - t.Fatal(err) 76 - } 77 - if result == nil { 78 - t.Fatal("result is nil") 79 - } 80 - if len(result.Data) == 0 { 81 - t.Fatal("expected non-empty token data") 82 - } 83 - if len(result.Data)%5 != 0 { 84 - t.Fatalf("token data length %d is not a multiple of 5", len(result.Data)) 85 - } 86 - // first token is the transaction date at line 0, col 0: deltas are 0, 0 87 - if result.Data[0] != 0 || result.Data[1] != 0 { 88 - t.Errorf("first token deltas = %d,%d, want 0,0", result.Data[0], result.Data[1]) 89 - } 90 -} 91 - 92 62 func TestServer_Semantic_EmptyDocument(t *testing.T) { 93 63 srv := NewServer("test") 94 64 srv.server.openDoc(uri.URI("file:///empty.journal"), "", 1, "journal") ··· 159 129 } 160 130 } 161 131 162 -// Golden 132 +func TestSemanticTokensEdits(t *testing.T) { 133 + tests := map[string]struct{ old, new []uint32 }{ 134 + "identical": {[]uint32{1, 2, 3}, []uint32{1, 2, 3}}, 135 + "both empty": {}, 136 + "empty to non": {nil, []uint32{1, 2}}, 137 + "non to empty": {[]uint32{1, 2}, nil}, 138 + "pure insert": {[]uint32{1, 2}, []uint32{1, 2, 3, 4}}, 139 + "pure delete": {[]uint32{1, 2, 3, 4}, []uint32{1, 2}}, 140 + "replace middle": {[]uint32{1, 2, 3, 4, 5}, []uint32{1, 2, 9, 4, 5}}, 141 + "replace all": {[]uint32{1, 2}, []uint32{3, 4}}, 142 + "replace tail": {[]uint32{1, 2, 3, 4, 5}, []uint32{1, 2, 3, 4, 6}}, 143 + } 144 + for tname, tt := range tests { 145 + t.Run(tname, func(t *testing.T) { 146 + edits := semanticTokensEdits(tt.old, tt.new) 147 + if got := applySemEdits(tt.old, edits); !slices.Equal(got, tt.new) { 148 + t.Errorf("apply(%v, %v) = %v, want %v", tt.old, edits, got, tt.new) 149 + } 150 + for _, e := range edits { 151 + if e.Start+e.DeleteCount > uint32(len(tt.old)) { 152 + t.Errorf("edit %+v out of bounds for old length %d", e, len(tt.old)) 153 + } 154 + } 155 + }) 156 + } 157 +} 158 + 159 +// applySemEdits applies LSP semantic token edits to a client-side copy of data. 160 +func applySemEdits(data []uint32, edits []protocol.SemanticTokensEdit) []uint32 { 161 + out := slices.Clone(data) 162 + for _, e := range edits { 163 + out = append(append(out[:e.Start], e.Data...), out[e.Start+e.DeleteCount:]...) 164 + } 165 + return out 166 +} 163 167 164 168 func TestGolden_SemanticTokens(t *testing.T) { 165 169 for _, tt := range []string{"semantic-empty", "semantic-journal", "semantic-directives", "semantic-unparseable", "semantic-with-errors"} { ··· 171 175 }) 172 176 173 177 t.Run(tt+"_no-overlap", func(t *testing.T) { 174 - toks := tokSem(ar.Get("in.journal")) 175 - slices.SortFunc(toks, func(a, b semanticToken) int { 176 - if a.line != b.line { 177 - return int(a.line) - int(b.line) 178 - } 179 - return int(a.col) - int(b.col) 178 + assertGoldenNoOverlap(t, tt, ar) 179 + }) 180 + } 181 +} 182 + 183 +func TestGolden_SemanticTokensDelta(t *testing.T) { 184 + for _, tt := range []string{"semantic-delta-edit", "semantic-delta-nochange", "semantic-delta-stale"} { 185 + ar := golden.Read(t, tt) 186 + 187 + t.Run(tt+"_no-overlap", func(t *testing.T) { 188 + assertGoldenNoOverlap(t, tt, ar) 189 + }) 190 + 191 + t.Run(tt, func(t *testing.T) { 192 + in := ar.Get("in.journal") 193 + 194 + u := uri.URI("file:///test.journal") 195 + srv := NewServer("test") 196 + srv.server.openDoc(u, string(in), 1, "journal") 197 + 198 + full, err := srv.server.SemanticTokensFull(t.Context(), &protocol.SemanticTokensParams{ 199 + TextDocument: protocol.TextDocumentIdentifier{URI: u}, 180 200 }) 201 + if err != nil { 202 + t.Fatal(err) 203 + } 204 + if full.ResultID == nil || *full.ResultID == "" { 205 + t.Fatal("expected a resultId on the full result") 206 + } 181 207 182 - for i := 1; i < len(toks); i++ { 183 - prev, cur := toks[i-1], toks[i] 184 - if prev.line != cur.line { 185 - continue 186 - } 187 - if cur.col < prev.col+prev.length { 188 - t.Errorf("%s: overlapping tokens on line %d: %s@%d+%d then %s@%d+%d", 189 - tt, prev.line, tokenTypeStrings[prev.tokenType], prev.col, prev.length, 190 - tokenTypeStrings[cur.tokenType], cur.col, cur.length) 191 - } 208 + finalText := in 209 + if ed := ar.Get("edited.journal"); ed != nil { 210 + srv.server.updateDoc(u, 2, []protocol.TextDocumentContentChangeEvent{ 211 + &protocol.TextDocumentContentChangeWholeDocument{Text: string(ed)}, 212 + }) 213 + finalText = ed 192 214 } 215 + prev := *full.ResultID 216 + if p := ar.Get("prev-id"); p != nil { 217 + prev = strings.TrimSpace(string(p)) 218 + } 219 + 220 + res, err := srv.server.SemanticTokensFullDelta(t.Context(), &protocol.SemanticTokensDeltaParams{ 221 + TextDocument: protocol.TextDocumentIdentifier{URI: u}, 222 + PreviousResultID: prev, 223 + }) 224 + if err != nil { 225 + t.Fatal(err) 226 + } 227 + 228 + var client []uint32 229 + var out strings.Builder 230 + switch r := res.(type) { 231 + case *protocol.SemanticTokensDelta: 232 + fmt.Fprintf(&out, "delta resultId %s\n", *r.ResultID) 233 + client = applySemEdits(full.Data, r.Edits) 234 + case *protocol.SemanticTokens: 235 + fmt.Fprintf(&out, "full resultId %s\n", *r.ResultID) 236 + client = r.Data 237 + default: 238 + t.Fatalf("unexpected result type %T", res) 239 + } 240 + 241 + // The client's token state must equal the final text's tokens: an 242 + // independent reference that keeps the golden from capturing bugs. 243 + if want := encodeSemTokens(tokSem(finalText)); !slices.Equal(client, want) { 244 + t.Errorf("delta flow produced %d elems, want %d", len(client), len(want)) 245 + } 246 + out.WriteString(renderSemanticTokens(tokSem(finalText))) 247 + golden.Assert(t, ar, out.String()) 193 248 }) 194 249 } 195 250 } 196 251 252 +func assertGoldenNoOverlap(t *testing.T, tt string, ar *golden.Archive) { 253 + t.Helper() 254 + for _, f := range ar.Files { 255 + if strings.HasSuffix(f.Name, ".journal") { 256 + assertNoOverlap(t, tt, tokSem(f.Data)) 257 + } 258 + } 259 +} 260 + 261 +func assertNoOverlap(t *testing.T, tt string, toks []semanticToken) { 262 + t.Helper() 263 + slices.SortFunc(toks, func(a, b semanticToken) int { 264 + if a.line != b.line { 265 + return int(a.line) - int(b.line) 266 + } 267 + return int(a.col) - int(b.col) 268 + }) 269 + for i := 1; i < len(toks); i++ { 270 + prev, cur := toks[i-1], toks[i] 271 + if prev.line != cur.line { 272 + continue 273 + } 274 + if cur.col < prev.col+prev.length { 275 + t.Errorf("%s: overlapping tokens on line %d: %s@%d+%d then %s@%d+%d", 276 + tt, prev.line, tokenTypeStrings[prev.tokenType], prev.col, prev.length, 277 + tokenTypeStrings[cur.tokenType], cur.col, cur.length) 278 + } 279 + } 280 +} 281 + 197 282 func renderSemanticTokens(tokens []semanticToken) string { 198 283 slices.SortFunc(tokens, func(a, b semanticToken) int { 199 284 if a.line != b.line { ··· 229 314 for b.Loop() { 230 315 tokens := tokenizeForSemantics(content, parseJournalStr(content)) 231 316 _ = encodeSemTokens(tokens) 317 + } 318 +} 319 + 320 +// BenchmarkSemanticTokensDelta measures the cost of one dela response after an edit. 321 +func BenchmarkSemanticTokensDelta(b *testing.B) { 322 + content := openJournal(b, "../../journal/testdata/journals/actual-1ktxns-100accts.journal") 323 + prev := encodeSemTokens(tokenizeForSemantics(content, parseJournalStr(content))) 324 + edited := content + "\n2000-06-15 transaction 2501\n expenses:new 1 C\n assets:cash\n" 325 + 326 + b.ReportAllocs() 327 + b.ResetTimer() 328 + for b.Loop() { 329 + data := encodeSemTokens(tokenizeForSemantics(edited, parseJournalStr(edited))) 330 + _ = semanticTokensEdits(prev, data) 331 + } 332 +} 333 + 334 +func BenchmarkSemanticTokensEdits(b *testing.B) { 335 + content := openJournal(b, "../../journal/testdata/journals/actual-1ktxns-100accts.journal") 336 + old := encodeSemTokens(tokenizeForSemantics(content, parseJournalStr(content))) 337 + new := encodeSemTokens(tokenizeForSemantics( 338 + content+"\n2000-06-15 transaction 2501\n expenses:new 1 C\n assets:cash\n", 339 + parseJournalStr(content+"\n2000-06-15 transaction 2501\n expenses:new 1 C\n assets:cash\n"), 340 + )) 341 + 342 + b.ReportAllocs() 343 + b.ResetTimer() 344 + for b.Loop() { 345 + _ = semanticTokensEdits(old, new) 232 346 } 233 347 } 234 348
M
internal/lsp/textdocument_sync.go
··· 2 2 3 3 import ( 4 4 "context" 5 + "strconv" 5 6 6 7 "go.lsp.dev/protocol" 7 8 "go.lsp.dev/uri" ··· 53 54 text string 54 55 version int32 55 56 languageID protocol.LanguageKind 56 - semTokens []semanticToken // cached semantic tokens 57 + paths map[string]bool // canonical paths of every file in the cached analysis 57 58 lineIdx *lsputil.LineIndex // cached line index for the text 58 - analysis *analyzer.Analysis // cached analysis, nil until first build 59 - paths map[string]bool // canonical paths of every file in the cached analysis 60 - dirty bool // true while the cached analysis may not reflect the current text 59 + 60 + analysis *analyzer.Analysis // cached analysis, nil until first build 61 + dirty bool // true while the cached analysis may not reflect the current text 62 + 63 + semTokens []semanticToken // cached tokens for the current text 64 + semBaseline []uint32 // encoded data of the last response; diff baseline for the next delta request 65 + semGen uint64 // semantic token generation, increments per response; 0 before the first 66 +} 67 + 68 +func (d *docState) resultID() string { return strconv.FormatUint(d.semGen, 10) } 69 +func (d *docState) nextResultID() string { 70 + d.semGen++ 71 + return d.resultID() 61 72 } 62 73 63 74 func (s *server) openDoc(u uri.URI, text string, version int32, langID protocol.LanguageKind) {