clerk/internal/lsp/textdocument_semantic.go (view raw)
| 1 | package lsp |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "slices" |
| 6 | "unicode/utf8" |
| 7 | |
| 8 | "go.lsp.dev/protocol" |
| 9 | "go.lsp.dev/uri" |
| 10 | |
| 11 | "olexsmir.xyz/clerk/internal/lsp/lsputil" |
| 12 | "olexsmir.xyz/clerk/journal/ast" |
| 13 | "olexsmir.xyz/clerk/journal/lexer" |
| 14 | "olexsmir.xyz/clerk/journal/token" |
| 15 | ) |
| 16 | |
| 17 | func (s *server) SemanticTokensFull(ctx context.Context, params *protocol.SemanticTokensParams) (*protocol.SemanticTokens, error) { |
| 18 | if !s.semanticHighlightingEnabled() { |
| 19 | return &protocol.SemanticTokens{}, nil |
| 20 | } |
| 21 | |
| 22 | tokens, ok := s.tokensForDoc(params.TextDocument.URI) |
| 23 | if !ok { |
| 24 | return &protocol.SemanticTokens{}, nil |
| 25 | } |
| 26 | return &protocol.SemanticTokens{Data: encodeSemTokens(tokens)}, nil |
| 27 | } |
| 28 | |
| 29 | func (s *server) SemanticTokensRange(ctx context.Context, params *protocol.SemanticTokensRangeParams) (*protocol.SemanticTokens, error) { |
| 30 | if !s.semanticHighlightingEnabled() { |
| 31 | return &protocol.SemanticTokens{}, nil |
| 32 | } |
| 33 | |
| 34 | tokens, ok := s.tokensForDoc(params.TextDocument.URI) |
| 35 | if !ok { |
| 36 | return &protocol.SemanticTokens{}, nil |
| 37 | } |
| 38 | start := int(params.Range.Start.Line) |
| 39 | end := int(params.Range.End.Line) |
| 40 | var filtered []semanticToken |
| 41 | for _, t := range tokens { |
| 42 | if t.line >= uint32(start) && t.line <= uint32(end) { |
| 43 | filtered = append(filtered, t) |
| 44 | } |
| 45 | } |
| 46 | return &protocol.SemanticTokens{Data: encodeSemTokens(filtered)}, nil |
| 47 | } |
| 48 | |
| 49 | func (s *server) tokensForDoc(doc uri.URI) ([]semanticToken, bool) { |
| 50 | s.mu.Lock() |
| 51 | defer s.mu.Unlock() |
| 52 | st, ok := s.openDocs[doc] |
| 53 | if !ok { |
| 54 | return nil, false |
| 55 | } |
| 56 | if st.semTokens != nil { |
| 57 | return st.semTokens, true |
| 58 | } |
| 59 | st.semTokens = tokenizeForSemantics(st.text, parseJournalStr(st.text)) |
| 60 | s.openDocs[doc] = st |
| 61 | return st.semTokens, true |
| 62 | } |
| 63 | |
| 64 | // Implementation |
| 65 | |
| 66 | const ( |
| 67 | semDirective uint32 = iota |
| 68 | semDate |
| 69 | semAccount |
| 70 | semCommodity |
| 71 | semAmount |
| 72 | semStatus |
| 73 | semComment |
| 74 | semString |
| 75 | semOperator |
| 76 | semProperty |
| 77 | ) |
| 78 | |
| 79 | var tokenTypeStrings = []string{ |
| 80 | string(protocol.SemanticTokenTypesKeyword), // directive |
| 81 | string(protocol.SemanticTokenTypesClass), // date |
| 82 | string(protocol.SemanticTokenTypesNamespace), // account |
| 83 | string(protocol.SemanticTokenTypesType), // commodity |
| 84 | string(protocol.SemanticTokenTypesNumber), // amount |
| 85 | string(protocol.SemanticTokenTypesOperator), // status |
| 86 | string(protocol.SemanticTokenTypesComment), // comment |
| 87 | string(protocol.SemanticTokenTypesString), // string |
| 88 | string(protocol.SemanticTokenTypesOperator), // operator |
| 89 | string(protocol.SemanticTokenTypesProperty), // property |
| 90 | } |
| 91 | |
| 92 | const ( |
| 93 | modifierAbstract = 1 << 0 // virtual account |
| 94 | modifierNegative = 1 << 1 // negative amount |
| 95 | ) |
| 96 | |
| 97 | var modifierStrings = []string{ |
| 98 | "abstract", // bit 0 |
| 99 | "negative", // bit 1 |
| 100 | } |
| 101 | |
| 102 | func getSemanticTokensLegend() protocol.SemanticTokensLegend { |
| 103 | return protocol.SemanticTokensLegend{ |
| 104 | TokenTypes: tokenTypeStrings, |
| 105 | TokenModifiers: modifierStrings, |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | type semanticToken struct { |
| 110 | line, col uint32 // 0-based |
| 111 | length uint32 |
| 112 | tokenType uint32 |
| 113 | modifiers uint32 |
| 114 | } |
| 115 | |
| 116 | func tokenizeForSemantics(content string, j *ast.Journal) []semanticToken { |
| 117 | var raw []rawSpan |
| 118 | emit := func(s token.Span, tokType, mods uint32) { |
| 119 | if s.Start.Offset >= s.End.Offset { |
| 120 | return |
| 121 | } |
| 122 | raw = append(raw, rawSpan{s, tokType, mods}) |
| 123 | } |
| 124 | for _, e := range j.Entries { |
| 125 | visitEntry(content, e, emit) |
| 126 | } |
| 127 | if len(j.Errors) > 0 { |
| 128 | // parser recovers per line; lexer fills the unparsed regions, keeping ast tokens where the parser succeeded |
| 129 | semLexerFallback(content, raw, emit) |
| 130 | } |
| 131 | return rawToSemanticTokens(content, raw) |
| 132 | } |
| 133 | |
| 134 | // rawSpan is a source span tagged with semantic token |
| 135 | type rawSpan struct { |
| 136 | span token.Span |
| 137 | tok, mods uint32 |
| 138 | } |
| 139 | |
| 140 | func rawToSemanticTokens(content string, raw []rawSpan) []semanticToken { |
| 141 | if len(raw) == 0 { |
| 142 | return nil |
| 143 | } |
| 144 | slices.SortFunc(raw, func(a, b rawSpan) int { return a.span.Start.Offset - b.span.Start.Offset }) |
| 145 | out := make([]semanticToken, len(raw)) |
| 146 | line, col, cursor := 0, 0, 0 |
| 147 | advance := func(end int) { |
| 148 | for cursor < end { |
| 149 | r, size := utf8.DecodeRuneInString(content[cursor:]) |
| 150 | if r == utf8.RuneError && size <= 1 { |
| 151 | break |
| 152 | } |
| 153 | if r == '\r' { |
| 154 | cursor += size |
| 155 | if cursor < len(content) && content[cursor] == '\n' { |
| 156 | cursor++ |
| 157 | } |
| 158 | line++ |
| 159 | col = 0 |
| 160 | continue |
| 161 | } |
| 162 | if r == '\n' { |
| 163 | cursor += size |
| 164 | line++ |
| 165 | col = 0 |
| 166 | continue |
| 167 | } |
| 168 | cursor += size |
| 169 | col += utf16Units(r) |
| 170 | } |
| 171 | } |
| 172 | for i, t := range raw { |
| 173 | if cursor < t.span.Start.Offset { |
| 174 | advance(t.span.Start.Offset) |
| 175 | } |
| 176 | out[i] = semanticToken{ |
| 177 | line: uint32(line), |
| 178 | col: uint32(col), |
| 179 | length: uint32(lsputil.Utf16Len(content, t.span.Start.Offset, t.span.End.Offset)), |
| 180 | tokenType: t.tok, |
| 181 | modifiers: t.mods, |
| 182 | } |
| 183 | advance(t.span.End.Offset) |
| 184 | } |
| 185 | return out |
| 186 | } |
| 187 | |
| 188 | func utf16Units(r rune) int { |
| 189 | if r >= 0x10000 && r <= 0x10FFFF { |
| 190 | return 2 |
| 191 | } |
| 192 | return 1 |
| 193 | } |
| 194 | |
| 195 | func visitEntry(content string, e ast.Entry, emit semEmitFunc) { |
| 196 | switch e := e.(type) { |
| 197 | case *ast.Transaction: |
| 198 | visitTransaction(content, e, emit) |
| 199 | case *ast.PeriodicTransaction: |
| 200 | visitPeriodicTransaction(content, e, emit) |
| 201 | case *ast.AutomatedTransaction: |
| 202 | visitAutomatedTransaction(content, e, emit) |
| 203 | case *ast.AccountDirective: |
| 204 | emit(directiveKeyword(e.Span, "account"), semDirective, 0) |
| 205 | emit(e.Account.Span, semAccount, 0) |
| 206 | emitComment(e.Comment, emit) |
| 207 | case *ast.CommodityDirective: |
| 208 | emit(directiveKeyword(e.Span, "commodity"), semDirective, 0) |
| 209 | if e.Format.Span.End.Offset > 0 { |
| 210 | semEmitAmount(content, &e.Format, emit) |
| 211 | } else if e.CommoditySpan.Start.Offset > 0 && e.CommoditySpan.End.Offset > 0 { |
| 212 | emit(e.CommoditySpan, semCommodity, 0) |
| 213 | } |
| 214 | emitComment(e.Comment, emit) |
| 215 | case *ast.IncludeDirective: |
| 216 | emitDirective(content, e.Span, len("include"), semString, e.Comment, emit) |
| 217 | case *ast.PayeeDirective: |
| 218 | emitDirective(content, e.Span, len("payee"), semProperty, e.Comment, emit) |
| 219 | case *ast.TagDirective: |
| 220 | emitDirective(content, e.Span, len("tag"), semProperty, e.Comment, emit) |
| 221 | case *ast.AliasDirective: |
| 222 | emit(directiveKeyword(e.Span, "alias"), semDirective, 0) |
| 223 | emit(e.From.Span, semAccount, 0) |
| 224 | if op, ok := betweenSpan(content, e.Span.Start.File, e.From.Span.End.Offset, e.To.Span.Start.Offset); ok { |
| 225 | emit(op, semOperator, 0) |
| 226 | } |
| 227 | emit(e.To.Span, semAccount, 0) |
| 228 | emitComment(e.Comment, emit) |
| 229 | case *ast.YearDirective: |
| 230 | kwLen := len("year") |
| 231 | if content[e.Span.Start.Offset] == 'Y' { |
| 232 | kwLen = 1 |
| 233 | } |
| 234 | emitDirective(content, e.Span, kwLen, semProperty, e.Comment, emit) |
| 235 | case *ast.DecimalMarkDirective: |
| 236 | emitDirective(content, e.Span, len("decimal-mark"), semProperty, e.Comment, emit) |
| 237 | case *ast.DefaultCommodityDirective: |
| 238 | emit(directiveKeyword(e.Span, "D"), semDirective, 0) |
| 239 | semEmitAmount(content, &e.Amount, emit) |
| 240 | emitComment(e.Comment, emit) |
| 241 | case *ast.MarketPriceDirective: |
| 242 | emit(directiveKeyword(e.Span, "P"), semDirective, 0) |
| 243 | emit(e.DateTime.Date.Span, semDate, 0) |
| 244 | if e.DateTime.Time != nil { |
| 245 | emit(e.DateTime.Time.Span, semDate, 0) |
| 246 | } |
| 247 | // commodity: text between the date (or time) and the amount |
| 248 | commStart := e.DateTime.Date.Span.End.Offset |
| 249 | if e.DateTime.Time != nil { |
| 250 | commStart = e.DateTime.Time.Span.End.Offset |
| 251 | } |
| 252 | if comm, ok := betweenSpan(content, e.Span.Start.File, commStart, e.Amount.Span.Start.Offset); ok { |
| 253 | emit(comm, semCommodity, 0) |
| 254 | } |
| 255 | semEmitAmount(content, &e.Amount, emit) |
| 256 | emitComment(e.Comment, emit) |
| 257 | case *ast.ConversionDirective: |
| 258 | emit(directiveKeyword(e.Span, "C"), semDirective, 0) |
| 259 | semEmitAmount(content, &e.From, emit) |
| 260 | // = operator: text between the two amounts |
| 261 | if op, ok := betweenSpan(content, e.Span.Start.File, e.From.Span.End.Offset, e.To.Span.Start.Offset); ok { |
| 262 | emit(op, semOperator, 0) |
| 263 | } |
| 264 | semEmitAmount(content, &e.To, emit) |
| 265 | emitComment(e.Comment, emit) |
| 266 | case *ast.Comment: |
| 267 | emitComment(e, emit) |
| 268 | case *ast.CommentBlockDirective: |
| 269 | emit(e.Span, semComment, 0) |
| 270 | case *ast.IgnoredDirective: |
| 271 | emitDirective(content, e.Span, len("N"), semProperty, e.Comment, emit) |
| 272 | case *ast.ApplyDirective: |
| 273 | emitDirective(content, e.Span, len("apply"), semProperty, e.Comment, emit) |
| 274 | case *ast.EndDirective: |
| 275 | emitDirective(content, e.Span, len("end"), semProperty, e.Comment, emit) |
| 276 | case *ast.BlankLine: |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | func visitTransaction(content string, t *ast.Transaction, emit semEmitFunc) { |
| 281 | emit(t.Date.Span, semDate, 0) |
| 282 | if t.SecondDate != nil { |
| 283 | emit(t.SecondDate.Span, semDate, 0) |
| 284 | } |
| 285 | if t.Status.Value != ast.StatusNone { |
| 286 | emit(t.Status.Span, semStatus, 0) |
| 287 | } |
| 288 | if t.Code != nil { |
| 289 | emit(t.Code.Span, semString, 0) |
| 290 | } |
| 291 | if t.Payee != nil { |
| 292 | emit(t.Payee.Span, semProperty, 0) |
| 293 | } |
| 294 | if t.Note != nil { |
| 295 | emit(t.Note.Span, semProperty, 0) |
| 296 | } |
| 297 | emitComment(t.Comment, emit) |
| 298 | for i := range t.HeaderComments { |
| 299 | emitComment(t.HeaderComments[i], emit) |
| 300 | } |
| 301 | for _, p := range t.Postings { |
| 302 | visitPosting(content, p, emit) |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | func visitPeriodicTransaction(content string, pt *ast.PeriodicTransaction, emit semEmitFunc) { |
| 307 | // ~ operator is at the start of the period span |
| 308 | emit(offsetSpan(pt.Span.Start.File, pt.Span.Start.Offset, pt.Span.Start.Offset+1), semOperator, 0) |
| 309 | |
| 310 | // The period span covers the whole expr, including any "from ... to ..." dates |
| 311 | if pt.Period.Span.End.Offset > pt.Period.Span.Start.Offset { |
| 312 | var dates []*ast.Date |
| 313 | if pt.Period.From != nil { |
| 314 | dates = append(dates, pt.Period.From) |
| 315 | } |
| 316 | if pt.Period.To != nil { |
| 317 | dates = append(dates, pt.Period.To) |
| 318 | } |
| 319 | pos := pt.Period.Span.Start.Offset |
| 320 | for _, d := range dates { |
| 321 | if d.Span.Start.Offset > pos { |
| 322 | emit(offsetSpan(pt.Period.Span.Start.File, pos, d.Span.Start.Offset), semProperty, 0) |
| 323 | } |
| 324 | emit(d.Span, semDate, 0) |
| 325 | pos = d.Span.End.Offset |
| 326 | } |
| 327 | if pos < pt.Period.Span.End.Offset { |
| 328 | emit(offsetSpan(pt.Period.Span.Start.File, pos, pt.Period.Span.End.Offset), semProperty, 0) |
| 329 | } |
| 330 | } |
| 331 | if pt.Description != nil { |
| 332 | emit(pt.Description.Span, semProperty, 0) |
| 333 | } |
| 334 | emitComment(pt.Comment, emit) |
| 335 | for i := range pt.HeaderComments { |
| 336 | emitComment(pt.HeaderComments[i], emit) |
| 337 | } |
| 338 | for _, p := range pt.Postings { |
| 339 | visitPosting(content, p, emit) |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | func visitAutomatedTransaction(content string, at *ast.AutomatedTransaction, emit semEmitFunc) { |
| 344 | // = operator is at the start of the expression span |
| 345 | emit(offsetSpan(at.Span.Start.File, at.Span.Start.Offset, at.Span.Start.Offset+1), semOperator, 0) |
| 346 | |
| 347 | if at.Expr.Value != "" { |
| 348 | emit(at.Expr.Span, semString, 0) |
| 349 | } |
| 350 | emitComment(at.Comment, emit) |
| 351 | for i := range at.HeaderComments { |
| 352 | emitComment(at.HeaderComments[i], emit) |
| 353 | } |
| 354 | for _, p := range at.Postings { |
| 355 | visitPosting(content, p, emit) |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | func visitPosting(content string, p *ast.Posting, emit semEmitFunc) { |
| 360 | if p.Status.Value != ast.StatusNone { |
| 361 | emit(p.Status.Span, semStatus, 0) |
| 362 | } |
| 363 | |
| 364 | // virtual brackets |
| 365 | if p.Type == ast.PostingVirtualUnbalanced || p.Type == ast.PostingVirtualBalanced { |
| 366 | // opening bracket |
| 367 | for off := p.Span.Start.Offset; off < p.Account.Span.Start.Offset && off < p.Span.End.Offset; off++ { |
| 368 | if content[off] == '(' || content[off] == '[' { |
| 369 | brSpan := token.Span{Start: offsetPos(p.Span.Start.File, off), End: offsetPos(p.Span.Start.File, off+1)} |
| 370 | emit(brSpan, semOperator, modifierAbstract) |
| 371 | break |
| 372 | } |
| 373 | } |
| 374 | // closing bracket |
| 375 | for off := p.Account.Span.End.Offset; off < p.Span.End.Offset; off++ { |
| 376 | if content[off] == ')' || content[off] == ']' { |
| 377 | brSpan := token.Span{Start: offsetPos(p.Span.Start.File, off), End: offsetPos(p.Span.Start.File, off+1)} |
| 378 | emit(brSpan, semOperator, modifierAbstract) |
| 379 | break |
| 380 | } |
| 381 | } |
| 382 | } |
| 383 | |
| 384 | emit(p.Account.Span, semAccount, 0) |
| 385 | |
| 386 | if p.Amount != nil { |
| 387 | semEmitAmount(content, p.Amount, emit) |
| 388 | } |
| 389 | if p.Cost != nil { |
| 390 | semEmitCost(content, p.Cost, emit) |
| 391 | } |
| 392 | if p.Balance != nil { |
| 393 | semEmitBalanceAssertion(content, p.Balance, emit) |
| 394 | } |
| 395 | emitComment(p.Comment, emit) |
| 396 | for i := range p.Comments { |
| 397 | emitComment(&p.Comments[i], emit) |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | // directiveKeyword returns the span of the leading keyword on a directive line. |
| 402 | func directiveKeyword(e token.Span, kw string) token.Span { |
| 403 | return token.Span{Start: e.Start, End: offsetPos(e.Start.File, e.Start.Offset+len(kw))} |
| 404 | } |
| 405 | |
| 406 | // directiveValue returns the trimmed span of the text after the keyword end |
| 407 | // offset, up to the inline comment or the end of the line. |
| 408 | func directiveValue(content string, e token.Span, comment *ast.Comment, kwEnd int) (token.Span, bool) { |
| 409 | end := e.End.Offset |
| 410 | if comment != nil { |
| 411 | end = comment.Span.Start.Offset |
| 412 | } |
| 413 | return betweenSpan(content, e.Start.File, kwEnd, end) |
| 414 | } |
| 415 | |
| 416 | func semEmitAmount(content string, a *ast.Amount, emit semEmitFunc) { |
| 417 | if a == nil { |
| 418 | return |
| 419 | } |
| 420 | hasCommodity := a.CommoditySpan.Start.Offset > 0 && a.CommoditySpan.End.Offset > 0 |
| 421 | if hasCommodity && a.CommodityPos == ast.CommodityBefore { |
| 422 | emit(a.CommoditySpan, semCommodity, 0) |
| 423 | semEmitQuantity(content, a, emit) |
| 424 | return |
| 425 | } |
| 426 | semEmitQuantity(content, a, emit) |
| 427 | if hasCommodity { |
| 428 | emit(a.CommoditySpan, semCommodity, 0) |
| 429 | } |
| 430 | } |
| 431 | |
| 432 | func semEmitQuantity(content string, a *ast.Amount, emit semEmitFunc) { |
| 433 | qStart, qEnd := quantitySpan(content, a) |
| 434 | if qEnd <= qStart { |
| 435 | return |
| 436 | } |
| 437 | mods := uint32(0) |
| 438 | if a.IsNegative { |
| 439 | mods |= modifierNegative |
| 440 | } |
| 441 | emit(offsetSpan(a.Span.Start.File, qStart, qEnd), semAmount, mods) |
| 442 | } |
| 443 | |
| 444 | func quantitySpan(content string, a *ast.Amount) (int, int) { |
| 445 | start, end := a.Span.Start.Offset, a.Span.End.Offset |
| 446 | switch { |
| 447 | case a.Commodity == "": |
| 448 | // bare quantity |
| 449 | case a.CommodityPos == ast.CommodityBefore: |
| 450 | // "$50.00" or "$ 50.00": quantity follows the commodity span |
| 451 | start = a.CommoditySpan.End.Offset |
| 452 | default: // CommodityAfter |
| 453 | // "50.00 USD" or "50.00 USD": quantity precedes the commodity |
| 454 | end = a.CommoditySpan.Start.Offset |
| 455 | } |
| 456 | for start < end && (content[start] == ' ' || content[start] == '\t') { |
| 457 | start++ |
| 458 | } |
| 459 | for end > start && (content[end-1] == ' ' || content[end-1] == '\t') { |
| 460 | end-- |
| 461 | } |
| 462 | return start, end |
| 463 | } |
| 464 | |
| 465 | type semEmitFunc func(tok token.Span, tokKind, modifier uint32) |
| 466 | |
| 467 | func emitComment(c *ast.Comment, emit semEmitFunc) { |
| 468 | if c == nil { |
| 469 | return |
| 470 | } |
| 471 | if len(c.Tags) == 0 { |
| 472 | emit(c.Span, semComment, 0) |
| 473 | return |
| 474 | } |
| 475 | pos := c.Span.Start.Offset |
| 476 | for _, t := range c.Tags { |
| 477 | if t.Span.Start.Offset > pos { |
| 478 | emit(offsetSpan(c.Span.Start.File, pos, t.Span.Start.Offset), semComment, 0) |
| 479 | } |
| 480 | emit(t.Span, semProperty, 0) |
| 481 | pos = t.Span.End.Offset |
| 482 | } |
| 483 | if pos < c.Span.End.Offset { |
| 484 | emit(offsetSpan(c.Span.Start.File, pos, c.Span.End.Offset), semComment, 0) |
| 485 | } |
| 486 | } |
| 487 | |
| 488 | func emitDirective(content string, e token.Span, kwLen int, valType uint32, comment *ast.Comment, emit semEmitFunc) { |
| 489 | kwEnd := e.Start.Offset + kwLen |
| 490 | emit(token.Span{Start: e.Start, End: offsetPos(e.Start.File, kwEnd)}, semDirective, 0) |
| 491 | if v, ok := directiveValue(content, e, comment, kwEnd); ok { |
| 492 | emit(v, valType, 0) |
| 493 | } |
| 494 | emitComment(comment, emit) |
| 495 | } |
| 496 | |
| 497 | func semEmitCost(content string, c *ast.Cost, emit semEmitFunc) { |
| 498 | if c.IsTotal { |
| 499 | emit(token.Span{Start: c.Span.Start, End: offsetPos(c.Span.Start.File, c.Span.Start.Offset+2)}, semOperator, 0) |
| 500 | } else { |
| 501 | emit(token.Span{Start: c.Span.Start, End: offsetPos(c.Span.Start.File, c.Span.Start.Offset+1)}, semOperator, 0) |
| 502 | } |
| 503 | semEmitAmount(content, &c.Amount, emit) |
| 504 | } |
| 505 | |
| 506 | func semEmitBalanceAssertion(content string, ba *ast.BalanceAssertion, emit semEmitFunc) { |
| 507 | // The operator is the run of '=', ':', '*' chars from the span start. |
| 508 | // The ':' of ':=' precedes the '=' token, so back up one offset. |
| 509 | opStart := ba.Span.Start.Offset |
| 510 | if ba.IsAssignment && opStart > 0 { |
| 511 | opStart-- |
| 512 | } |
| 513 | opEnd := opStart |
| 514 | for opEnd < ba.Span.End.Offset && (content[opEnd] == '=' || content[opEnd] == ':' || content[opEnd] == '*') { |
| 515 | opEnd++ |
| 516 | } |
| 517 | emit(token.Span{Start: offsetPos(ba.Span.Start.File, opStart), End: offsetPos(ba.Span.Start.File, opEnd)}, semOperator, 0) |
| 518 | semEmitAmount(content, &ba.Amount, emit) |
| 519 | if ba.Cost != nil { |
| 520 | semEmitCost(content, ba.Cost, emit) |
| 521 | } |
| 522 | } |
| 523 | |
| 524 | func semLexerFallback(content string, base []rawSpan, emit semEmitFunc) { |
| 525 | l := lexer.New("", []byte(content)) |
| 526 | |
| 527 | var commentStart, commentEnd int // 0 = not inside a comment line |
| 528 | lineStart := true // the next significant token starts a line |
| 529 | skipLine := false // the line starts with an unclassifiable token; emit nothing |
| 530 | i := 0 // next base span to compare against |
| 531 | |
| 532 | take := func(span token.Span, tokType uint32, mods uint32) { |
| 533 | for i < len(base) && base[i].span.End.Offset <= span.Start.Offset { |
| 534 | i++ |
| 535 | } |
| 536 | if i < len(base) && base[i].span.Start.Offset < span.End.Offset { |
| 537 | return // overlaps an AST token; AST wins |
| 538 | } |
| 539 | emit(span, tokType, mods) |
| 540 | } |
| 541 | |
| 542 | for { |
| 543 | tok := l.Next() |
| 544 | if tok.Type == token.EOF { |
| 545 | if commentStart > 0 { |
| 546 | take(token.Span{Start: offsetPos("", commentStart), End: offsetPos("", commentEnd)}, semComment, 0) |
| 547 | } |
| 548 | break |
| 549 | } |
| 550 | if tok.Type == token.NEWLINE { |
| 551 | if commentStart > 0 { |
| 552 | take(token.Span{Start: offsetPos("", commentStart), End: offsetPos("", commentEnd)}, semComment, 0) |
| 553 | commentStart, commentEnd = 0, 0 |
| 554 | } |
| 555 | lineStart, skipLine = true, false |
| 556 | continue |
| 557 | } |
| 558 | if tok.Type == token.WHITESPACE || tok.Type == token.INDENT { |
| 559 | continue |
| 560 | } |
| 561 | if lineStart { |
| 562 | lineStart = false |
| 563 | if !isLineStartToken(tok.Type) { |
| 564 | skipLine = true |
| 565 | } |
| 566 | } |
| 567 | if skipLine { |
| 568 | continue |
| 569 | } |
| 570 | tokType := semProperty |
| 571 | if commentStart > 0 { |
| 572 | tokType = semComment |
| 573 | if tok.Span.End.Offset > commentEnd { |
| 574 | commentEnd = tok.Span.End.Offset |
| 575 | } |
| 576 | continue |
| 577 | } |
| 578 | |
| 579 | switch tok.Type { |
| 580 | case token.SEMICOLON, token.HASH, token.PERCENT, token.STAR: |
| 581 | tokType = semComment |
| 582 | commentStart = tok.Span.Start.Offset |
| 583 | commentEnd = tok.Span.End.Offset |
| 584 | continue |
| 585 | case token.STRING: |
| 586 | tokType = semString |
| 587 | case token.DATE: |
| 588 | tokType = semDate |
| 589 | case token.INT, token.DECIMAL: |
| 590 | tokType = semAmount |
| 591 | case token.COMMODITYMARK: |
| 592 | tokType = semCommodity |
| 593 | case token.BANG: |
| 594 | tokType = semStatus |
| 595 | case token.AT, token.ATAT, token.EQ, token.EQEQ, token.EQEQEQ, token.EQSTAR: |
| 596 | tokType = semOperator |
| 597 | case token.COMMENTKW, token.ACCOUNT, token.COMMODITY, token.INCLUDE, |
| 598 | token.ALIAS, token.PAYEE, token.TAG, token.APPLY, token.END, |
| 599 | token.YEAR, token.DECIMALMARK, token.D, token.P, token.N, token.C: |
| 600 | tokType = semDirective |
| 601 | } |
| 602 | take(tok.Span, tokType, 0) |
| 603 | } |
| 604 | } |
| 605 | |
| 606 | func isLineStartToken(t token.Type) bool { |
| 607 | switch t { |
| 608 | case token.DATE, token.TILDE, token.EQ, token.BANG, token.AT, |
| 609 | token.SEMICOLON, token.HASH, token.PERCENT, token.STAR, |
| 610 | token.COMMENTKW, token.ACCOUNT, token.COMMODITY, token.INCLUDE, |
| 611 | token.ALIAS, token.PAYEE, token.TAG, token.APPLY, token.END, |
| 612 | token.YEAR, token.DECIMALMARK, token.D, token.P, token.N, token.C: |
| 613 | return true |
| 614 | } |
| 615 | return false |
| 616 | } |
| 617 | |
| 618 | func encodeSemTokens(tokens []semanticToken) []uint32 { |
| 619 | if len(tokens) == 0 { |
| 620 | return nil |
| 621 | } |
| 622 | slices.SortFunc(tokens, func(a, b semanticToken) int { |
| 623 | if a.line != b.line { |
| 624 | return int(a.line) - int(b.line) |
| 625 | } |
| 626 | return int(a.col) - int(b.col) |
| 627 | }) |
| 628 | data := make([]uint32, 0, len(tokens)*5) |
| 629 | var prevLine, prevCol uint32 |
| 630 | for _, t := range tokens { |
| 631 | var deltaLine, deltaCol uint32 |
| 632 | if t.line == prevLine { |
| 633 | deltaLine = 0 |
| 634 | deltaCol = t.col - prevCol |
| 635 | } else { |
| 636 | deltaLine = t.line - prevLine |
| 637 | deltaCol = t.col |
| 638 | } |
| 639 | data = append(data, deltaLine, deltaCol, t.length, t.tokenType, t.modifiers) |
| 640 | prevLine = t.line |
| 641 | prevCol = t.col |
| 642 | } |
| 643 | return data |
| 644 | } |
| 645 | |
| 646 | // betweenSpan returns the span of the text between two offsets, trimmed of surrounding whitespace. |
| 647 | func betweenSpan(content, file string, start, end int) (token.Span, bool) { |
| 648 | for start < end && (content[start] == ' ' || content[start] == '\t') { |
| 649 | start++ |
| 650 | } |
| 651 | for end > start && (content[end-1] == ' ' || content[end-1] == '\t' || content[end-1] == '\n' || content[end-1] == '\r') { |
| 652 | end-- |
| 653 | } |
| 654 | if end <= start { |
| 655 | return token.Span{}, false |
| 656 | } |
| 657 | return token.Span{Start: offsetPos(file, start), End: offsetPos(file, end)}, true |
| 658 | } |
| 659 | |
| 660 | func offsetPos(file string, offset int) token.Pos { return token.Pos{File: file, Offset: offset} } |
| 661 | func offsetSpan(file string, start, end int) token.Span { |
| 662 | return token.Span{Start: offsetPos(file, start), End: offsetPos(file, end)} |
| 663 | } |