json2go/lexer_test.go (view raw)
Oleksandr Smirnov
Oleksandr Smirnov
olexsmir@gmail.com refactor: use an actual parser instead of reflection..., 11 days ago
olexsmir@gmail.com refactor: use an actual parser instead of reflection..., 11 days ago
| 1 | package json2go |
| 2 | |
| 3 | import "testing" |
| 4 | |
| 5 | func TestLexer_Next(t *testing.T) { |
| 6 | input := `{"name": "John", "age": 30, "active": true, "score": 3.14, "nothing": null}` |
| 7 | tests := []struct { |
| 8 | t TokenType |
| 9 | l string |
| 10 | }{ |
| 11 | {LBRACE, "{"}, |
| 12 | {STRING, "name"}, |
| 13 | {COLON, ":"}, |
| 14 | {INDENT, " "}, |
| 15 | {STRING, "John"}, |
| 16 | {COMMA, ","}, |
| 17 | {INDENT, " "}, |
| 18 | {STRING, "age"}, |
| 19 | {COLON, ":"}, |
| 20 | {INDENT, " "}, |
| 21 | {NUMBER, "30"}, |
| 22 | {COMMA, ","}, |
| 23 | {INDENT, " "}, |
| 24 | {STRING, "active"}, |
| 25 | {COLON, ":"}, |
| 26 | {INDENT, " "}, |
| 27 | {BOOL, "true"}, |
| 28 | {COMMA, ","}, |
| 29 | {INDENT, " "}, |
| 30 | {STRING, "score"}, |
| 31 | {COLON, ":"}, |
| 32 | {INDENT, " "}, |
| 33 | {DECIMAL, "3.14"}, |
| 34 | {COMMA, ","}, |
| 35 | {INDENT, " "}, |
| 36 | {STRING, "nothing"}, |
| 37 | {COLON, ":"}, |
| 38 | {INDENT, " "}, |
| 39 | {NULL, "null"}, |
| 40 | {RBRACE, "}"}, |
| 41 | {EOF, ""}, |
| 42 | } |
| 43 | |
| 44 | l := NewLexer([]byte(input)) |
| 45 | for i, tt := range tests { |
| 46 | tok := l.Next() |
| 47 | if tok.Type != tt.t { |
| 48 | t.Errorf("tests[%d] - wrong token type. expected=%q, got=%q (literal=%q)", i, tt.t, tok.Type, tok.Literal) |
| 49 | } |
| 50 | if tok.Literal != tt.l { |
| 51 | t.Errorf("tests[%d] - wrong literal. expected=%q, got=%q", i, tt.l, tok.Literal) |
| 52 | } |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | func TestLexer_comments(t *testing.T) { |
| 57 | input := `{ |
| 58 | // line comment |
| 59 | "key": /* block comment */ "value" |
| 60 | }` |
| 61 | tests := []struct { |
| 62 | t TokenType |
| 63 | l string |
| 64 | }{ |
| 65 | {LBRACE, "{"}, |
| 66 | {NEWLINE, "\n"}, |
| 67 | {INDENT, "\t\t"}, |
| 68 | {COMMENTLINE, " line comment"}, |
| 69 | {NEWLINE, "\n"}, |
| 70 | {INDENT, "\t\t"}, |
| 71 | {STRING, "key"}, |
| 72 | {COLON, ":"}, |
| 73 | {INDENT, " "}, |
| 74 | {COMMENTBLOCK, " block comment "}, |
| 75 | {INDENT, " "}, |
| 76 | {STRING, "value"}, |
| 77 | {NEWLINE, "\n"}, |
| 78 | {INDENT, "\t"}, |
| 79 | {RBRACE, "}"}, |
| 80 | {EOF, ""}, |
| 81 | } |
| 82 | |
| 83 | l := NewLexer([]byte(input)) |
| 84 | for i, tt := range tests { |
| 85 | tok := l.Next() |
| 86 | if tok.Type != tt.t { |
| 87 | t.Errorf("tests[%d] - wrong token type. expected=%q, got=%q (literal=%q)", i, tt.t, tok.Type, tok.Literal) |
| 88 | } |
| 89 | if tok.Literal != tt.l { |
| 90 | t.Errorf("tests[%d] - wrong literal. expected=%q, got=%q", i, tt.l, tok.Literal) |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | func TestLexer_numbers(t *testing.T) { |
| 96 | input := `[0, -1, 42, 3.14, -0.5, 1e10, 2.5E-3]` |
| 97 | tests := []struct { |
| 98 | t TokenType |
| 99 | l string |
| 100 | }{ |
| 101 | {LBRACKET, "["}, |
| 102 | {NUMBER, "0"}, |
| 103 | {COMMA, ","}, |
| 104 | {INDENT, " "}, |
| 105 | {NUMBER, "-1"}, |
| 106 | {COMMA, ","}, |
| 107 | {INDENT, " "}, |
| 108 | {NUMBER, "42"}, |
| 109 | {COMMA, ","}, |
| 110 | {INDENT, " "}, |
| 111 | {DECIMAL, "3.14"}, |
| 112 | {COMMA, ","}, |
| 113 | {INDENT, " "}, |
| 114 | {DECIMAL, "-0.5"}, |
| 115 | {COMMA, ","}, |
| 116 | {INDENT, " "}, |
| 117 | {DECIMAL, "1e10"}, |
| 118 | {COMMA, ","}, |
| 119 | {INDENT, " "}, |
| 120 | {DECIMAL, "2.5E-3"}, |
| 121 | {RBRACKET, "]"}, |
| 122 | {EOF, ""}, |
| 123 | } |
| 124 | |
| 125 | l := NewLexer([]byte(input)) |
| 126 | for i, tt := range tests { |
| 127 | tok := l.Next() |
| 128 | if tok.Type != tt.t { |
| 129 | t.Errorf("tests[%d] - wrong token type. expected=%q, got=%q (literal=%q)", i, tt.t, tok.Type, tok.Literal) |
| 130 | } |
| 131 | if tok.Literal != tt.l { |
| 132 | t.Errorf("tests[%d] - wrong literal. expected=%q, got=%q", i, tt.l, tok.Literal) |
| 133 | } |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | func TestLexer_illegal(t *testing.T) { |
| 138 | tests := []struct{ input, l string }{ |
| 139 | {`"unterminated`, "unterminated string"}, |
| 140 | {`/* unterminated`, "unterminated block comment"}, |
| 141 | {`01`, "leading zero in number"}, |
| 142 | {`-`, "invalid number"}, |
| 143 | {`1.`, "expected digit after decimal point"}, |
| 144 | {`1e`, "expected digit in exponent"}, |
| 145 | } |
| 146 | for i, tt := range tests { |
| 147 | l := NewLexer([]byte(tt.input)) |
| 148 | tok := l.Next() |
| 149 | if tok.Type != ILLEGAL { |
| 150 | t.Errorf("tests[%d] - expected ILLEGAL, got=%q (literal=%q)", i, tok.Type, tok.Literal) |
| 151 | } |
| 152 | if tok.Literal != tt.l { |
| 153 | t.Errorf("tests[%d] - wrong error literal. expected=%q, got=%q", i, tt.l, tok.Literal) |
| 154 | } |
| 155 | } |
| 156 | } |