lexer.c (14101B)
1 #include <fcntl.h> 2 3 #include "dlib.h" 4 #include "lexer.h" 5 6 Sv keywords[] = { 7 [T_RETURN - T_KEYWORDS_] = sv_literal("return"), 8 [T_IF - T_KEYWORDS_] = sv_literal("if"), 9 [T_ELSE - T_KEYWORDS_] = sv_literal("else"), 10 [T_WHILE - T_KEYWORDS_] = sv_literal("while"), 11 [T_FOR - T_KEYWORDS_] = sv_literal("for"), 12 [T_SWITCH - T_KEYWORDS_] = sv_literal("switch"), 13 [T_CASE - T_KEYWORDS_] = sv_literal("case"), 14 [T_DEFAULT - T_KEYWORDS_] = sv_literal("default"), 15 [T_BREAK - T_KEYWORDS_] = sv_literal("break"), 16 [T_CONTINUE - T_KEYWORDS_] = sv_literal("continue"), 17 [T_TYPEDEF - T_KEYWORDS_] = sv_literal("typedef"), 18 [T_FUNC - T_KEYWORDS_] = sv_literal("func"), 19 [T_VAR - T_KEYWORDS_] = sv_literal("var"), 20 [T_CONST - T_KEYWORDS_] = sv_literal("const"), 21 [T_STRUCT - T_KEYWORDS_] = sv_literal("struct"), 22 [T_UNION - T_KEYWORDS_] = sv_literal("union"), 23 [T_ENUM - T_KEYWORDS_] = sv_literal("enum"), 24 [T_OPERATOR - T_KEYWORDS_] = sv_literal("operator"), 25 }; 26 27 static_assert(T_COUNT_ - T_KEYWORDS_ == 18, "keywords[] wrong length"); 28 int keywords_count = (sizeof(keywords) / sizeof(keywords[0])); 29 30 void 31 sb_append_loc(Sb *sb, Loc loc) 32 { 33 sb_printf(sb, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column); 34 } 35 36 void 37 sb_append_token_kind(Sb *sb, Token_kind kind) 38 { 39 if (kind >= 32 && kind <= 127) sb_append_char(sb, kind); 40 else if (kind == T_ERROR_) sb_append_cstr(sb, "T_ERROR_"); 41 else if (kind == T_EOF) sb_append_cstr(sb, "T_EOF"); 42 else if (kind == T_INVALID) sb_append_cstr(sb, "T_INVALID"); 43 else if (kind == T_WHITESPACE) sb_append_cstr(sb, "T_WHITESPACE"); 44 else if (kind == T_IDENT) sb_append_cstr(sb, "T_IDENT"); 45 else if (kind == T_NUMBER) sb_append_cstr(sb, "T_NUMBER"); 46 else if (kind == T_SQ_STRING) sb_append_cstr(sb, "T_SQ_STRING"); 47 else if (kind == T_DQ_STRING) sb_append_cstr(sb, "T_DQ_STRING"); 48 else if (kind == T_RETURN) sb_append_cstr(sb, "T_RETURN"); 49 else if (kind == T_IF) sb_append_cstr(sb, "T_IF"); 50 else if (kind == T_ELSE) sb_append_cstr(sb, "T_ELSE"); 51 else if (kind == T_WHILE) sb_append_cstr(sb, "T_WHILE"); 52 else if (kind == T_FOR) sb_append_cstr(sb, "T_FOR"); 53 else if (kind == T_SWITCH) sb_append_cstr(sb, "T_SWITCH"); 54 else if (kind == T_CASE) sb_append_cstr(sb, "T_CASE"); 55 else if (kind == T_DEFAULT) sb_append_cstr(sb, "T_DEFAULT"); 56 else if (kind == T_BREAK) sb_append_cstr(sb, "T_BREAK"); 57 else if (kind == T_CONTINUE) sb_append_cstr(sb, "T_CONTINUE"); 58 else if (kind == T_TYPEDEF) sb_append_cstr(sb, "T_TYPEDEF"); 59 else if (kind == T_FUNC) sb_append_cstr(sb, "T_FUNC"); 60 else if (kind == T_VAR) sb_append_cstr(sb, "T_VAR"); 61 else if (kind == T_CONST) sb_append_cstr(sb, "T_CONST"); 62 else if (kind == T_STRUCT) sb_append_cstr(sb, "T_STRUCT"); 63 else if (kind == T_UNION) sb_append_cstr(sb, "T_UNION"); 64 else if (kind == T_ENUM) sb_append_cstr(sb, "T_ENUM"); 65 else if (kind == T_OPERATOR) sb_append_cstr(sb, "T_OPERATOR"); 66 else sb_append_cstr(sb, "(unknown token)"); 67 } 68 69 void 70 print_loc(Loc loc) 71 { 72 fprintf(stderr, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column); 73 } 74 75 void 76 print_loc_pad(Loc loc, int min_width) 77 { 78 int len = fprintf(stderr, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column); 79 if (len < min_width) { 80 fprintf(stderr, "%*s", min_width - len, ""); 81 } 82 } 83 84 void 85 debug_print_token(Token t) 86 { 87 print_loc(t.loc); 88 if (t.kind == T_ERROR_) { 89 fprintf(stderr, "ERROR"); 90 } else if (t.kind >= 32 && t.kind < 127) { 91 fprintf(stderr, "'%c'", t.kind); 92 } else if (t.kind == T_EOF) { 93 fprintf(stderr, "EOF"); 94 } else if (t.kind == T_WHITESPACE || t.kind == T_WHITESPACE_NL) { 95 if (t.kind == T_WHITESPACE) { 96 fprintf(stderr, "WHITESPACE("); 97 } else { 98 fprintf(stderr, "WHITESPACE_NL("); 99 } 100 char last_char = '?'; 101 int last_char_count = 0; 102 for (int i = 0; i < t.sv.count; i++) { 103 char c = '?'; 104 switch (t.sv.items[i]) { 105 case ' ' : c = 's'; break; 106 case '\t': c = 't'; break; 107 case '\n': c = 'n'; break; 108 case '\r': c = 'r'; break; 109 case '\v': c = 'v'; break; 110 case '\f': c = 'f'; break; 111 } 112 if (last_char != c) { 113 if (last_char_count > 1) { 114 fprintf(stderr, "(%i)", last_char_count); 115 } 116 last_char_count = 0; 117 fputc(c, stderr); 118 } else if (!last_char_count) { 119 fputc(c, stderr); 120 } 121 last_char = c; 122 last_char_count++; 123 } 124 if (last_char_count > 1) { 125 fprintf(stderr, "(%i)", last_char_count); 126 } 127 fputc(')', stderr); 128 } else if (t.kind == T_IDENT) { 129 fprintf(stderr, "IDENT(%.*s)", (int) t.sv.count, t.sv.items); 130 } else if (t.kind == T_NUMBER) { 131 fprintf(stderr, "NUMBER(%.*s)", (int) t.sv.count, t.sv.items); 132 } else if (t.kind == T_SQ_STRING || t.kind == T_DQ_STRING) { 133 fprintf(stderr, "STRING(%.*s)", (int) t.sv.count, t.sv.items); 134 } else if (t.kind == T_RETURN) { 135 fprintf(stderr, "return"); 136 } else if (t.kind == T_IF) { 137 fprintf(stderr, "if"); 138 } else if (t.kind == T_ELSE) { 139 fprintf(stderr, "else"); 140 } else if (t.kind == T_WHILE) { 141 fprintf(stderr, "while"); 142 } else if (t.kind == T_FOR) { 143 fprintf(stderr, "for"); 144 } else if (t.kind == T_SWITCH) { 145 fprintf(stderr, "switch"); 146 } else if (t.kind == T_CASE) { 147 fprintf(stderr, "case"); 148 } else if (t.kind == T_DEFAULT) { 149 fprintf(stderr, "default"); 150 } else if (t.kind == T_BREAK) { 151 fprintf(stderr, "break"); 152 } else if (t.kind == T_CONTINUE) { 153 fprintf(stderr, "continue"); 154 } else if (t.kind == T_TYPEDEF) { 155 fprintf(stderr, "typedef"); 156 } else if (t.kind == T_FUNC) { 157 fprintf(stderr, "func"); 158 } else if (t.kind == T_VAR) { 159 fprintf(stderr, "var"); 160 } else if (t.kind == T_CONST) { 161 fprintf(stderr, "const"); 162 } else if (t.kind == T_STRUCT) { 163 fprintf(stderr, "struct"); 164 } else if (t.kind == T_UNION) { 165 fprintf(stderr, "union"); 166 } else if (t.kind == T_ENUM) { 167 fprintf(stderr, "enum"); 168 } else if (t.kind == T_OPERATOR) { 169 fprintf(stderr, "operator"); 170 } else { 171 fprintf(stderr, "unknown token 0x%02x", t.kind); 172 } 173 fprintf(stderr, "\n"); 174 } 175 176 int 177 next_char(Lexer *lexer) 178 { 179 if (!lexer->source.count) return EOF; 180 char c = *lexer->source.items++; 181 lexer->source.count--; 182 if (c == '\n') { 183 lexer->current_loc.column = 1; 184 lexer->current_loc.line++; 185 } else { 186 lexer->current_loc.column++; 187 } 188 return c; 189 } 190 191 int 192 peek_char(Lexer *lexer) 193 { 194 if (!lexer->source.count || lexer->source.count == 1) return EOF; 195 return *lexer->source.items; 196 } 197 198 int 199 next_char_if(Lexer *lexer, int (*pred)(int)) 200 { 201 if (!lexer->source.count) return EOF; 202 if (pred(*lexer->source.items)) { 203 return next_char(lexer); 204 } 205 return CHAR_INVALID; 206 } 207 208 int 209 is_ident_char(int c) 210 { 211 return c == '_' || 212 (c >= '0' && c <= '9') || 213 (c >= 'A' && c <= 'Z') || 214 (c >= 'a' && c <= 'z'); 215 } 216 217 int 218 is_symbol_char(int c) 219 { 220 return c == '+' || 221 c == '-' || 222 c == '*' || 223 c == '/' || 224 c == '%' || 225 c == '(' || 226 c == ')' || 227 c == '[' || 228 c == ']' || 229 c == '<' || 230 c == '>' || 231 c == '{' || 232 c == '}' || 233 c == '=' || 234 c == '&' || 235 c == '|' || 236 c == '^' || 237 c == '~' || 238 c == '?' || 239 c == '!' || 240 c == '.' || 241 c == ',' || 242 c == ':' || 243 c == ';' || 244 /* c == '`' || */ 245 /* c == '#' || */ 246 /* c == '@' || */ 247 /* c == '$' || */ 248 0; 249 } 250 251 bool 252 is_operator_token(Token_kind kind) { 253 return kind == '+' || 254 kind == '-' || 255 kind == '*' || 256 kind == '/' || 257 kind == '%' || 258 /* kind == '(' || */ 259 /* kind == ')' || */ 260 /* kind == '[' || */ 261 /* kind == ']' || */ 262 kind == '<' || 263 kind == '>' || 264 /* kind == '{' || */ 265 /* kind == '}' || */ 266 kind == '=' || 267 kind == '&' || 268 kind == '|' || 269 kind == '^' || 270 kind == '~' || 271 kind == '?' || 272 kind == '!' || 273 /* kind == '.' || */ 274 /* kind == ',' || */ 275 kind == T_IDENT || 276 0; 277 } 278 279 int 280 skip_comment(Lexer *lexer) 281 { 282 Loc start_loc = lexer->current_loc; 283 int len = 0; 284 if (sv_starts_with_cstr(lexer->source, "/*")) { 285 len = 2; 286 next_char(lexer); 287 next_char(lexer); 288 while (lexer->source.count && !sv_starts_with_cstr(lexer->source, "*/")) { 289 if (sv_starts_with_cstr(lexer->source, "/*")) { 290 len += skip_comment(lexer); 291 } else { 292 len++; 293 next_char(lexer); 294 } 295 } 296 if (!lexer->source.count) { 297 print_loc(start_loc); 298 fprintf(stderr, "Error: unterminated comment\n"); 299 return -1; 300 } 301 next_char(lexer); 302 next_char(lexer); 303 len += 2; 304 } else if (sv_starts_with_cstr(lexer->source, "//")) { 305 char next; 306 while ((next = next_char(lexer)) != '\n' && next != EOF) len++; 307 } 308 return len; 309 } 310 311 Token 312 next_token(Lexer *lexer) 313 { 314 Loc start_loc = lexer->current_loc; 315 Token result = { .loc = start_loc, .sv = { .items = lexer->source.items, .count = 0 } }; 316 317 if (lexer->last_tok.kind != T_WHITESPACE) { 318 lexer->last_valid_loc = start_loc; 319 } 320 321 if (!lexer->source.count) { 322 result.kind = T_EOF; 323 lexer->last_tok = result; 324 return result; 325 } 326 327 int c = next_char_if(lexer, isspace); 328 size_t len = 0; 329 330 { 331 bool whitespace = false; 332 int comment_len = 0; 333 while (isspace(c) || (comment_len = skip_comment(lexer))) { 334 if (comment_len < 0) { 335 return (Token) { 336 .kind = T_ERROR_, 337 .loc = start_loc, 338 }; 339 } else if (comment_len > 0) { 340 len += comment_len; 341 } else { 342 len++; 343 } 344 whitespace = true; 345 while ((c = next_char_if(lexer, isspace)) >= 0) { 346 len++; 347 } 348 } 349 if (whitespace) { 350 if (start_loc.line < lexer->current_loc.line) { 351 result.kind = T_WHITESPACE_NL; 352 } else { 353 result.kind = T_WHITESPACE; 354 } 355 result.sv.count = len; 356 lexer->last_tok = result; 357 return result; 358 } 359 } 360 361 len = 1; 362 363 c = next_char(lexer); 364 365 if (c == '-' && isdigit(peek_char(lexer))) { 366 c = next_char(lexer); 367 len++; 368 } 369 if (isdigit(c)) { 370 while (next_char_if(lexer, isdigit) >= 0) len++; 371 result.kind = T_NUMBER; 372 result.sv.count = len; 373 lexer->last_tok = result; 374 return result; 375 } 376 377 if (is_ident_char(c)) { 378 while (next_char_if(lexer, is_ident_char) >= 0) len++; 379 result.sv.count = len; 380 result.kind = get_keyword_kind(result.sv); 381 lexer->last_tok = result; 382 return result; 383 } 384 385 if (is_symbol_char(c)) { 386 result.kind = c; 387 result.sv.count = len; 388 lexer->last_tok = result; 389 return result; 390 } 391 392 if (c == '\'' || c == '"') { 393 char next; 394 while ((next = next_char(lexer)) != EOF && next != c) { 395 len++; 396 if (next == '\\') { 397 next_char(lexer); 398 len++; 399 } 400 } 401 if (next != c) { 402 print_loc(start_loc); 403 fprintf(stderr, "Error: unterminated %s quoted string\n", c == '"' ? "double" : "single"); 404 405 return (Token) { 406 .kind = T_ERROR_, 407 .loc = start_loc, 408 }; 409 } 410 result.kind = c == '"' ? T_DQ_STRING : T_SQ_STRING; 411 result.sv.count = ++len; 412 lexer->last_tok = result; 413 return result; 414 } 415 416 print_loc(start_loc); 417 fprintf(stderr, "Error: unknown character '%c'\n", c); 418 419 return (Token) { 420 .kind = T_ERROR_, 421 .loc = start_loc, 422 }; 423 } 424 425 Token 426 peek_token(Lexer *lexer) 427 { 428 Lexer peek = *lexer; 429 return next_token(&peek); 430 } 431 432 Token_kind 433 get_keyword_kind(Sv sv) 434 { 435 int i = 0; 436 for (;i < keywords_count; i++) { 437 if (sv_equal(sv, keywords[i])) { 438 return i + T_KEYWORDS_; 439 } 440 } 441 return T_IDENT; 442 } 443 444 Token 445 next_token_if(Lexer *lexer, int (*pred)(Token)) 446 { 447 Lexer peek = *lexer; 448 Token t = next_token(&peek); 449 if (pred(t)) { 450 *lexer = peek; 451 return t; 452 } 453 return (Token) { 454 .kind = T_INVALID, 455 .loc = lexer->current_loc, 456 }; 457 } 458 459 void 460 skip_whitespace(Lexer *lexer) 461 { 462 Lexer peek = *lexer; 463 Token t = next_token(&peek); 464 if (t.kind == T_WHITESPACE || t.kind == T_WHITESPACE_NL) { 465 *lexer = peek; 466 } 467 } 468 469 bool 470 expect_token_kind(Lexer *lexer, Token_kind kind) 471 { 472 Token t = next_token(lexer); 473 if (t.kind != kind) { 474 Sb error = {0}; 475 476 /* sb_append_loc(&error, t.loc); */ 477 sb_append_loc(&error, lexer->last_valid_loc); 478 sb_append_cstr(&error, "Error: expected token "); 479 sb_append_token_kind(&error, kind); 480 sb_append_cstr(&error, ", but got "); 481 sb_append_token_kind(&error, t.kind); 482 483 fprintf(stderr, "%.*s\n", (int) error.count, error.items); 484 485 sb_release(&error); 486 return false; 487 /* return (Token) { */ 488 /* .kind = T_ERROR_, */ 489 /* .loc = t.loc, */ 490 /* }; */ 491 } 492 return true; 493 } 494 495 bool 496 expect_end_statement_impl(Lexer *lexer, const char *file, int line) 497 { 498 if (lexer->last_tok.kind == T_WHITESPACE_NL) { 499 fprintf(stderr, "%s:%i: expect_end_statement hack\n", file, line); 500 return true; 501 /* return lexer->last_tok; */ 502 } 503 Token t = next_token(lexer); 504 if (t.kind == ';' || t.kind == T_WHITESPACE_NL) { 505 /* return t; */ 506 return true; 507 } 508 if (t.kind == T_WHITESPACE) { 509 if (peek_token(lexer).kind == ';') { 510 next_token(lexer); 511 return true; 512 } 513 } 514 Sb error = {0}; 515 516 /* sb_append_loc(&error, t.loc); */ 517 sb_append_loc(&error, lexer->last_valid_loc); 518 sb_append_cstr(&error, "Error: expected ; or newline, but got "); 519 sb_append_token_kind(&error, t.kind); 520 521 fprintf(stderr, "%.*s\n", (int) error.count, error.items); 522 523 sb_release(&error); 524 return false; 525 /* return (Token) { */ 526 /* .kind = T_ERROR_, */ 527 /* .loc = t.loc, */ 528 /* }; */ 529 }