compiler_experiment

Unnamed repository; edit this file 'description' to name the repository.
git clone https://git.deepztream.com/compiler_experiment
Log | Files | Refs | Submodules

lexer.c (14101B)


      1 #include <fcntl.h>
      2 
      3 #include "dlib.h"
      4 #include "lexer.h"
      5 
      6 Sv keywords[] = {
      7   [T_RETURN - T_KEYWORDS_]   = sv_literal("return"),
      8   [T_IF - T_KEYWORDS_]       = sv_literal("if"),
      9   [T_ELSE - T_KEYWORDS_]     = sv_literal("else"),
     10   [T_WHILE - T_KEYWORDS_]    = sv_literal("while"),
     11   [T_FOR - T_KEYWORDS_]      = sv_literal("for"),
     12   [T_SWITCH - T_KEYWORDS_]   = sv_literal("switch"),
     13   [T_CASE - T_KEYWORDS_]     = sv_literal("case"),
     14   [T_DEFAULT - T_KEYWORDS_]  = sv_literal("default"),
     15   [T_BREAK - T_KEYWORDS_]    = sv_literal("break"),
     16   [T_CONTINUE - T_KEYWORDS_] = sv_literal("continue"),
     17   [T_TYPEDEF - T_KEYWORDS_]  = sv_literal("typedef"),
     18   [T_FUNC - T_KEYWORDS_]     = sv_literal("func"),
     19   [T_VAR - T_KEYWORDS_]      = sv_literal("var"),
     20   [T_CONST - T_KEYWORDS_]    = sv_literal("const"),
     21   [T_STRUCT - T_KEYWORDS_]   = sv_literal("struct"),
     22   [T_UNION - T_KEYWORDS_]    = sv_literal("union"),
     23   [T_ENUM - T_KEYWORDS_]     = sv_literal("enum"),
     24   [T_OPERATOR - T_KEYWORDS_] = sv_literal("operator"),
     25 };
     26 
     27 static_assert(T_COUNT_ - T_KEYWORDS_ == 18, "keywords[] wrong length");
     28 int keywords_count = (sizeof(keywords) / sizeof(keywords[0]));
     29 
     30 void
     31 sb_append_loc(Sb *sb, Loc loc)
     32 {
     33   sb_printf(sb, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column);
     34 }
     35 
     36 void
     37 sb_append_token_kind(Sb *sb, Token_kind kind)
     38 {
     39        if (kind >= 32 && kind <= 127) sb_append_char(sb, kind);
     40   else if (kind == T_ERROR_)          sb_append_cstr(sb, "T_ERROR_");
     41   else if (kind == T_EOF)             sb_append_cstr(sb, "T_EOF");
     42   else if (kind == T_INVALID)         sb_append_cstr(sb, "T_INVALID");
     43   else if (kind == T_WHITESPACE)      sb_append_cstr(sb, "T_WHITESPACE");
     44   else if (kind == T_IDENT)           sb_append_cstr(sb, "T_IDENT");
     45   else if (kind == T_NUMBER)          sb_append_cstr(sb, "T_NUMBER");
     46   else if (kind == T_SQ_STRING)       sb_append_cstr(sb, "T_SQ_STRING");
     47   else if (kind == T_DQ_STRING)       sb_append_cstr(sb, "T_DQ_STRING");
     48   else if (kind == T_RETURN)          sb_append_cstr(sb, "T_RETURN");
     49   else if (kind == T_IF)              sb_append_cstr(sb, "T_IF");
     50   else if (kind == T_ELSE)            sb_append_cstr(sb, "T_ELSE");
     51   else if (kind == T_WHILE)           sb_append_cstr(sb, "T_WHILE");
     52   else if (kind == T_FOR)             sb_append_cstr(sb, "T_FOR");
     53   else if (kind == T_SWITCH)          sb_append_cstr(sb, "T_SWITCH");
     54   else if (kind == T_CASE)            sb_append_cstr(sb, "T_CASE");
     55   else if (kind == T_DEFAULT)         sb_append_cstr(sb, "T_DEFAULT");
     56   else if (kind == T_BREAK)           sb_append_cstr(sb, "T_BREAK");
     57   else if (kind == T_CONTINUE)        sb_append_cstr(sb, "T_CONTINUE");
     58   else if (kind == T_TYPEDEF)         sb_append_cstr(sb, "T_TYPEDEF");
     59   else if (kind == T_FUNC)            sb_append_cstr(sb, "T_FUNC");
     60   else if (kind == T_VAR)             sb_append_cstr(sb, "T_VAR");
     61   else if (kind == T_CONST)           sb_append_cstr(sb, "T_CONST");
     62   else if (kind == T_STRUCT)          sb_append_cstr(sb, "T_STRUCT");
     63   else if (kind == T_UNION)           sb_append_cstr(sb, "T_UNION");
     64   else if (kind == T_ENUM)            sb_append_cstr(sb, "T_ENUM");
     65   else if (kind == T_OPERATOR)        sb_append_cstr(sb, "T_OPERATOR");
     66   else                                sb_append_cstr(sb, "(unknown token)");
     67 }
     68 
     69 void
     70 print_loc(Loc loc)
     71 {
     72   fprintf(stderr, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column);
     73 }
     74 
     75 void
     76 print_loc_pad(Loc loc, int min_width)
     77 {
     78   int len = fprintf(stderr, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column);
     79   if (len < min_width) {
     80     fprintf(stderr, "%*s", min_width - len, "");
     81   }
     82 }
     83 
     84 void
     85 debug_print_token(Token t)
     86 {
     87   print_loc(t.loc);
     88   if (t.kind == T_ERROR_) {
     89     fprintf(stderr, "ERROR");
     90   } else if (t.kind >= 32 && t.kind < 127) {
     91     fprintf(stderr, "'%c'", t.kind);
     92   } else if (t.kind == T_EOF)  {
     93     fprintf(stderr, "EOF");
     94   } else if (t.kind == T_WHITESPACE || t.kind == T_WHITESPACE_NL)  {
     95     if (t.kind == T_WHITESPACE) {
     96       fprintf(stderr, "WHITESPACE(");
     97     } else {
     98       fprintf(stderr, "WHITESPACE_NL(");
     99     }
    100     char last_char = '?';
    101     int last_char_count = 0;
    102     for (int i = 0; i < t.sv.count; i++) {
    103       char c = '?';
    104       switch (t.sv.items[i]) {
    105         case ' ' : c = 's'; break;
    106         case '\t': c = 't'; break;
    107         case '\n': c = 'n'; break;
    108         case '\r': c = 'r'; break;
    109         case '\v': c = 'v'; break;
    110         case '\f': c = 'f'; break;
    111       }
    112       if (last_char != c) {
    113         if (last_char_count > 1) {
    114           fprintf(stderr, "(%i)", last_char_count);
    115         }
    116         last_char_count = 0;
    117         fputc(c, stderr);
    118       } else if (!last_char_count) {
    119         fputc(c, stderr);
    120       }
    121       last_char = c;
    122       last_char_count++;
    123     }
    124     if (last_char_count > 1) {
    125       fprintf(stderr, "(%i)", last_char_count);
    126     }
    127     fputc(')', stderr);
    128   } else if (t.kind == T_IDENT)  {
    129     fprintf(stderr, "IDENT(%.*s)", (int) t.sv.count, t.sv.items);
    130   } else if (t.kind == T_NUMBER)  {
    131     fprintf(stderr, "NUMBER(%.*s)", (int) t.sv.count, t.sv.items);
    132   } else if (t.kind == T_SQ_STRING || t.kind == T_DQ_STRING)  {
    133     fprintf(stderr, "STRING(%.*s)", (int) t.sv.count, t.sv.items);
    134   } else if (t.kind == T_RETURN) {
    135     fprintf(stderr, "return");
    136   } else if (t.kind == T_IF) {
    137     fprintf(stderr, "if");
    138   } else if (t.kind == T_ELSE) {
    139     fprintf(stderr, "else");
    140   } else if (t.kind == T_WHILE) {
    141     fprintf(stderr, "while");
    142   } else if (t.kind == T_FOR) {
    143     fprintf(stderr, "for");
    144   } else if (t.kind == T_SWITCH) {
    145     fprintf(stderr, "switch");
    146   } else if (t.kind == T_CASE) {
    147     fprintf(stderr, "case");
    148   } else if (t.kind == T_DEFAULT) {
    149     fprintf(stderr, "default");
    150   } else if (t.kind == T_BREAK) {
    151     fprintf(stderr, "break");
    152   } else if (t.kind == T_CONTINUE) {
    153     fprintf(stderr, "continue");
    154   } else if (t.kind == T_TYPEDEF) {
    155     fprintf(stderr, "typedef");
    156   } else if (t.kind == T_FUNC) {
    157     fprintf(stderr, "func");
    158   } else if (t.kind == T_VAR) {
    159     fprintf(stderr, "var");
    160   } else if (t.kind == T_CONST) {
    161     fprintf(stderr, "const");
    162   } else if (t.kind == T_STRUCT) {
    163     fprintf(stderr, "struct");
    164   } else if (t.kind == T_UNION) {
    165     fprintf(stderr, "union");
    166   } else if (t.kind == T_ENUM) {
    167     fprintf(stderr, "enum");
    168   } else if (t.kind == T_OPERATOR) {
    169     fprintf(stderr, "operator");
    170   } else  {
    171     fprintf(stderr, "unknown token 0x%02x", t.kind);
    172   }
    173   fprintf(stderr, "\n");
    174 }
    175 
    176 int
    177 next_char(Lexer *lexer)
    178 {
    179   if (!lexer->source.count) return EOF;
    180   char c = *lexer->source.items++;
    181   lexer->source.count--;
    182   if (c == '\n') {
    183     lexer->current_loc.column = 1;
    184     lexer->current_loc.line++;
    185   } else {
    186     lexer->current_loc.column++;
    187   }
    188   return c;
    189 }
    190 
    191 int
    192 peek_char(Lexer *lexer)
    193 {
    194   if (!lexer->source.count || lexer->source.count == 1) return EOF;
    195   return *lexer->source.items;
    196 }
    197 
    198 int
    199 next_char_if(Lexer *lexer, int (*pred)(int))
    200 {
    201   if (!lexer->source.count) return EOF;
    202   if (pred(*lexer->source.items)) {
    203     return next_char(lexer);
    204   }
    205   return CHAR_INVALID;
    206 }
    207 
    208 int
    209 is_ident_char(int c)
    210 {
    211   return c == '_' ||
    212         (c >= '0' && c <= '9') ||
    213         (c >= 'A' && c <= 'Z') ||
    214         (c >= 'a' && c <= 'z');
    215 }
    216 
    217 int
    218 is_symbol_char(int c)
    219 {
    220   return c == '+' ||
    221          c == '-' ||
    222          c == '*' ||
    223          c == '/' ||
    224          c == '%' ||
    225          c == '(' ||
    226          c == ')' ||
    227          c == '[' ||
    228          c == ']' ||
    229          c == '<' ||
    230          c == '>' ||
    231          c == '{' ||
    232          c == '}' ||
    233          c == '=' ||
    234          c == '&' ||
    235          c == '|' ||
    236          c == '^' ||
    237          c == '~' ||
    238          c == '?' ||
    239          c == '!' ||
    240          c == '.' ||
    241          c == ',' ||
    242          c == ':' ||
    243          c == ';' ||
    244          /* c == '`' || */
    245          /* c == '#' || */
    246          /* c == '@' || */
    247          /* c == '$' || */
    248          0;
    249 }
    250 
    251 bool
    252 is_operator_token(Token_kind kind) {
    253   return kind == '+' ||
    254          kind == '-' ||
    255          kind == '*' ||
    256          kind == '/' ||
    257          kind == '%' ||
    258          /* kind == '(' || */
    259          /* kind == ')' || */
    260          /* kind == '[' || */
    261          /* kind == ']' || */
    262          kind == '<' ||
    263          kind == '>' ||
    264          /* kind == '{' || */
    265          /* kind == '}' || */
    266          kind == '=' ||
    267          kind == '&' ||
    268          kind == '|' ||
    269          kind == '^' ||
    270          kind == '~' ||
    271          kind == '?' ||
    272          kind == '!' ||
    273          /* kind == '.' || */
    274          /* kind == ',' || */
    275          kind == T_IDENT ||
    276          0;
    277 }
    278 
    279 int
    280 skip_comment(Lexer *lexer)
    281 {
    282   Loc start_loc = lexer->current_loc;
    283   int len = 0;
    284   if (sv_starts_with_cstr(lexer->source, "/*")) {
    285     len = 2;
    286     next_char(lexer);
    287     next_char(lexer);
    288     while (lexer->source.count && !sv_starts_with_cstr(lexer->source, "*/")) {
    289       if (sv_starts_with_cstr(lexer->source, "/*")) {
    290         len += skip_comment(lexer);
    291       } else {
    292         len++;
    293         next_char(lexer);
    294       }
    295     }
    296     if (!lexer->source.count) {
    297       print_loc(start_loc);
    298       fprintf(stderr, "Error: unterminated comment\n");
    299       return -1;
    300     }
    301     next_char(lexer);
    302     next_char(lexer);
    303     len += 2;
    304   } else if (sv_starts_with_cstr(lexer->source, "//")) {
    305     char next;
    306     while ((next = next_char(lexer)) != '\n' && next != EOF) len++;
    307   }
    308   return len;
    309 }
    310 
    311 Token
    312 next_token(Lexer *lexer)
    313 {
    314   Loc start_loc = lexer->current_loc;
    315   Token result = { .loc = start_loc, .sv = { .items = lexer->source.items, .count = 0 } };
    316 
    317   if (lexer->last_tok.kind != T_WHITESPACE) {
    318     lexer->last_valid_loc = start_loc;
    319   }
    320 
    321   if (!lexer->source.count) {
    322     result.kind = T_EOF;
    323     lexer->last_tok = result;
    324     return result;
    325   }
    326 
    327   int c = next_char_if(lexer, isspace);
    328   size_t len = 0;
    329 
    330   {
    331     bool whitespace = false;
    332     int comment_len = 0;
    333     while (isspace(c) || (comment_len = skip_comment(lexer))) {
    334       if (comment_len < 0) {
    335         return (Token) {
    336           .kind = T_ERROR_,
    337           .loc = start_loc,
    338         };
    339       } else if (comment_len > 0) {
    340         len += comment_len;
    341       } else {
    342         len++;
    343       }
    344       whitespace = true;
    345       while ((c = next_char_if(lexer, isspace)) >= 0) {
    346         len++;
    347       }
    348     }
    349     if (whitespace) {
    350       if (start_loc.line < lexer->current_loc.line) {
    351         result.kind = T_WHITESPACE_NL;
    352       } else {
    353         result.kind = T_WHITESPACE;
    354       }
    355       result.sv.count = len;
    356       lexer->last_tok = result;
    357       return result;
    358     }
    359   }
    360 
    361   len = 1;
    362 
    363   c = next_char(lexer);
    364 
    365   if (c == '-' && isdigit(peek_char(lexer))) {
    366     c = next_char(lexer);
    367     len++;
    368   }
    369   if (isdigit(c)) {
    370     while (next_char_if(lexer, isdigit) >= 0) len++;
    371     result.kind = T_NUMBER;
    372     result.sv.count = len;
    373     lexer->last_tok = result;
    374     return result;
    375   }
    376 
    377   if (is_ident_char(c)) {
    378     while (next_char_if(lexer, is_ident_char) >= 0) len++;
    379     result.sv.count = len;
    380     result.kind = get_keyword_kind(result.sv);
    381     lexer->last_tok = result;
    382     return result;
    383   }
    384 
    385   if (is_symbol_char(c)) {
    386     result.kind = c;
    387     result.sv.count = len;
    388     lexer->last_tok = result;
    389     return result;
    390   }
    391 
    392   if (c == '\'' || c == '"') {
    393     char next;
    394     while ((next = next_char(lexer)) != EOF && next != c) {
    395       len++;
    396       if (next == '\\') {
    397         next_char(lexer);
    398         len++;
    399       }
    400     }
    401     if (next != c) {
    402       print_loc(start_loc);
    403       fprintf(stderr, "Error: unterminated %s quoted string\n", c == '"' ? "double" : "single");
    404 
    405       return (Token) {
    406         .kind = T_ERROR_,
    407         .loc = start_loc,
    408       };
    409     }
    410     result.kind = c == '"' ? T_DQ_STRING : T_SQ_STRING;
    411     result.sv.count = ++len;
    412     lexer->last_tok = result;
    413     return result;
    414   }
    415 
    416   print_loc(start_loc);
    417   fprintf(stderr, "Error: unknown character '%c'\n", c);
    418 
    419   return (Token) {
    420     .kind = T_ERROR_,
    421     .loc = start_loc,
    422   };
    423 }
    424 
    425 Token
    426 peek_token(Lexer *lexer)
    427 {
    428   Lexer peek = *lexer;
    429   return next_token(&peek);
    430 }
    431 
    432 Token_kind
    433 get_keyword_kind(Sv sv)
    434 {
    435   int i = 0;
    436   for (;i < keywords_count; i++) {
    437     if (sv_equal(sv, keywords[i])) {
    438       return i + T_KEYWORDS_;
    439     }
    440   }
    441   return T_IDENT;
    442 }
    443 
    444 Token
    445 next_token_if(Lexer *lexer, int (*pred)(Token))
    446 {
    447   Lexer peek = *lexer;
    448   Token t = next_token(&peek);
    449   if (pred(t)) {
    450     *lexer = peek;
    451     return t;
    452   }
    453   return (Token) {
    454     .kind = T_INVALID,
    455     .loc = lexer->current_loc,
    456   };
    457 }
    458 
    459 void
    460 skip_whitespace(Lexer *lexer)
    461 {
    462   Lexer peek = *lexer;
    463   Token t = next_token(&peek);
    464   if (t.kind == T_WHITESPACE || t.kind == T_WHITESPACE_NL) {
    465     *lexer = peek;
    466   }
    467 }
    468 
    469 bool
    470 expect_token_kind(Lexer *lexer, Token_kind kind)
    471 {
    472   Token t = next_token(lexer);
    473   if (t.kind != kind) {
    474     Sb error = {0};
    475 
    476     /* sb_append_loc(&error, t.loc); */
    477     sb_append_loc(&error, lexer->last_valid_loc);
    478     sb_append_cstr(&error, "Error: expected token ");
    479     sb_append_token_kind(&error, kind);
    480     sb_append_cstr(&error, ", but got ");
    481     sb_append_token_kind(&error, t.kind);
    482 
    483     fprintf(stderr, "%.*s\n", (int) error.count, error.items);
    484 
    485     sb_release(&error);
    486     return false;
    487     /* return (Token) { */
    488     /*   .kind = T_ERROR_, */
    489     /*   .loc = t.loc, */
    490     /* }; */
    491   }
    492   return true;
    493 }
    494 
    495 bool
    496 expect_end_statement_impl(Lexer *lexer, const char *file, int line)
    497 {
    498   if (lexer->last_tok.kind == T_WHITESPACE_NL) {
    499     fprintf(stderr, "%s:%i: expect_end_statement hack\n", file, line);
    500     return true;
    501     /* return lexer->last_tok; */
    502   }
    503   Token t = next_token(lexer);
    504   if (t.kind == ';' || t.kind == T_WHITESPACE_NL) {
    505     /* return t; */
    506     return true;
    507   }
    508   if (t.kind == T_WHITESPACE) {
    509     if (peek_token(lexer).kind == ';') {
    510       next_token(lexer);
    511       return true;
    512     }
    513   }
    514   Sb error = {0};
    515 
    516   /* sb_append_loc(&error, t.loc); */
    517   sb_append_loc(&error, lexer->last_valid_loc);
    518   sb_append_cstr(&error, "Error: expected ; or newline, but got ");
    519   sb_append_token_kind(&error, t.kind);
    520 
    521   fprintf(stderr, "%.*s\n", (int) error.count, error.items);
    522 
    523   sb_release(&error);
    524   return false;
    525   /* return (Token) { */
    526   /*   .kind = T_ERROR_, */
    527   /*   .loc = t.loc, */
    528   /* }; */
    529 }