compiler_experiment

Unnamed repository; edit this file 'description' to name the repository.
git clone https://git.deepztream.com/compiler_experiment
Log | Files | Refs

lexer.c (10905B)

      1 #include <fcntl.h>
      2 
      3 #include "dlib.h"
      4 #include "lexer.h"
      5 
      6 Sv keywords[] = {
      7   [T_RETURN - T_KEYWORDS_]   = sv_literal("return"),
      8   [T_IF - T_KEYWORDS_]       = sv_literal("if"),
      9   [T_ELSE - T_KEYWORDS_]     = sv_literal("else"),
     10   [T_WHILE - T_KEYWORDS_]    = sv_literal("while"),
     11   [T_FOR - T_KEYWORDS_]      = sv_literal("for"),
     12   [T_SWITCH - T_KEYWORDS_]   = sv_literal("switch"),
     13   [T_CASE - T_KEYWORDS_]     = sv_literal("case"),
     14   [T_DEFAULT - T_KEYWORDS_]  = sv_literal("default"),
     15   [T_BREAK - T_KEYWORDS_]    = sv_literal("break"),
     16   [T_CONTINUE - T_KEYWORDS_] = sv_literal("continue"),
     17   [T_TYPEDEF - T_KEYWORDS_]  = sv_literal("typedef"),
     18   [T_FUNC - T_KEYWORDS_]     = sv_literal("func"),
     19   [T_VAR - T_KEYWORDS_]      = sv_literal("var"),
     20   [T_CONST - T_KEYWORDS_]    = sv_literal("const"),
     21   [T_STRUCT - T_KEYWORDS_]   = sv_literal("struct"),
     22   [T_UNION - T_KEYWORDS_]    = sv_literal("union"),
     23   [T_ENUM - T_KEYWORDS_]     = sv_literal("enum"),
     24   [T_OPERATOR - T_KEYWORDS_] = sv_literal("operator"),
     25 };
     26 
     27 static_assert(T_COUNT_ - T_KEYWORDS_ == 18, "keywords[] wrong length");
     28 int keywords_count = (sizeof(keywords) / sizeof(keywords[0]));
     29 
     30 void
     31 sb_append_loc(Sb *sb, Loc loc)
     32 {
     33   sb_printf(sb, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column);
     34 }
     35 
     36 void
     37 sb_append_token_kind(Sb *sb, Token_kind kind)
     38 {
     39        if (kind >= 32 && kind <= 127) sb_append_char(sb, kind);
     40   else if (kind == T_ERROR_)          sb_append_cstr(sb, "T_ERROR_");
     41   else if (kind == T_EOF)             sb_append_cstr(sb, "T_EOF");
     42   else if (kind == T_INVALID)         sb_append_cstr(sb, "T_INVALID");
     43   else if (kind == T_WHITESPACE)      sb_append_cstr(sb, "T_WHITESPACE");
     44   else if (kind == T_IDENT)           sb_append_cstr(sb, "T_IDENT");
     45   else if (kind == T_NUMBER)          sb_append_cstr(sb, "T_NUMBER");
     46   else if (kind == T_SQ_STRING)       sb_append_cstr(sb, "T_SQ_STRING");
     47   else if (kind == T_DQ_STRING)       sb_append_cstr(sb, "T_DQ_STRING");
     48   else if (kind == T_RETURN)          sb_append_cstr(sb, "T_RETURN");
     49   else if (kind == T_IF)              sb_append_cstr(sb, "T_IF");
     50   else if (kind == T_ELSE)            sb_append_cstr(sb, "T_ELSE");
     51   else if (kind == T_WHILE)           sb_append_cstr(sb, "T_WHILE");
     52   else if (kind == T_FOR)             sb_append_cstr(sb, "T_FOR");
     53   else if (kind == T_SWITCH)          sb_append_cstr(sb, "T_SWITCH");
     54   else if (kind == T_CASE)            sb_append_cstr(sb, "T_CASE");
     55   else if (kind == T_DEFAULT)         sb_append_cstr(sb, "T_DEFAULT");
     56   else if (kind == T_BREAK)           sb_append_cstr(sb, "T_BREAK");
     57   else if (kind == T_CONTINUE)        sb_append_cstr(sb, "T_CONTINUE");
     58   else if (kind == T_TYPEDEF)         sb_append_cstr(sb, "T_TYPEDEF");
     59   else if (kind == T_FUNC)            sb_append_cstr(sb, "T_FUNC");
     60   else if (kind == T_VAR)             sb_append_cstr(sb, "T_VAR");
     61   else if (kind == T_CONST)           sb_append_cstr(sb, "T_CONST");
     62   else if (kind == T_STRUCT)          sb_append_cstr(sb, "T_STRUCT");
     63   else if (kind == T_UNION)           sb_append_cstr(sb, "T_UNION");
     64   else if (kind == T_ENUM)            sb_append_cstr(sb, "T_ENUM");
     65   else if (kind == T_OPERATOR)        sb_append_cstr(sb, "T_OPERATOR");
     66   else                                sb_append_cstr(sb, "(unknown token)");
     67 }
     68 
     69 void
     70 print_loc(Loc loc)
     71 {
     72   fprintf(stderr, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column);
     73 }
     74 
     75 void
     76 print_loc_pad(Loc loc, int min_width)
     77 {
     78   int len = fprintf(stderr, "%.*s:%li:%li: ", (int) loc.filename.count, loc.filename.items, loc.line, loc.column);
     79   if (len < min_width) {
     80     fprintf(stderr, "%*s", min_width - len, "");
     81   }
     82 }
     83 
     84 void
     85 debug_print_token(Token t)
     86 {
     87   print_loc(t.loc);
     88   if (t.kind == T_ERROR_) {
     89     fprintf(stderr, "ERROR");
     90   } else if (t.kind >= 32 && t.kind < 127) {
     91     fprintf(stderr, "'%c'", t.kind);
     92   } else if (t.kind == T_EOF)  {
     93     fprintf(stderr, "EOF");
     94   } else if (t.kind == T_WHITESPACE)  {
     95     fprintf(stderr, "WHITESPACE(%.*s)", (int) t.sv.count, t.sv.items);
     96   } else if (t.kind == T_IDENT)  {
     97     fprintf(stderr, "IDENT(%.*s)", (int) t.sv.count, t.sv.items);
     98   } else if (t.kind == T_EOF)  {
     99     fprintf(stderr, "NUMBER(%.*s)", (int) t.sv.count, t.sv.items);
    100   } else if (t.kind == T_SQ_STRING || t.kind == T_DQ_STRING)  {
    101     fprintf(stderr, "STRING(%.*s)", (int) t.sv.count, t.sv.items);
    102   } else if (t.kind == T_RETURN) {
    103     fprintf(stderr, "return");
    104   } else if (t.kind == T_IF) {
    105     fprintf(stderr, "if");
    106   } else if (t.kind == T_ELSE) {
    107     fprintf(stderr, "else");
    108   } else if (t.kind == T_WHILE) {
    109     fprintf(stderr, "while");
    110   } else if (t.kind == T_FOR) {
    111     fprintf(stderr, "for");
    112   } else if (t.kind == T_SWITCH) {
    113     fprintf(stderr, "switch");
    114   } else if (t.kind == T_CASE) {
    115     fprintf(stderr, "case");
    116   } else if (t.kind == T_DEFAULT) {
    117     fprintf(stderr, "default");
    118   } else if (t.kind == T_BREAK) {
    119     fprintf(stderr, "break");
    120   } else if (t.kind == T_CONTINUE) {
    121     fprintf(stderr, "continue");
    122   } else if (t.kind == T_TYPEDEF) {
    123     fprintf(stderr, "typedef");
    124   } else if (t.kind == T_FUNC) {
    125     fprintf(stderr, "func");
    126   } else if (t.kind == T_VAR) {
    127     fprintf(stderr, "var");
    128   } else if (t.kind == T_CONST) {
    129     fprintf(stderr, "const");
    130   } else if (t.kind == T_STRUCT) {
    131     fprintf(stderr, "struct");
    132   } else if (t.kind == T_UNION) {
    133     fprintf(stderr, "union");
    134   } else if (t.kind == T_ENUM) {
    135     fprintf(stderr, "enum");
    136   } else if (t.kind == T_OPERATOR) {
    137     fprintf(stderr, "operator");
    138   } else  {
    139     fprintf(stderr, "unknown token 0x%02x", t.kind);
    140   }
    141   fprintf(stderr, "\n");
    142 }
    143 
    144 int
    145 next_char(Lexer *lexer)
    146 {
    147   if (!lexer->source.count) return EOF;
    148   char c = *lexer->source.items++;
    149   lexer->source.count--;
    150   if (c == '\n') {
    151     lexer->current_loc.column = 1;
    152     lexer->current_loc.line++;
    153   } else {
    154     lexer->current_loc.column++;
    155   }
    156   return c;
    157 }
    158 
    159 int
    160 next_char_if(Lexer *lexer, int (*pred)(int))
    161 {
    162   if (!lexer->source.count) return EOF;
    163   if (pred(*lexer->source.items)) {
    164     return next_char(lexer);
    165   }
    166   return CHAR_INVALID;
    167 }
    168 
    169 int
    170 is_ident_char(int c)
    171 {
    172   return c == '_' ||
    173         (c >= '0' && c <= '9') ||
    174         (c >= 'A' && c <= 'Z') ||
    175         (c >= 'a' && c <= 'z');
    176 }
    177 
    178 int
    179 is_symbol_char(int c)
    180 {
    181   return c == '+' ||
    182          c == '-' ||
    183          c == '*' ||
    184          c == '/' ||
    185          c == '%' ||
    186          c == '(' ||
    187          c == ')' ||
    188          c == '[' ||
    189          c == ']' ||
    190          c == '<' ||
    191          c == '>' ||
    192          c == '{' ||
    193          c == '}' ||
    194          c == '=' ||
    195          c == '&' ||
    196          c == '|' ||
    197          c == '^' ||
    198          c == '~' ||
    199          c == '?' ||
    200          c == '!' ||
    201          c == '.' ||
    202          c == ',' ||
    203          c == ':' ||
    204          c == ';' ||
    205          /* c == '`' || */
    206          /* c == '#' || */
    207          /* c == '@' || */
    208          /* c == '$' || */
    209          0;
    210 }
    211 
    212 int
    213 skip_comment(Lexer *lexer)
    214 {
    215   Loc start_loc = lexer->current_loc;
    216   int len = 0;
    217   if (sv_starts_with_cstr(lexer->source, "/*")) {
    218     len = 2;
    219     next_char(lexer);
    220     next_char(lexer);
    221     while (lexer->source.count && !sv_starts_with_cstr(lexer->source, "*/")) {
    222       if (sv_starts_with_cstr(lexer->source, "/*")) {
    223         len += skip_comment(lexer);
    224       } else {
    225         len++;
    226         next_char(lexer);
    227       }
    228     }
    229     if (!lexer->source.count) {
    230       print_loc(start_loc);
    231       fprintf(stderr, "Error: unterminated comment\n");
    232       return -1;
    233     }
    234     next_char(lexer);
    235     next_char(lexer);
    236     len += 2;
    237   } else if (sv_starts_with_cstr(lexer->source, "//")) {
    238     char next;
    239     while ((next = next_char(lexer)) != '\n' && next != EOF) len++;
    240   }
    241   return len;
    242 }
    243 
    244 Token
    245 next_token(Lexer *lexer)
    246 {
    247   Loc start_loc = lexer->current_loc;
    248   Token result = { .loc = start_loc, .sv = { .items = lexer->source.items, .count = 0 } };
    249 
    250   if (!lexer->source.count) {
    251     result.kind = T_EOF;
    252     return result;
    253   }
    254 
    255   int c = next_char_if(lexer, isspace);
    256   size_t len = 0;
    257 
    258   {
    259     bool whitespace = false;
    260     int comment_len = 0;
    261     while (isspace(c) || (comment_len = skip_comment(lexer))) {
    262       if (comment_len < 0) {
    263         return (Token) {
    264           .kind = T_ERROR_,
    265           .loc = start_loc,
    266         };
    267       } else if (comment_len > 0) {
    268         len += comment_len;
    269       } else {
    270         len++;
    271       }
    272       whitespace = true;
    273       while ((c = next_char_if(lexer, isspace)) >= 0) len++;
    274     }
    275     if (whitespace) {
    276       result.kind = T_WHITESPACE;
    277       result.sv.count = len;
    278       return result;
    279     }
    280   }
    281 
    282   len = 1;
    283 
    284   c = next_char(lexer);
    285 
    286   if (isdigit(c)) {
    287     while (next_char_if(lexer, isdigit) >= 0) len++;
    288     result.kind = T_NUMBER;
    289     result.sv.count = len;
    290     return result;
    291   }
    292 
    293   if (is_ident_char(c)) {
    294     while (next_char_if(lexer, is_ident_char) >= 0) len++;
    295     result.sv.count = len;
    296     result.kind = get_keyword_kind(result.sv);
    297     return result;
    298   }
    299 
    300   if (is_symbol_char(c)) {
    301     result.kind = c;
    302     result.sv.count = len;
    303     return result;
    304   }
    305 
    306   if (c == '\'' || c == '"') {
    307     char next;
    308     while ((next = next_char(lexer)) != EOF && next != c) {
    309       len++;
    310       if (next == '\\') {
    311         next_char(lexer);
    312         len++;
    313       }
    314     }
    315     if (next != c) {
    316       print_loc(start_loc);
    317       fprintf(stderr, "Error: unterminated %s quoted string\n", c == '"' ? "double" : "single");
    318 
    319       return (Token) {
    320         .kind = T_ERROR_,
    321         .loc = start_loc,
    322       };
    323     }
    324     result.kind = c == '"' ? T_DQ_STRING : T_SQ_STRING;
    325     result.sv.count = ++len;
    326     return result;
    327   }
    328 
    329   print_loc(start_loc);
    330   fprintf(stderr, "Error: unknown character '%c'\n", c);
    331 
    332   return (Token) {
    333     .kind = T_ERROR_,
    334     .loc = start_loc,
    335   };
    336 }
    337 
    338 Token
    339 peek_token(Lexer *lexer)
    340 {
    341   Lexer peek = *lexer;
    342   return next_token(&peek);
    343 }
    344 
    345 Token_kind
    346 get_keyword_kind(Sv sv)
    347 {
    348   int i = 0;
    349   for (;i < keywords_count; i++) {
    350     if (sv_equal(sv, keywords[i])) {
    351       return i + T_KEYWORDS_;
    352     }
    353   }
    354   return T_IDENT;
    355 }
    356 
    357 Token
    358 next_token_if(Lexer *lexer, int (*pred)(Token))
    359 {
    360   Lexer peek = *lexer;
    361   Token t = next_token(&peek);
    362   if (pred(t)) {
    363     *lexer = peek;
    364     return t;
    365   }
    366   return (Token) {
    367     .kind = T_INVALID,
    368     .loc = lexer->current_loc,
    369   };
    370 }
    371 
    372 void
    373 skip_whitespace(Lexer *lexer)
    374 {
    375   Lexer peek = *lexer;
    376   Token t = next_token(&peek);
    377   if (t.kind == T_WHITESPACE) {
    378     *lexer = peek;
    379   }
    380 }
    381 
    382 Token
    383 expect_token_kind(Lexer *lexer, Token_kind kind)
    384 {
    385   Token t = next_token(lexer);
    386   if (t.kind != kind) {
    387     Sb error = {0};
    388 
    389     sb_append_loc(&error, t.loc);
    390     sb_append_cstr(&error, "Error: expected token ");
    391     sb_append_token_kind(&error, kind);
    392     sb_append_cstr(&error, ", but got ");
    393     sb_append_token_kind(&error, t.kind);
    394 
    395     fprintf(stderr, "%.*s\n", (int) error.count, error.items);
    396 
    397     sb_release(&error);
    398     return (Token) {
    399       .kind = T_ERROR_,
    400       .loc = t.loc,
    401     };
    402   }
    403   return t;
    404 }