/* spl_lexer.c — SPL lexical analyzer */ #include "spl_lexer.h" #include #include #include /* Character classification */ static int is_ident_start(char c) { return isalpha((unsigned char)c) || c == '_'; } static int is_ident_cont(char c) { return isalnum((unsigned char)c) || c == '_'; } /* Keyword lookup: if ident is a keyword, return its token type, else TOK_IDENT */ static spl_tok_type_t keyword_type(const char *ident, usize len) { #define X(name, enum_name, dummy) \ if (len == sizeof(#name) - 1 && memcmp(ident, #name, len) == 0) \ return enum_name; KEYWORD_TABLE #undef X return TOK_IDENT; } /* Escape sequence decoder — returns the decoded character, * advances *s past the escape sequence, returns 0 on success, * non-zero on error. */ int spl_decode_escape(const char **s, char *out) { if (**s != '\\') { *out = **s; (*s)++; return 0; } (*s)++; /* skip backslash */ switch (**s) { case 'n': *out = '\n'; break; case 't': *out = '\t'; break; case 'r': *out = '\r'; break; case '\\': *out = '\\'; break; case '"': *out = '"'; break; case '\'': *out = '\''; break; case '0': *out = '\0'; break; case 'x': { (*s)++; char hex[3] = {0, 0, 0}; int i; for (i = 0; i < 2 && isxdigit((unsigned char)**s); i++, (*s)++) { hex[i] = **s; } if (i == 0) return -1; *out = (char)strtol(hex, NULL, 16); return 0; } default: return -1; } (*s)++; return 0; } spl_tok_vec_t spl_lex(const char *source, const char *fname) { spl_tok_vec_t toks; vec_init(toks); usize line = 1; usize col = 1; usize offset = 0; usize len = strlen(source); while (offset < len) { const char *start = source + offset; char c = *start; /* Skip whitespace (but not newlines — emit TOK_ENDLINE) */ if (c == ' ' || c == '\t' || c == '\r') { offset++; col++; continue; } /* Newline */ if (c == '\n') { spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = TOK_ENDLINE; tok.lexeme = start; tok.len = 1; tok.fname = fname; tok.offset = offset; tok.line = line; tok.col = col; vec_push(toks, tok); offset++; line++; col = 1; continue; } /* Line comment */ if (c == '/' && offset + 1 < len && source[offset + 1] == '/') { const char *nl = (const char *)memchr(start, '\n', len - offset); usize clen = nl ? (usize)(nl - start) : (len - offset); spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = TOK_LINE_COMMENT; tok.lexeme = start; tok.len = clen; tok.fname = fname; tok.offset = offset; tok.line = line; tok.col = col; vec_push(toks, tok); offset += clen; col += clen; continue; } /* Block comment */ if (c == '/' && offset + 1 < len && source[offset + 1] == '*') { offset += 2; col += 2; usize depth = 1; while (offset + 1 < len && depth > 0) { if (source[offset] == '*' && source[offset + 1] == '/') { depth--; offset += 2; col += 2; if (depth == 0) break; } else if (source[offset] == '/' && source[offset + 1] == '*') { depth++; offset += 2; col += 2; } else { if (source[offset] == '\n') { line++; col = 1; } else { col++; } offset++; } } continue; } /* Char literal: 'x' */ if (c == '\'') { offset++; col++; char buf[16]; int bi = 0; memset(buf, 0, sizeof(buf)); if (offset < len) { const char *cp = source + offset; if (spl_decode_escape(&cp, &buf[bi])) { buf[bi] = source[offset]; cp = source + offset + 1; } bi++; offset = (usize)(cp - source); col += (usize)(cp - (start + 1)); } if (offset < len && source[offset] == '\'') { offset++; col++; } spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = TOK_CHAR_LITERAL; tok.lexeme = start; tok.len = (usize)((source + offset) - start); tok.fname = fname; tok.offset = start - source; tok.line = line; tok.col = col - tok.len; vec_push(toks, tok); continue; } /* String literal: "..." */ if (c == '"') { offset++; col++; while (offset < len) { if (source[offset] == '\\') { offset++; col++; if (offset < len) { offset++; col++; } } else if (source[offset] == '"') { offset++; col++; break; } else if (source[offset] == '\n') { line++; col = 1; offset++; } else { offset++; col++; } } spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = TOK_STRING_LITERAL; tok.lexeme = start; tok.len = (usize)((source + offset) - start); tok.fname = fname; tok.offset = start - source; tok.line = line; tok.col = col - tok.len; vec_push(toks, tok); continue; } /* Identifiers and keywords */ if (is_ident_start(c)) { const char *id_start = start; usize id_len = 0; while (offset < len && is_ident_cont(source[offset])) { offset++; id_len++; col++; } spl_tok_type_t tt = keyword_type(id_start, id_len); spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = tt; tok.lexeme = id_start; tok.len = id_len; tok.fname = fname; tok.offset = id_start - source; tok.line = line; tok.col = col - id_len; vec_push(toks, tok); continue; } /* Numbers: integers and floats */ if (isdigit((unsigned char)c)) { const char *num_start = start; int is_float = 0; /* Check for hex/bin/oct prefix */ if (c == '0' && offset + 1 < len) { char nc = source[offset + 1]; if (nc == 'x' || nc == 'X') { /* Hex literal */ offset += 2; col += 2; while (offset < len && isxdigit((unsigned char)source[offset])) { offset++; col++; } goto emit_int; } if (nc == 'b' || nc == 'B') { /* Binary literal */ offset += 2; col += 2; while (offset < len && (source[offset] == '0' || source[offset] == '1')) { offset++; col++; } goto emit_int; } if (nc == 'o' || nc == 'O') { /* Octal literal */ offset += 2; col += 2; while (offset < len && source[offset] >= '0' && source[offset] <= '7') { offset++; col++; } goto emit_int; } } /* Decimal integer or float */ while (offset < len && isdigit((unsigned char)source[offset])) { offset++; col++; } if (offset < len && source[offset] == '.' && offset + 1 < len && isdigit((unsigned char)source[offset + 1])) { is_float = 1; offset++; col++; while (offset < len && isdigit((unsigned char)source[offset])) { offset++; col++; } } if (is_float) { spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = TOK_FLOAT_LITERAL; tok.lexeme = num_start; tok.len = (usize)((source + offset) - num_start); tok.fname = fname; tok.offset = num_start - source; tok.line = line; tok.col = col - tok.len; vec_push(toks, tok); continue; } emit_int: { spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = TOK_INT_LITERAL; tok.lexeme = num_start; tok.len = (usize)((source + offset) - num_start); tok.fname = fname; tok.offset = num_start - source; tok.line = line; tok.col = col - tok.len; vec_push(toks, tok); continue; } } /* Multi-character operators (longest match) */ /* Helper: try to match a two-char operator */ #define TRY_OP2(c1, c2, tok2, tok1) \ if (c == (c1) && offset + 1 < len && source[offset + 1] == (c2)) { \ spl_tok_type_t op_type = (tok2); \ usize op_len = 2; \ spl_tok_t tok; \ memset(&tok, 0, sizeof(tok)); \ tok.type = op_type; \ tok.lexeme = start; \ tok.len = op_len; \ tok.fname = fname; \ tok.offset = offset; \ tok.line = line; \ tok.col = col; \ vec_push(toks, tok); \ offset += op_len; \ col += op_len; \ continue; \ } #define TRY_OP3(c1, c2, c3, tok3, tok2, tok1) \ if (c == (c1) && offset + 1 < len && source[offset + 1] == (c2)) { \ spl_tok_type_t op_type = (tok2); \ usize op_len = 2; \ if (offset + 2 < len && source[offset + 2] == (c3)) { \ op_type = (tok3); \ op_len = 3; \ } \ spl_tok_t tok; \ memset(&tok, 0, sizeof(tok)); \ tok.type = op_type; \ tok.lexeme = start; \ tok.len = op_len; \ tok.fname = fname; \ tok.offset = offset; \ tok.line = line; \ tok.col = col; \ vec_push(toks, tok); \ offset += op_len; \ col += op_len; \ continue; \ } /* Three-char operators first */ TRY_OP3('<', '<', '=', TOK_ASSIGN_L_SH, TOK_L_SH, TOK_LT) TRY_OP3('>', '>', '=', TOK_ASSIGN_R_SH, TOK_R_SH, TOK_GT) TRY_OP3('.', '.', '.', TOK_ELLIPSIS, TOK_RANGE, TOK_DOT) /* Two-char operators */ TRY_OP2('=', '=', TOK_EQ, TOK_ASSIGN) TRY_OP2('!', '=', TOK_NEQ, TOK_NOT) TRY_OP2('<', '=', TOK_LE, TOK_LT) TRY_OP2('>', '=', TOK_GE, TOK_GT) TRY_OP2('&', '&', TOK_AND_AND, TOK_AND) TRY_OP2('|', '|', TOK_OR_OR, TOK_OR) TRY_OP2('+', '=', TOK_ASSIGN_ADD, TOK_ADD) TRY_OP2('-', '=', TOK_ASSIGN_SUB, TOK_SUB) TRY_OP2('*', '=', TOK_ASSIGN_MUL, TOK_MUL) TRY_OP2('/', '=', TOK_ASSIGN_DIV, TOK_DIV) TRY_OP2('%', '=', TOK_ASSIGN_MOD, TOK_MOD) TRY_OP2('&', '=', TOK_ASSIGN_AND, TOK_AND) TRY_OP2('|', '=', TOK_ASSIGN_OR, TOK_OR) TRY_OP2('^', '=', TOK_ASSIGN_XOR, TOK_XOR) TRY_OP2('<', '-', TOK_LEFT_ARRAY, TOK_LT) TRY_OP2('-', '>', TOK_RIGHT_ARRAY, TOK_SUB) TRY_OP2(':', '=', TOK_COLON_ASSIGN, TOK_COLON) /* Single-character operators */ { spl_tok_type_t tt = TOK_UNKNOWN; switch (c) { case '+': tt = TOK_ADD; break; case '-': tt = TOK_SUB; break; case '*': tt = TOK_MUL; break; case '/': tt = TOK_DIV; break; case '%': tt = TOK_MOD; break; case '&': tt = TOK_AND; break; case '|': tt = TOK_OR; break; case '^': tt = TOK_XOR; break; case '~': tt = TOK_BIT_NOT; break; case '!': tt = TOK_NOT; break; case '<': tt = TOK_LT; break; case '>': tt = TOK_GT; break; case '=': tt = TOK_ASSIGN; break; case '.': tt = TOK_DOT; break; case ',': tt = TOK_COMMA; break; case ';': tt = TOK_SEMICOLON; break; case ':': tt = TOK_COLON; break; case '(': tt = TOK_L_PAREN; break; case ')': tt = TOK_R_PAREN; break; case '[': tt = TOK_L_BRACKET; break; case ']': tt = TOK_R_BRACKET; break; case '{': tt = TOK_L_BRACE; break; case '}': tt = TOK_R_BRACE; break; case '#': tt = TOK_SHARP; break; case '@': tt = TOK_AT; break; case '?': tt = TOK_COND; break; default: tt = TOK_UNKNOWN; break; } spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = tt; tok.lexeme = start; tok.len = 1; tok.fname = fname; tok.offset = offset; tok.line = line; tok.col = col; vec_push(toks, tok); offset++; col++; } } /* EOF token */ { spl_tok_t tok; memset(&tok, 0, sizeof(tok)); tok.type = TOK_EOF; tok.lexeme = source + offset; tok.len = 0; tok.fname = fname; tok.offset = offset; tok.line = line; tok.col = col; vec_push(toks, tok); } return toks; }