534 lines
18 KiB
C
534 lines
18 KiB
C
/* spl_lexer.c — SPL lexical analyzer */
|
|
|
|
#include "spl_lexer.h"
|
|
#include <ctype.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
/* Character classification */
|
|
static int is_ident_start(char c) { return isalpha((unsigned char)c) || c == '_'; }
|
|
|
|
static int is_ident_cont(char c) { return isalnum((unsigned char)c) || c == '_'; }
|
|
|
|
/* Keyword lookup: if ident is a keyword, return its token type, else TOK_IDENT */
|
|
static spl_tok_type_t keyword_type(const char *ident, usize len) {
|
|
#define X(name, enum_name, dummy) \
|
|
if (len == sizeof(#name) - 1 && memcmp(ident, #name, len) == 0) \
|
|
return enum_name;
|
|
KEYWORD_TABLE
|
|
#undef X
|
|
return TOK_IDENT;
|
|
}
|
|
|
|
/* Escape sequence decoder — returns the decoded character,
|
|
* advances *s past the escape sequence, returns 0 on success,
|
|
* non-zero on error. */
|
|
int spl_decode_escape(const char **s, char *out) {
|
|
if (**s != '\\') {
|
|
*out = **s;
|
|
(*s)++;
|
|
return 0;
|
|
}
|
|
(*s)++; /* skip backslash */
|
|
switch (**s) {
|
|
case 'n':
|
|
*out = '\n';
|
|
break;
|
|
case 't':
|
|
*out = '\t';
|
|
break;
|
|
case 'r':
|
|
*out = '\r';
|
|
break;
|
|
case '\\':
|
|
*out = '\\';
|
|
break;
|
|
case '"':
|
|
*out = '"';
|
|
break;
|
|
case '\'':
|
|
*out = '\'';
|
|
break;
|
|
case '0':
|
|
*out = '\0';
|
|
break;
|
|
case 'x': {
|
|
(*s)++;
|
|
char hex[3] = {0, 0, 0};
|
|
int i;
|
|
for (i = 0; i < 2 && isxdigit((unsigned char)**s); i++, (*s)++) {
|
|
hex[i] = **s;
|
|
}
|
|
if (i == 0)
|
|
return -1;
|
|
*out = (char)strtol(hex, NULL, 16);
|
|
return 0;
|
|
}
|
|
default:
|
|
return -1;
|
|
}
|
|
(*s)++;
|
|
return 0;
|
|
}
|
|
|
|
spl_tok_vec_t spl_lex(const char *source, const char *fname) {
|
|
spl_tok_vec_t toks;
|
|
vec_init(toks);
|
|
|
|
usize line = 1;
|
|
usize col = 1;
|
|
usize offset = 0;
|
|
usize len = strlen(source);
|
|
|
|
while (offset < len) {
|
|
const char *start = source + offset;
|
|
char c = *start;
|
|
|
|
/* Skip whitespace (but not newlines — emit TOK_ENDLINE) */
|
|
if (c == ' ' || c == '\t' || c == '\r') {
|
|
offset++;
|
|
col++;
|
|
continue;
|
|
}
|
|
|
|
/* Newline */
|
|
if (c == '\n') {
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = TOK_ENDLINE;
|
|
tok.lexeme = start;
|
|
tok.len = 1;
|
|
tok.fname = fname;
|
|
tok.offset = offset;
|
|
tok.line = line;
|
|
tok.col = col;
|
|
vec_push(toks, tok);
|
|
offset++;
|
|
line++;
|
|
col = 1;
|
|
continue;
|
|
}
|
|
|
|
/* Line comment */
|
|
if (c == '/' && offset + 1 < len && source[offset + 1] == '/') {
|
|
const char *nl = (const char *)memchr(start, '\n', len - offset);
|
|
usize clen = nl ? (usize)(nl - start) : (len - offset);
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = TOK_LINE_COMMENT;
|
|
tok.lexeme = start;
|
|
tok.len = clen;
|
|
tok.fname = fname;
|
|
tok.offset = offset;
|
|
tok.line = line;
|
|
tok.col = col;
|
|
vec_push(toks, tok);
|
|
offset += clen;
|
|
col += clen;
|
|
continue;
|
|
}
|
|
|
|
/* Block comment */
|
|
if (c == '/' && offset + 1 < len && source[offset + 1] == '*') {
|
|
offset += 2;
|
|
col += 2;
|
|
usize depth = 1;
|
|
while (offset + 1 < len && depth > 0) {
|
|
if (source[offset] == '*' && source[offset + 1] == '/') {
|
|
depth--;
|
|
offset += 2;
|
|
col += 2;
|
|
if (depth == 0)
|
|
break;
|
|
} else if (source[offset] == '/' && source[offset + 1] == '*') {
|
|
depth++;
|
|
offset += 2;
|
|
col += 2;
|
|
} else {
|
|
if (source[offset] == '\n') {
|
|
line++;
|
|
col = 1;
|
|
} else {
|
|
col++;
|
|
}
|
|
offset++;
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
/* Char literal: 'x' */
|
|
if (c == '\'') {
|
|
offset++;
|
|
col++;
|
|
char buf[16];
|
|
int bi = 0;
|
|
memset(buf, 0, sizeof(buf));
|
|
|
|
if (offset < len) {
|
|
const char *cp = source + offset;
|
|
if (spl_decode_escape(&cp, &buf[bi])) {
|
|
buf[bi] = source[offset];
|
|
cp = source + offset + 1;
|
|
}
|
|
bi++;
|
|
offset = (usize)(cp - source);
|
|
col += (usize)(cp - (start + 1));
|
|
}
|
|
|
|
if (offset < len && source[offset] == '\'') {
|
|
offset++;
|
|
col++;
|
|
}
|
|
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = TOK_CHAR_LITERAL;
|
|
tok.lexeme = start;
|
|
tok.len = (usize)((source + offset) - start);
|
|
tok.fname = fname;
|
|
tok.offset = start - source;
|
|
tok.line = line;
|
|
tok.col = col - tok.len;
|
|
vec_push(toks, tok);
|
|
continue;
|
|
}
|
|
|
|
/* String literal: "..." */
|
|
if (c == '"') {
|
|
offset++;
|
|
col++;
|
|
while (offset < len) {
|
|
if (source[offset] == '\\') {
|
|
offset++;
|
|
col++;
|
|
if (offset < len) {
|
|
offset++;
|
|
col++;
|
|
}
|
|
} else if (source[offset] == '"') {
|
|
offset++;
|
|
col++;
|
|
break;
|
|
} else if (source[offset] == '\n') {
|
|
line++;
|
|
col = 1;
|
|
offset++;
|
|
} else {
|
|
offset++;
|
|
col++;
|
|
}
|
|
}
|
|
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = TOK_STRING_LITERAL;
|
|
tok.lexeme = start;
|
|
tok.len = (usize)((source + offset) - start);
|
|
tok.fname = fname;
|
|
tok.offset = start - source;
|
|
tok.line = line;
|
|
tok.col = col - tok.len;
|
|
vec_push(toks, tok);
|
|
continue;
|
|
}
|
|
|
|
/* Identifiers and keywords */
|
|
if (is_ident_start(c)) {
|
|
const char *id_start = start;
|
|
usize id_len = 0;
|
|
while (offset < len && is_ident_cont(source[offset])) {
|
|
offset++;
|
|
id_len++;
|
|
col++;
|
|
}
|
|
|
|
spl_tok_type_t tt = keyword_type(id_start, id_len);
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = tt;
|
|
tok.lexeme = id_start;
|
|
tok.len = id_len;
|
|
tok.fname = fname;
|
|
tok.offset = id_start - source;
|
|
tok.line = line;
|
|
tok.col = col - id_len;
|
|
vec_push(toks, tok);
|
|
continue;
|
|
}
|
|
|
|
/* Numbers: integers and floats */
|
|
if (isdigit((unsigned char)c)) {
|
|
const char *num_start = start;
|
|
int is_float = 0;
|
|
|
|
/* Check for hex/bin/oct prefix */
|
|
if (c == '0' && offset + 1 < len) {
|
|
char nc = source[offset + 1];
|
|
if (nc == 'x' || nc == 'X') {
|
|
/* Hex literal */
|
|
offset += 2;
|
|
col += 2;
|
|
while (offset < len && isxdigit((unsigned char)source[offset])) {
|
|
offset++;
|
|
col++;
|
|
}
|
|
goto emit_int;
|
|
}
|
|
if (nc == 'b' || nc == 'B') {
|
|
/* Binary literal */
|
|
offset += 2;
|
|
col += 2;
|
|
while (offset < len && (source[offset] == '0' || source[offset] == '1')) {
|
|
offset++;
|
|
col++;
|
|
}
|
|
goto emit_int;
|
|
}
|
|
if (nc == 'o' || nc == 'O') {
|
|
/* Octal literal */
|
|
offset += 2;
|
|
col += 2;
|
|
while (offset < len && source[offset] >= '0' && source[offset] <= '7') {
|
|
offset++;
|
|
col++;
|
|
}
|
|
goto emit_int;
|
|
}
|
|
}
|
|
|
|
/* Decimal integer or float */
|
|
while (offset < len && isdigit((unsigned char)source[offset])) {
|
|
offset++;
|
|
col++;
|
|
}
|
|
|
|
if (offset < len && source[offset] == '.' && offset + 1 < len &&
|
|
isdigit((unsigned char)source[offset + 1])) {
|
|
is_float = 1;
|
|
offset++;
|
|
col++;
|
|
while (offset < len && isdigit((unsigned char)source[offset])) {
|
|
offset++;
|
|
col++;
|
|
}
|
|
}
|
|
|
|
if (is_float) {
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = TOK_FLOAT_LITERAL;
|
|
tok.lexeme = num_start;
|
|
tok.len = (usize)((source + offset) - num_start);
|
|
tok.fname = fname;
|
|
tok.offset = num_start - source;
|
|
tok.line = line;
|
|
tok.col = col - tok.len;
|
|
vec_push(toks, tok);
|
|
continue;
|
|
}
|
|
|
|
emit_int: {
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = TOK_INT_LITERAL;
|
|
tok.lexeme = num_start;
|
|
tok.len = (usize)((source + offset) - num_start);
|
|
tok.fname = fname;
|
|
tok.offset = num_start - source;
|
|
tok.line = line;
|
|
tok.col = col - tok.len;
|
|
vec_push(toks, tok);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
/* Multi-character operators (longest match) */
|
|
|
|
/* Helper: try to match a two-char operator */
|
|
#define TRY_OP2(c1, c2, tok2, tok1) \
|
|
if (c == (c1) && offset + 1 < len && source[offset + 1] == (c2)) { \
|
|
spl_tok_type_t op_type = (tok2); \
|
|
usize op_len = 2; \
|
|
spl_tok_t tok; \
|
|
memset(&tok, 0, sizeof(tok)); \
|
|
tok.type = op_type; \
|
|
tok.lexeme = start; \
|
|
tok.len = op_len; \
|
|
tok.fname = fname; \
|
|
tok.offset = offset; \
|
|
tok.line = line; \
|
|
tok.col = col; \
|
|
vec_push(toks, tok); \
|
|
offset += op_len; \
|
|
col += op_len; \
|
|
continue; \
|
|
}
|
|
|
|
#define TRY_OP3(c1, c2, c3, tok3, tok2, tok1) \
|
|
if (c == (c1) && offset + 1 < len && source[offset + 1] == (c2)) { \
|
|
spl_tok_type_t op_type = (tok2); \
|
|
usize op_len = 2; \
|
|
if (offset + 2 < len && source[offset + 2] == (c3)) { \
|
|
op_type = (tok3); \
|
|
op_len = 3; \
|
|
} \
|
|
spl_tok_t tok; \
|
|
memset(&tok, 0, sizeof(tok)); \
|
|
tok.type = op_type; \
|
|
tok.lexeme = start; \
|
|
tok.len = op_len; \
|
|
tok.fname = fname; \
|
|
tok.offset = offset; \
|
|
tok.line = line; \
|
|
tok.col = col; \
|
|
vec_push(toks, tok); \
|
|
offset += op_len; \
|
|
col += op_len; \
|
|
continue; \
|
|
}
|
|
|
|
/* Three-char operators first */
|
|
TRY_OP3('<', '<', '=', TOK_ASSIGN_L_SH, TOK_L_SH, TOK_LT)
|
|
TRY_OP3('>', '>', '=', TOK_ASSIGN_R_SH, TOK_R_SH, TOK_GT)
|
|
TRY_OP3('.', '.', '.', TOK_ELLIPSIS, TOK_RANGE, TOK_DOT)
|
|
|
|
/* Two-char operators */
|
|
TRY_OP2('=', '=', TOK_EQ, TOK_ASSIGN)
|
|
TRY_OP2('!', '=', TOK_NEQ, TOK_NOT)
|
|
TRY_OP2('<', '=', TOK_LE, TOK_LT)
|
|
TRY_OP2('>', '=', TOK_GE, TOK_GT)
|
|
TRY_OP2('&', '&', TOK_AND_AND, TOK_AND)
|
|
TRY_OP2('|', '|', TOK_OR_OR, TOK_OR)
|
|
TRY_OP2('+', '=', TOK_ASSIGN_ADD, TOK_ADD)
|
|
TRY_OP2('-', '=', TOK_ASSIGN_SUB, TOK_SUB)
|
|
TRY_OP2('*', '=', TOK_ASSIGN_MUL, TOK_MUL)
|
|
TRY_OP2('/', '=', TOK_ASSIGN_DIV, TOK_DIV)
|
|
TRY_OP2('%', '=', TOK_ASSIGN_MOD, TOK_MOD)
|
|
TRY_OP2('&', '=', TOK_ASSIGN_AND, TOK_AND)
|
|
TRY_OP2('|', '=', TOK_ASSIGN_OR, TOK_OR)
|
|
TRY_OP2('^', '=', TOK_ASSIGN_XOR, TOK_XOR)
|
|
TRY_OP2('<', '-', TOK_LEFT_ARRAY, TOK_LT)
|
|
TRY_OP2('-', '>', TOK_RIGHT_ARRAY, TOK_SUB)
|
|
TRY_OP2(':', '=', TOK_COLON_ASSIGN, TOK_COLON)
|
|
|
|
/* Single-character operators */
|
|
{
|
|
spl_tok_type_t tt = TOK_UNKNOWN;
|
|
switch (c) {
|
|
case '+':
|
|
tt = TOK_ADD;
|
|
break;
|
|
case '-':
|
|
tt = TOK_SUB;
|
|
break;
|
|
case '*':
|
|
tt = TOK_MUL;
|
|
break;
|
|
case '/':
|
|
tt = TOK_DIV;
|
|
break;
|
|
case '%':
|
|
tt = TOK_MOD;
|
|
break;
|
|
case '&':
|
|
tt = TOK_AND;
|
|
break;
|
|
case '|':
|
|
tt = TOK_OR;
|
|
break;
|
|
case '^':
|
|
tt = TOK_XOR;
|
|
break;
|
|
case '~':
|
|
tt = TOK_BIT_NOT;
|
|
break;
|
|
case '!':
|
|
tt = TOK_NOT;
|
|
break;
|
|
case '<':
|
|
tt = TOK_LT;
|
|
break;
|
|
case '>':
|
|
tt = TOK_GT;
|
|
break;
|
|
case '=':
|
|
tt = TOK_ASSIGN;
|
|
break;
|
|
case '.':
|
|
tt = TOK_DOT;
|
|
break;
|
|
case ',':
|
|
tt = TOK_COMMA;
|
|
break;
|
|
case ';':
|
|
tt = TOK_SEMICOLON;
|
|
break;
|
|
case ':':
|
|
tt = TOK_COLON;
|
|
break;
|
|
case '(':
|
|
tt = TOK_L_PAREN;
|
|
break;
|
|
case ')':
|
|
tt = TOK_R_PAREN;
|
|
break;
|
|
case '[':
|
|
tt = TOK_L_BRACKET;
|
|
break;
|
|
case ']':
|
|
tt = TOK_R_BRACKET;
|
|
break;
|
|
case '{':
|
|
tt = TOK_L_BRACE;
|
|
break;
|
|
case '}':
|
|
tt = TOK_R_BRACE;
|
|
break;
|
|
case '#':
|
|
tt = TOK_SHARP;
|
|
break;
|
|
case '@':
|
|
tt = TOK_AT;
|
|
break;
|
|
case '?':
|
|
tt = TOK_COND;
|
|
break;
|
|
default:
|
|
tt = TOK_UNKNOWN;
|
|
break;
|
|
}
|
|
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = tt;
|
|
tok.lexeme = start;
|
|
tok.len = 1;
|
|
tok.fname = fname;
|
|
tok.offset = offset;
|
|
tok.line = line;
|
|
tok.col = col;
|
|
vec_push(toks, tok);
|
|
offset++;
|
|
col++;
|
|
}
|
|
}
|
|
|
|
/* EOF token */
|
|
{
|
|
spl_tok_t tok;
|
|
memset(&tok, 0, sizeof(tok));
|
|
tok.type = TOK_EOF;
|
|
tok.lexeme = source + offset;
|
|
tok.len = 0;
|
|
tok.fname = fname;
|
|
tok.offset = offset;
|
|
tok.line = line;
|
|
tok.col = col;
|
|
vec_push(toks, tok);
|
|
}
|
|
|
|
return toks;
|
|
}
|