refactor(lexer): 重构词法分析器头文件结构并优化缓冲区管理

移除了旧的lexer_stream.c实现,引入新的环形缓冲区机制来替代原有的
动态数组缓冲区。更新了词法分析器的核心数据结构,修改了token获取
相关函数的实现以支持新的缓冲区管理方式。

BREAKING CHANGE: 移除了scc_lexer_stream_t相关的API,替换为基于
环形缓冲区的新接口scc_lexer_to_ring和相关函数。

feat(lexer_token): 添加词法分析结果内存泄漏警告注释

docs: 移除预处理器模块的测试文件和相关配置
This commit is contained in:
zzy
2026-02-16 21:21:23 +08:00
parent 0e7dec202a
commit b4929be6b8
72 changed files with 119 additions and 2474 deletions

View File

@@ -1,5 +1,6 @@
#include <lexer.h>
#include "scc_lexer.h"
#include <lexer_log.h>
#include <scc_lexer.h>
static const struct {
const char *name;
@@ -41,7 +42,8 @@ static int keyword_cmp(const char *name, int len) {
}
void scc_lexer_init(scc_lexer_t *lexer, scc_sstream_ring_t *stream_ref) {
lexer->stream_ref = *stream_ref;
lexer->stream_ref = stream_ref;
lexer->ring_ref_count = 0;
lexer->jump_macro = false;
}
@@ -68,7 +70,7 @@ static inline cbool is_hex_digit(int ch) {
/* 从环形缓冲区预览一个字符带EOF检测 */
static inline cbool peek_char(scc_lexer_t *lexer, scc_sstream_char_t *out) {
cbool ok;
scc_ring_peek(lexer->stream_ref, *out, ok);
scc_ring_peek(*lexer->stream_ref, *out, ok);
return ok;
}
@@ -76,7 +78,7 @@ static inline cbool peek_char(scc_lexer_t *lexer, scc_sstream_char_t *out) {
static inline cbool next_char(scc_lexer_t *lexer, scc_cstring_t *lexeme,
scc_sstream_char_t *out) {
cbool ok;
scc_ring_next(lexer->stream_ref, *out, ok);
scc_ring_next(*lexer->stream_ref, *out, ok);
if (!ok)
return false;
scc_cstring_append_ch(lexeme, out->character);
@@ -132,7 +134,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
next_char(lexer, &lex, &cur); // 消费 '/'
while (peek_char(lexer, &cur) && !is_newline(cur.character)) {
next_char(lexer, &lex, &cur);
scc_ring_consume(lexer->stream_ref);
scc_ring_consume(*lexer->stream_ref);
}
// 注释结束不包含换行符换行符单独成token
} else if (next.character == '*') {
@@ -150,7 +152,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
next_char(lexer, &lex, &cur); // 消费 '/'
break;
}
scc_ring_consume(lexer->stream_ref);
scc_ring_consume(*lexer->stream_ref);
}
} else {
// 只是除号 /
@@ -161,7 +163,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
token->type = SCC_TOK_IDENT; // 暂定
while (peek_char(lexer, &cur) && is_identifier_part(cur.character)) {
next_char(lexer, &lex, &cur);
scc_ring_consume(lexer->stream_ref);
scc_ring_consume(*lexer->stream_ref);
}
// 检查是否为关键字
int idx = keyword_cmp(scc_cstring_as_cstr(&lex), scc_cstring_len(&lex));
@@ -241,7 +243,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
} else {
next_char(lexer, &lex, &cur);
}
scc_ring_consume(lexer->stream_ref);
scc_ring_consume(*lexer->stream_ref);
}
} else {
scc_sstream_char_t next = {0};
@@ -447,7 +449,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
}
// 设置token
scc_ring_consume(lexer->stream_ref);
scc_ring_consume(*lexer->stream_ref);
token->type = token->type; // 上面已设
token->loc = start_loc;
token->lexeme = lex; // 转移所有权
@@ -469,3 +471,42 @@ void scc_lexer_get_valid_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
} while (subtype == SCC_TOK_SUBTYPE_EMPTYSPACE ||
subtype == SCC_TOK_SUBTYPE_COMMENT);
}
static int fill_token(scc_lexer_tok_t *out, void *userdata) {
scc_lexer_t *lexer = userdata;
scc_lexer_get_token(lexer, out);
return 0;
}
static int fill_valid_token(scc_lexer_tok_t *out, void *userdata) {
scc_lexer_t *lexer = userdata;
scc_lexer_get_valid_token(lexer, out);
return 0;
}
scc_lexer_tok_ring_t *scc_lexer_to_ring(scc_lexer_t *lexer, int ring_size,
cbool need_comment) {
scc_ring_init(lexer->ring, ring_size,
need_comment ? fill_token : fill_valid_token, lexer);
lexer->ring_ref_count++;
return &lexer->ring;
}
void scc_lexer_drop_ring(scc_lexer_tok_ring_t *ring_ref) {
scc_lexer_t *lexer = ring_ref->userdata;
if (lexer->ring_ref_count > 0) {
lexer->ring_ref_count--;
} else {
LOG_WARN("double drop sstream ring");
}
}
void scc_lexer_drop(scc_lexer_t *lexer) {
Assert(lexer != null);
if (lexer->ring_ref_count) {
LOG_FATAL("drop sstream must be drop ring before ref [%d]",
lexer->ring_ref_count);
}
scc_ring_free(lexer->ring);
scc_sstream_drop_ring(lexer->stream_ref);
}