refactor(lexer): 重构词法分析器头文件结构并优化缓冲区管理
移除了旧的lexer_stream.c实现,引入新的环形缓冲区机制来替代原有的 动态数组缓冲区。更新了词法分析器的核心数据结构,修改了token获取 相关函数的实现以支持新的缓冲区管理方式。 BREAKING CHANGE: 移除了scc_lexer_stream_t相关的API,替换为基于 环形缓冲区的新接口scc_lexer_to_ring和相关函数。 feat(lexer_token): 添加词法分析结果内存泄漏警告注释 docs: 移除预处理器模块的测试文件和相关配置
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
#include <lexer.h>
|
||||
#include "scc_lexer.h"
|
||||
#include <lexer_log.h>
|
||||
#include <scc_lexer.h>
|
||||
|
||||
static const struct {
|
||||
const char *name;
|
||||
@@ -41,7 +42,8 @@ static int keyword_cmp(const char *name, int len) {
|
||||
}
|
||||
|
||||
void scc_lexer_init(scc_lexer_t *lexer, scc_sstream_ring_t *stream_ref) {
|
||||
lexer->stream_ref = *stream_ref;
|
||||
lexer->stream_ref = stream_ref;
|
||||
lexer->ring_ref_count = 0;
|
||||
lexer->jump_macro = false;
|
||||
}
|
||||
|
||||
@@ -68,7 +70,7 @@ static inline cbool is_hex_digit(int ch) {
|
||||
/* 从环形缓冲区预览一个字符(带EOF检测) */
|
||||
static inline cbool peek_char(scc_lexer_t *lexer, scc_sstream_char_t *out) {
|
||||
cbool ok;
|
||||
scc_ring_peek(lexer->stream_ref, *out, ok);
|
||||
scc_ring_peek(*lexer->stream_ref, *out, ok);
|
||||
return ok;
|
||||
}
|
||||
|
||||
@@ -76,7 +78,7 @@ static inline cbool peek_char(scc_lexer_t *lexer, scc_sstream_char_t *out) {
|
||||
static inline cbool next_char(scc_lexer_t *lexer, scc_cstring_t *lexeme,
|
||||
scc_sstream_char_t *out) {
|
||||
cbool ok;
|
||||
scc_ring_next(lexer->stream_ref, *out, ok);
|
||||
scc_ring_next(*lexer->stream_ref, *out, ok);
|
||||
if (!ok)
|
||||
return false;
|
||||
scc_cstring_append_ch(lexeme, out->character);
|
||||
@@ -132,7 +134,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
|
||||
next_char(lexer, &lex, &cur); // 消费 '/'
|
||||
while (peek_char(lexer, &cur) && !is_newline(cur.character)) {
|
||||
next_char(lexer, &lex, &cur);
|
||||
scc_ring_consume(lexer->stream_ref);
|
||||
scc_ring_consume(*lexer->stream_ref);
|
||||
}
|
||||
// 注释结束,不包含换行符(换行符单独成token)
|
||||
} else if (next.character == '*') {
|
||||
@@ -150,7 +152,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
|
||||
next_char(lexer, &lex, &cur); // 消费 '/'
|
||||
break;
|
||||
}
|
||||
scc_ring_consume(lexer->stream_ref);
|
||||
scc_ring_consume(*lexer->stream_ref);
|
||||
}
|
||||
} else {
|
||||
// 只是除号 /
|
||||
@@ -161,7 +163,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
|
||||
token->type = SCC_TOK_IDENT; // 暂定
|
||||
while (peek_char(lexer, &cur) && is_identifier_part(cur.character)) {
|
||||
next_char(lexer, &lex, &cur);
|
||||
scc_ring_consume(lexer->stream_ref);
|
||||
scc_ring_consume(*lexer->stream_ref);
|
||||
}
|
||||
// 检查是否为关键字
|
||||
int idx = keyword_cmp(scc_cstring_as_cstr(&lex), scc_cstring_len(&lex));
|
||||
@@ -241,7 +243,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
|
||||
} else {
|
||||
next_char(lexer, &lex, &cur);
|
||||
}
|
||||
scc_ring_consume(lexer->stream_ref);
|
||||
scc_ring_consume(*lexer->stream_ref);
|
||||
}
|
||||
} else {
|
||||
scc_sstream_char_t next = {0};
|
||||
@@ -447,7 +449,7 @@ void scc_lexer_get_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
|
||||
}
|
||||
|
||||
// 设置token
|
||||
scc_ring_consume(lexer->stream_ref);
|
||||
scc_ring_consume(*lexer->stream_ref);
|
||||
token->type = token->type; // 上面已设
|
||||
token->loc = start_loc;
|
||||
token->lexeme = lex; // 转移所有权
|
||||
@@ -469,3 +471,42 @@ void scc_lexer_get_valid_token(scc_lexer_t *lexer, scc_lexer_tok_t *token) {
|
||||
} while (subtype == SCC_TOK_SUBTYPE_EMPTYSPACE ||
|
||||
subtype == SCC_TOK_SUBTYPE_COMMENT);
|
||||
}
|
||||
|
||||
static int fill_token(scc_lexer_tok_t *out, void *userdata) {
|
||||
scc_lexer_t *lexer = userdata;
|
||||
scc_lexer_get_token(lexer, out);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int fill_valid_token(scc_lexer_tok_t *out, void *userdata) {
|
||||
scc_lexer_t *lexer = userdata;
|
||||
scc_lexer_get_valid_token(lexer, out);
|
||||
return 0;
|
||||
}
|
||||
|
||||
scc_lexer_tok_ring_t *scc_lexer_to_ring(scc_lexer_t *lexer, int ring_size,
|
||||
cbool need_comment) {
|
||||
scc_ring_init(lexer->ring, ring_size,
|
||||
need_comment ? fill_token : fill_valid_token, lexer);
|
||||
lexer->ring_ref_count++;
|
||||
return &lexer->ring;
|
||||
}
|
||||
|
||||
void scc_lexer_drop_ring(scc_lexer_tok_ring_t *ring_ref) {
|
||||
scc_lexer_t *lexer = ring_ref->userdata;
|
||||
if (lexer->ring_ref_count > 0) {
|
||||
lexer->ring_ref_count--;
|
||||
} else {
|
||||
LOG_WARN("double drop sstream ring");
|
||||
}
|
||||
}
|
||||
|
||||
void scc_lexer_drop(scc_lexer_t *lexer) {
|
||||
Assert(lexer != null);
|
||||
if (lexer->ring_ref_count) {
|
||||
LOG_FATAL("drop sstream must be drop ring before ref [%d]",
|
||||
lexer->ring_ref_count);
|
||||
}
|
||||
scc_ring_free(lexer->ring);
|
||||
scc_sstream_drop_ring(lexer->stream_ref);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user