#include static void lexer_stream_extend(scc_lexer_stream_t *stream, usize n) { Assert(stream != null); // 检查是否需要扩容 if ((stream->probe_pos - stream->curr_pos + n) >= stream->toks.cap) { // 需要扩容 - 创建新缓冲区 usize new_cap = stream->toks.cap * 2; if (new_cap < stream->probe_pos - stream->curr_pos + n + 1) { new_cap = stream->probe_pos - stream->curr_pos + n + 1; } scc_lexer_tok_t *new_data = scc_realloc(null, new_cap * sizeof(scc_lexer_tok_t)); if (!new_data) { LOG_FATAL("lexer_stream_extend: realloc failed\n"); return; } // 将旧缓冲区中的数据拷贝到新缓冲区,保持顺序 usize data_count = stream->probe_pos - stream->curr_pos; for (usize i = 0; i < data_count; ++i) { usize old_idx = (stream->curr_pos + i) % stream->toks.cap; new_data[i] = stream->toks.data[old_idx]; } // 释放旧缓冲区 if (stream->toks.data) { scc_free(stream->toks.data); } // 更新结构体 stream->toks.data = new_data; stream->toks.cap = new_cap; stream->curr_pos = 0; stream->probe_pos = data_count; } // 填充新token for (usize i = 0; i < n; ++i) { usize idx = (stream->probe_pos + i) % stream->toks.cap; if (stream->need_comment) scc_lexer_get_token(stream->lexer, &stream->toks.data[idx]); else scc_lexer_get_valid_token(stream->lexer, &stream->toks.data[idx]); } stream->probe_pos += n; } static const scc_lexer_tok_t *lexer_stream_peek(scc_lexer_stream_t *stream, usize n) { Assert(stream != null); // 计算需要的前看token数量 usize available = stream->probe_pos - stream->curr_pos; if (n >= available) { // 需要扩展缓冲区 usize need = n - available + 1; lexer_stream_extend(stream, need); } // 计算实际缓冲区中的位置 usize idx = (stream->curr_pos + n) % stream->toks.cap; return &stream->toks.data[idx]; } static void lexer_stream_advance(scc_lexer_stream_t *stream, usize offset) { Assert(stream != null); if (stream->curr_pos + offset > stream->probe_pos) { // 尝试填充更多token usize need = stream->curr_pos + offset - stream->probe_pos; lexer_stream_extend(stream, need); } stream->curr_pos += offset; // 可选:当已消费的token过多时,压缩缓冲区 if (stream->curr_pos > stream->toks.cap * 3 / 4) { // 压缩缓冲区:将有效数据移动到前面 usize data_count = stream->probe_pos - stream->curr_pos; scc_lexer_tok_t *temp = scc_realloc(null, data_count * sizeof(scc_lexer_tok_t)); if (!temp) return; // 压缩失败也没关系 for (usize i = 0; i < data_count; ++i) { usize old_idx = (stream->curr_pos + i) % stream->toks.cap; temp[i] = stream->toks.data[old_idx]; } scc_free(stream->toks.data); stream->toks.data = temp; stream->toks.cap = data_count; stream->curr_pos = 0; stream->probe_pos = data_count; } } static void lexer_stream_drop(scc_lexer_stream_t *stream) { Assert(stream != null); // 清理所有token(如果有需要清理的内部资源) for (usize i = 0; i < stream->toks.cap; ++i) { // 这里假设scc_lexer_tok_t可能包含需要释放的资源 // 如果有,需要调用相应的清理函数 // 例如: if (stream->toks.data[i].needs_free) // scc_free(stream->toks.data[i].ptr); } scc_vec_free(stream->toks); stream->lexer = null; stream->curr_pos = 0; stream->probe_pos = 0; stream->need_comment = false; stream->peek = null; stream->advance = null; stream->drop = null; } void scc_lexer_to_stream(scc_lexer_t *lexer, scc_lexer_stream_t *stream, cbool need_comment) { Assert(lexer != null && stream != null); stream->lexer = lexer; stream->curr_pos = 0; stream->probe_pos = 0; stream->need_comment = need_comment; // 初始化循环缓冲区 scc_vec_init(stream->toks); scc_vec_realloc(stream->toks, 8); // 初始容量为8 stream->peek = lexer_stream_peek; stream->advance = lexer_stream_advance; stream->drop = lexer_stream_drop; }