scc/libs/lexer/src/lexer_stream.c

#include <lexer.h>

static void lexer_stream_extend(scc_lexer_stream_t *stream, usize n) {
    Assert(stream != null);
    // 检查是否需要扩容
    if ((stream->probe_pos - stream->curr_pos + n) >= stream->toks.cap) {
        // 需要扩容 - 创建新缓冲区
        usize new_cap = stream->toks.cap * 2;
        if (new_cap < stream->probe_pos - stream->curr_pos + n + 1) {
            new_cap = stream->probe_pos - stream->curr_pos + n + 1;
        }

        scc_lexer_tok_t *new_data =
            scc_realloc(null, new_cap * sizeof(scc_lexer_tok_t));
        if (!new_data) {
            LOG_FATAL("lexer_stream_extend: realloc failed\n");
            return;
        }

        // 将旧缓冲区中的数据拷贝到新缓冲区，保持顺序
        usize data_count = stream->probe_pos - stream->curr_pos;
        for (usize i = 0; i < data_count; ++i) {
            usize old_idx = (stream->curr_pos + i) % stream->toks.cap;
            new_data[i] = stream->toks.data[old_idx];
        }

        // 释放旧缓冲区
        if (stream->toks.data) {
            scc_free(stream->toks.data);
        }

        // 更新结构体
        stream->toks.data = new_data;
        stream->toks.cap = new_cap;
        stream->curr_pos = 0;
        stream->probe_pos = data_count;
    }

    // 填充新token
    for (usize i = 0; i < n; ++i) {
        usize idx = (stream->probe_pos + i) % stream->toks.cap;
        if (stream->need_comment)
            scc_lexer_get_token(stream->lexer, &stream->toks.data[idx]);
        else
            scc_lexer_get_valid_token(stream->lexer, &stream->toks.data[idx]);
    }

    stream->probe_pos += n;
}

static const scc_lexer_tok_t *lexer_stream_peek(scc_lexer_stream_t *stream,
                                                usize n) {
    Assert(stream != null);

    // 计算需要的前看token数量
    usize available = stream->probe_pos - stream->curr_pos;
    if (n >= available) {
        // 需要扩展缓冲区
        usize need = n - available + 1;
        lexer_stream_extend(stream, need);
    }

    // 计算实际缓冲区中的位置
    usize idx = (stream->curr_pos + n) % stream->toks.cap;
    return &stream->toks.data[idx];
}

static void lexer_stream_advance(scc_lexer_stream_t *stream, usize offset) {
    Assert(stream != null);

    if (stream->curr_pos + offset > stream->probe_pos) {
        // 尝试填充更多token
        usize need = stream->curr_pos + offset - stream->probe_pos;
        lexer_stream_extend(stream, need);
    }

    stream->curr_pos += offset;

    // 可选：当已消费的token过多时，压缩缓冲区
    if (stream->curr_pos > stream->toks.cap * 3 / 4) {
        // 压缩缓冲区：将有效数据移动到前面
        usize data_count = stream->probe_pos - stream->curr_pos;
        scc_lexer_tok_t *temp =
            scc_realloc(null, data_count * sizeof(scc_lexer_tok_t));
        if (!temp)
            return; // 压缩失败也没关系

        for (usize i = 0; i < data_count; ++i) {
            usize old_idx = (stream->curr_pos + i) % stream->toks.cap;
            temp[i] = stream->toks.data[old_idx];
        }

        scc_free(stream->toks.data);
        stream->toks.data = temp;
        stream->toks.cap = data_count;
        stream->curr_pos = 0;
        stream->probe_pos = data_count;
    }
}

static void lexer_stream_drop(scc_lexer_stream_t *stream) {
    Assert(stream != null);

    // 清理所有token（如果有需要清理的内部资源）
    for (usize i = 0; i < stream->toks.cap; ++i) {
        // 这里假设scc_lexer_tok_t可能包含需要释放的资源
        // 如果有，需要调用相应的清理函数
        // 例如: if (stream->toks.data[i].needs_free)
        // scc_free(stream->toks.data[i].ptr);
    }

    scc_vec_free(stream->toks);
    stream->lexer = null;
    stream->curr_pos = 0;
    stream->probe_pos = 0;
    stream->need_comment = false;

    stream->peek = null;
    stream->advance = null;
    stream->drop = null;
}

void scc_lexer_to_stream(scc_lexer_t *lexer, scc_lexer_stream_t *stream,
                         cbool need_comment) {
    Assert(lexer != null && stream != null);

    stream->lexer = lexer;
    stream->curr_pos = 0;
    stream->probe_pos = 0;
    stream->need_comment = need_comment;

    // 初始化循环缓冲区
    scc_vec_init(stream->toks);
    scc_vec_realloc(stream->toks, 8); // 初始容量为8

    stream->peek = lexer_stream_peek;
    stream->advance = lexer_stream_advance;
    stream->drop = lexer_stream_drop;
}