// Expect retcode = 0 // UTF-8 工具:解码/编码、校验、码点数、以及 UTF-8 字节偏移 <-> UTF-16 code unit 偏移 // (LSP 位置基于 UTF-16)。严格校验过长编码与代理区。 type .{ .Error, .Status, .make } = @import("error.spl"); type .{ .ByteBuffer } = @import("byte_buffer.spl"); // 解码结果:Ok 携带消耗字节数;明确区分越界/截断/非法,避免 0 哨兵歧义。 type DecodeResult = enum { Ok: usize, Eof, // index 已越过输入 Truncated, // 序列尾部字节不足 Invalid, // 续字节/过长编码/代理区/超出码点范围 } // 从 index 解码一个码点到 out_cp;返回 DecodeResult。 fn decode(data: *u8, len: usize, index: usize, out_cp: *u32): DecodeResult { if index >= len { ret .Eof; } var b0 = data[index]; if b0 < 0x80 { out_cp.* = b0 as u32; ret .Ok(1); } if b0 < 0xC2 { ret .Invalid; } // 续字节或过长编码的前导 if b0 < 0xE0 { if index + 1 >= len { ret .Truncated; } var b1 = data[index + 1]; if (b1 & 0xC0) != 0x80 { ret .Invalid; } out_cp.* = (((b0 & 0x1F) as u32) << 6) | ((b1 & 0x3F) as u32); ret .Ok(2); } if b0 < 0xF0 { if index + 2 >= len { ret .Truncated; } var b1 = data[index + 1]; var b2 = data[index + 2]; if (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 { ret .Invalid; } var cp = (((b0 & 0x0F) as u32) << 12) | (((b1 & 0x3F) as u32) << 6) | ((b2 & 0x3F) as u32); if cp < 0x800 { ret .Invalid; } // 过长 if cp >= 0xD800 && cp <= 0xDFFF { ret .Invalid; } // 代理区 out_cp.* = cp; ret .Ok(3); } if b0 < 0xF5 { if index + 3 >= len { ret .Truncated; } var b1 = data[index + 1]; var b2 = data[index + 2]; var b3 = data[index + 3]; if (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 || (b3 & 0xC0) != 0x80 { ret .Invalid; } var cp = (((b0 & 0x07) as u32) << 18) | (((b1 & 0x3F) as u32) << 12) | (((b2 & 0x3F) as u32) << 6) | ((b3 & 0x3F) as u32); if cp < 0x10000 || cp > 0x10FFFF { ret .Invalid; } out_cp.* = cp; ret .Ok(4); } ret .Invalid; } // 测试辅助:把 DecodeResult 归约为“消耗字节数”,非 Ok 视为 0。 fn decode_used(data: *u8, len: usize, index: usize, out_cp: *u32): usize { match decode(data, len, index, out_cp) { .Ok(used) => { ret used; }, .Eof => { ret 0; }, .Truncated => { ret 0; }, .Invalid => { ret 0; }, } ret 0; } fn valid(data: *u8, len: usize): bool { var i: usize = 0; var cp: u32 = 0; while i < len { match decode(data, len, i, &cp) { .Ok(used) => { i = i + used; }, _ => { ret false; }, } } ret true; } fn encode_into(out: *ByteBuffer, cp: u32): Status { if cp < 0x80 { ret ByteBuffer.append_byte(out, cp as u8); } if cp < 0x800 { var b0 = (0xC0 | ((cp >> 6) as u8)); var b1 = (0x80 | ((cp & 0x3F) as u8)); match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } } ret ByteBuffer.append_byte(out, b1); } if cp >= 0xD800 && cp <= 0xDFFF { ret .Err(make(.InvalidArgument)); } if cp < 0x10000 { var b0 = (0xE0 | ((cp >> 12) as u8)); var b1 = (0x80 | (((cp >> 6) & 0x3F) as u8)); var b2 = (0x80 | ((cp & 0x3F) as u8)); match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } } match ByteBuffer.append_byte(out, b1) { .Ok => {}, .Err(e) => { ret .Err(e); } } ret ByteBuffer.append_byte(out, b2); } if cp <= 0x10FFFF { var b0 = (0xF0 | ((cp >> 18) as u8)); var b1 = (0x80 | (((cp >> 12) & 0x3F) as u8)); var b2 = (0x80 | (((cp >> 6) & 0x3F) as u8)); var b3 = (0x80 | ((cp & 0x3F) as u8)); match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } } match ByteBuffer.append_byte(out, b1) { .Ok => {}, .Err(e) => { ret .Err(e); } } match ByteBuffer.append_byte(out, b2) { .Ok => {}, .Err(e) => { ret .Err(e); } } ret ByteBuffer.append_byte(out, b3); } ret .Err(make(.InvalidArgument)); } fn count_codepoints(data: *u8, len: usize): usize { var i: usize = 0; var n: usize = 0; var cp: u32 = 0; while i < len { match decode(data, len, i, &cp) { .Ok(used) => { i = i + used; n = n + 1; }, _ => { ret n; }, } } ret n; } // 码点占用的 UTF-16 code unit 数:BMP 为 1,补充平面为 2。 fn utf16_units(data: *u8, len: usize): usize { var i: usize = 0; var n: usize = 0; var cp: u32 = 0; while i < len { match decode(data, len, i, &cp) { .Ok(used) => { if cp >= 0x10000 { n = n + 2; } else { n = n + 1; } i = i + used; }, _ => { ret n; }, } } ret n; } // 字节偏移 -> UTF-16 偏移(夹取到 [0, utf16_units])。 fn byte_to_utf16(data: *u8, len: usize, byte_off: usize): usize { var i: usize = 0; var n: usize = 0; var cp: u32 = 0; while i < len && i < byte_off { match decode(data, len, i, &cp) { .Ok(used) => { if cp >= 0x10000 { n = n + 2; } else { n = n + 1; } i = i + used; }, _ => { ret n; }, } } ret n; } // UTF-16 偏移 -> 字节偏移(夹取;落在码点内部时返回该码点起始字节)。 fn utf16_to_byte(data: *u8, len: usize, utf16_off: usize): usize { var i: usize = 0; var n: usize = 0; var cp: u32 = 0; while i < len { if n >= utf16_off { ret i; } var used: usize = 0; match decode(data, len, i, &cp) { .Ok(value) => { used = value; }, _ => { ret i; }, } var units: usize = 1; if cp >= 0x10000 { units = 2; } if n + units > utf16_off { ret i; } n = n + units; i = i + used; } ret i; } fn expect_ok(result: Status) { match result { .Ok => {}, .Err(error) => { @assert(false, "unexpected utf8 error"); }, } } fn test_decode_encode_roundtrip() { var out = ByteBuffer.new(); defer { ByteBuffer.drop(&out); } expect_ok(encode_into(&out, 0x41)); // 'A' expect_ok(encode_into(&out, 0xE9)); // é expect_ok(encode_into(&out, 0x20AC)); // € expect_ok(encode_into(&out, 0x1F600)); // 😀 var bytes = ByteBuffer.as_slice(out); @assert(bytes.len == 1 + 2 + 3 + 4, "encoded length"); @assert(bytes.ptr[0] == 0x41, "A"); @assert(bytes.ptr[1] == 0xC3 && bytes.ptr[2] == 0xA9, "e-acute"); @assert(bytes.ptr[3] == 0xE2 && bytes.ptr[4] == 0x82 && bytes.ptr[5] == 0xAC, "euro"); @assert(bytes.ptr[6] == 0xF0 && bytes.ptr[9] == 0x80, "emoji head/tail"); @assert(valid(bytes.ptr, bytes.len), "valid"); @assert(count_codepoints(bytes.ptr, bytes.len) == 4, "codepoints"); @assert(utf16_units(bytes.ptr, bytes.len) == 1 + 1 + 1 + 2, "utf16 units"); var cp: u32 = 0; @assert(decode_used(bytes.ptr, bytes.len, 0, &cp) == 1 && cp == 0x41, "dec A"); @assert(decode_used(bytes.ptr, bytes.len, 1, &cp) == 2 && cp == 0xE9, "dec e"); @assert(decode_used(bytes.ptr, bytes.len, 3, &cp) == 3 && cp == 0x20AC, "dec euro"); @assert(decode_used(bytes.ptr, bytes.len, 6, &cp) == 4 && cp == 0x1F600, "dec emoji"); } fn test_invalid() { var bad1: [2]u8 = .[0xC0, 0x80]; // 过长 var bad2: [3]u8 = .[0xED, 0xA0, 0x80]; // 代理区 var bad3: [2]u8 = .[0xC3, 0x28]; // 坏续字节 var bad4: [1]u8 = .[0x80]; // 孤立续字节 var bad5: [3]u8 = .[0xE2, 0x82]; // 截断 var s1 = bad1[0..2]; var s2 = bad2[0..3]; var s3 = bad3[0..2]; var s4 = bad4[0..1]; var s5 = bad5[0..2]; @assert(!valid(s1.ptr, s1.len), "overlong"); @assert(!valid(s2.ptr, s2.len), "surrogate"); @assert(!valid(s3.ptr, s3.len), "bad cont"); @assert(!valid(s4.ptr, s4.len), "lone cont"); @assert(!valid(s5.ptr, s5.len), "truncated"); // 新 ADT:越界/截断/非法可区分。 var cp: u32 = 0; match decode(s5.ptr, s5.len, 0, &cp) { .Truncated => {}, _ => { @assert(false, "truncated sequence should be Truncated"); }, } match decode(s5.ptr, s5.len, s5.len, &cp) { .Eof => {}, _ => { @assert(false, "past end should be Eof"); }, } match decode(s1.ptr, s1.len, 0, &cp) { .Invalid => {}, _ => { @assert(false, "overlong should be Invalid"); }, } match decode(s4.ptr, s4.len, 0, &cp) { .Invalid => {}, _ => { @assert(false, "lone continuation should be Invalid"); }, } var out = ByteBuffer.new(); defer { ByteBuffer.drop(&out); } match encode_into(&out, 0xD800) { .Ok => { @assert(false, "surrogate encode"); }, .Err(error) => {}, } match encode_into(&out, 0x110000) { .Ok => { @assert(false, "out of range encode"); }, .Err(error) => {}, } } fn test_offsets() { var out = ByteBuffer.new(); defer { ByteBuffer.drop(&out); } expect_ok(encode_into(&out, 0x41)); // byte 0, utf16 0 expect_ok(encode_into(&out, 0x1F600)); // bytes 1..4, utf16 1..2 expect_ok(encode_into(&out, 0x42)); // byte 5, utf16 3 var b = ByteBuffer.as_slice(out); @assert(byte_to_utf16(b.ptr, b.len, 0) == 0, "b0"); @assert(byte_to_utf16(b.ptr, b.len, 1) == 1, "b1"); @assert(byte_to_utf16(b.ptr, b.len, 5) == 3, "b5"); @assert(byte_to_utf16(b.ptr, b.len, 6) == 4, "clamp"); @assert(utf16_to_byte(b.ptr, b.len, 0) == 0, "u0"); @assert(utf16_to_byte(b.ptr, b.len, 1) == 1, "u1"); @assert(utf16_to_byte(b.ptr, b.len, 2) == 1, "u2 stays in emoji"); @assert(utf16_to_byte(b.ptr, b.len, 3) == 5, "u3"); @assert(utf16_to_byte(b.ptr, b.len, 99) == 6, "clamp"); } fn main() { test_decode_encode_roundtrip(); test_invalid(); test_offsets(); }