From 16df10181dab7834370823efc7cce4a55db570c5 Mon Sep 17 00:00:00 2001 From: Michael Chaten Date: Sat, 13 Sep 2025 07:11:50 -0700 Subject: Update build to 0.15.0 --- build.zig | 176 ++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 90 insertions(+), 86 deletions(-) diff --git a/build.zig b/build.zig index ca0eeef..67a009a 100644 --- a/build.zig +++ b/build.zig @@ -42,27 +42,33 @@ pub fn build(b: *std.Build) void { // Grapheme break const gbp_gen_exe = b.addExecutable(.{ .name = "gbp", - .root_source_file = b.path("codegen/gbp.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/gbp.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_gbp_gen_exe = b.addRunArtifact(gbp_gen_exe); const gbp_gen_out = run_gbp_gen_exe.addOutputFileArg("gbp.bin.z"); const wbp_gen_exe = b.addExecutable(.{ .name = "wbp", - .root_source_file = b.path("codegen/wbp.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/wbp.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_wbp_gen_exe = b.addRunArtifact(wbp_gen_exe); const wbp_gen_out = run_wbp_gen_exe.addOutputFileArg("wbp.bin.z"); const dwp_gen_exe = b.addExecutable(.{ .name = "dwp", - .root_source_file = b.path("codegen/dwp.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/dwp.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); dwp_gen_exe.root_module.addOptions("options", dwp_options); const run_dwp_gen_exe = b.addRunArtifact(dwp_gen_exe); @@ -71,63 +77,77 @@ pub fn build(b: *std.Build) void { // Normalization properties const canon_gen_exe = b.addExecutable(.{ .name = "canon", - .root_source_file = b.path("codegen/canon.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/canon.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_canon_gen_exe = b.addRunArtifact(canon_gen_exe); const canon_gen_out = run_canon_gen_exe.addOutputFileArg("canon.bin.z"); const compat_gen_exe = b.addExecutable(.{ .name = "compat", - .root_source_file = b.path("codegen/compat.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/compat.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_compat_gen_exe = b.addRunArtifact(compat_gen_exe); const compat_gen_out = run_compat_gen_exe.addOutputFileArg("compat.bin.z"); const hangul_gen_exe = b.addExecutable(.{ .name = "hangul", - .root_source_file = b.path("codegen/hangul.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/hangul.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_hangul_gen_exe = b.addRunArtifact(hangul_gen_exe); const hangul_gen_out = run_hangul_gen_exe.addOutputFileArg("hangul.bin.z"); const normp_gen_exe = b.addExecutable(.{ .name = "normp", - .root_source_file = b.path("codegen/normp.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/normp.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_normp_gen_exe = b.addRunArtifact(normp_gen_exe); const normp_gen_out = run_normp_gen_exe.addOutputFileArg("normp.bin.z"); const ccc_gen_exe = b.addExecutable(.{ .name = "ccc", - .root_source_file = b.path("codegen/ccc.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/ccc.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_ccc_gen_exe = b.addRunArtifact(ccc_gen_exe); const ccc_gen_out = run_ccc_gen_exe.addOutputFileArg("ccc.bin.z"); const gencat_gen_exe = b.addExecutable(.{ .name = "gencat", - .root_source_file = b.path("codegen/gencat.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/gencat.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_gencat_gen_exe = b.addRunArtifact(gencat_gen_exe); const gencat_gen_out = run_gencat_gen_exe.addOutputFileArg("gencat.bin.z"); const fold_gen_exe = b.addExecutable(.{ .name = "fold", - .root_source_file = b.path("codegen/fold.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/fold.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_fold_gen_exe = b.addRunArtifact(fold_gen_exe); const fold_gen_out = run_fold_gen_exe.addOutputFileArg("fold.bin.z"); @@ -135,9 +155,11 @@ pub fn build(b: *std.Build) void { // Numeric types const num_gen_exe = b.addExecutable(.{ .name = "numeric", - .root_source_file = b.path("codegen/numeric.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/numeric.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_num_gen_exe = b.addRunArtifact(num_gen_exe); const num_gen_out = run_num_gen_exe.addOutputFileArg("numeric.bin.z"); @@ -145,9 +167,11 @@ pub fn build(b: *std.Build) void { // Letter case properties const case_prop_gen_exe = b.addExecutable(.{ .name = "case_prop", - .root_source_file = b.path("codegen/case_prop.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/case_prop.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_case_prop_gen_exe = b.addRunArtifact(case_prop_gen_exe); const case_prop_gen_out = run_case_prop_gen_exe.addOutputFileArg("case_prop.bin.z"); @@ -155,9 +179,11 @@ pub fn build(b: *std.Build) void { // Uppercase mappings const upper_gen_exe = b.addExecutable(.{ .name = "upper", - .root_source_file = b.path("codegen/upper.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/upper.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_upper_gen_exe = b.addRunArtifact(upper_gen_exe); const upper_gen_out = run_upper_gen_exe.addOutputFileArg("upper.bin.z"); @@ -165,36 +191,44 @@ pub fn build(b: *std.Build) void { // Lowercase mappings const lower_gen_exe = b.addExecutable(.{ .name = "lower", - .root_source_file = b.path("codegen/lower.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/lower.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_lower_gen_exe = b.addRunArtifact(lower_gen_exe); const lower_gen_out = run_lower_gen_exe.addOutputFileArg("lower.bin.z"); const scripts_gen_exe = b.addExecutable(.{ .name = "scripts", - .root_source_file = b.path("codegen/scripts.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/scripts.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_scripts_gen_exe = b.addRunArtifact(scripts_gen_exe); const scripts_gen_out = run_scripts_gen_exe.addOutputFileArg("scripts.bin.z"); const core_gen_exe = b.addExecutable(.{ .name = "core", - .root_source_file = b.path("codegen/core_props.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/core_props.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_core_gen_exe = b.addRunArtifact(core_gen_exe); const core_gen_out = run_core_gen_exe.addOutputFileArg("core_props.bin.z"); const props_gen_exe = b.addExecutable(.{ .name = "props", - .root_source_file = b.path("codegen/props.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/props.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_props_gen_exe = b.addRunArtifact(props_gen_exe); const props_gen_out = run_props_gen_exe.addOutputFileArg("props.bin.z"); @@ -212,8 +246,6 @@ pub fn build(b: *std.Build) void { const code_point_t = b.addTest(.{ .name = "code_point", .root_module = code_point, - .target = target, - .optimize = optimize, }); const code_point_tr = b.addRunArtifact(code_point_t); @@ -230,8 +262,6 @@ pub fn build(b: *std.Build) void { const grapheme_t = b.addTest(.{ .name = "Graphemes", .root_module = graphemes, - .target = target, - .optimize = optimize, }); const grapheme_tr = b.addRunArtifact(grapheme_t); @@ -247,8 +277,6 @@ pub fn build(b: *std.Build) void { const words_t = b.addTest(.{ .name = "WordBreak", .root_module = words, - .target = target, - .optimize = optimize, }); const words_tr = b.addRunArtifact(words_t); @@ -262,8 +290,6 @@ pub fn build(b: *std.Build) void { const ascii_t = b.addTest(.{ .name = "ascii", .root_module = ascii, - .target = target, - .optimize = optimize, }); const ascii_tr = b.addRunArtifact(ascii_t); @@ -282,8 +308,6 @@ pub fn build(b: *std.Build) void { const display_width_t = b.addTest(.{ .name = "display_width", .root_module = display_width, - .target = target, - .optimize = optimize, }); const display_width_tr = b.addRunArtifact(display_width_t); @@ -298,8 +322,6 @@ pub fn build(b: *std.Build) void { const ccc_data_t = b.addTest(.{ .name = "ccc_data", .root_module = ccc_data, - .target = target, - .optimize = optimize, }); const ccc_data_tr = b.addRunArtifact(ccc_data_t); @@ -314,8 +336,6 @@ pub fn build(b: *std.Build) void { const canon_data_t = b.addTest(.{ .name = "canon_data", .root_module = canon_data, - .target = target, - .optimize = optimize, }); const canon_data_tr = b.addRunArtifact(canon_data_t); @@ -330,8 +350,6 @@ pub fn build(b: *std.Build) void { const compat_data_t = b.addTest(.{ .name = "compat_data", .root_module = compat_data, - .target = target, - .optimize = optimize, }); const compat_data_tr = b.addRunArtifact(compat_data_t); @@ -345,8 +363,6 @@ pub fn build(b: *std.Build) void { const hangul_data_t = b.addTest(.{ .name = "hangul_data", .root_module = hangul_data, - .target = target, - .optimize = optimize, }); const hangul_data_tr = b.addRunArtifact(hangul_data_t); @@ -360,8 +376,6 @@ pub fn build(b: *std.Build) void { const normp_data_t = b.addTest(.{ .name = "normp_data", .root_module = normp_data, - .target = target, - .optimize = optimize, }); const normp_data_tr = b.addRunArtifact(normp_data_t); @@ -381,8 +395,6 @@ pub fn build(b: *std.Build) void { const norm_t = b.addTest(.{ .name = "norm", .root_module = norm, - .target = target, - .optimize = optimize, }); const norm_tr = b.addRunArtifact(norm_t); @@ -397,8 +409,6 @@ pub fn build(b: *std.Build) void { const gencat_t = b.addTest(.{ .name = "gencat", .root_module = gencat, - .target = target, - .optimize = optimize, }); const gencat_tr = b.addRunArtifact(gencat_t); @@ -415,8 +425,6 @@ pub fn build(b: *std.Build) void { const case_fold_t = b.addTest(.{ .name = "case_fold", .root_module = case_fold, - .target = target, - .optimize = optimize, }); const case_fold_tr = b.addRunArtifact(case_fold_t); @@ -434,8 +442,6 @@ pub fn build(b: *std.Build) void { const letter_case_t = b.addTest(.{ .name = "lettercase", .root_module = letter_case, - .target = target, - .optimize = optimize, }); const letter_case_tr = b.addRunArtifact(letter_case_t); @@ -450,8 +456,6 @@ pub fn build(b: *std.Build) void { const scripts_t = b.addTest(.{ .name = "scripts", .root_module = scripts, - .target = target, - .optimize = optimize, }); const scripts_tr = b.addRunArtifact(scripts_t); @@ -468,16 +472,16 @@ pub fn build(b: *std.Build) void { const properties_t = b.addTest(.{ .name = "properties", .root_module = properties, - .target = target, - .optimize = optimize, }); const properties_tr = b.addRunArtifact(properties_t); // Unicode Tests const unicode_tests = b.addTest(.{ - .root_source_file = b.path("src/unicode_tests.zig"), - .target = target, - .optimize = optimize, + .root_module = b.createModule(.{ + .root_source_file = b.path("src/unicode_tests.zig"), + .target = target, + .optimize = optimize, + }), }); unicode_tests.root_module.addImport("Graphemes", graphemes); unicode_tests.root_module.addImport("Normalize", norm); -- cgit v1.2.3 From a3b5e884b12fdaa341010ef41bb9382fa0cd89f8 Mon Sep 17 00:00:00 2001 From: Michael Chaten Date: Sat, 13 Sep 2025 08:38:24 -0700 Subject: Update codebase to Zig 0.15.1. Removes compression support --- build.zig.zon | 4 +-- codegen/canon.zig | 24 +++++++------ codegen/case_prop.zig | 31 +++++++++-------- codegen/ccc.zig | 31 +++++++++-------- codegen/compat.zig | 27 ++++++++------- codegen/core_props.zig | 32 +++++++++-------- codegen/dwp.zig | 42 ++++++++++++---------- codegen/fold.zig | 59 ++++++++++++++++--------------- codegen/gbp.zig | 57 +++++++++++++++++------------- codegen/gencat.zig | 35 ++++++++++--------- codegen/hangul.zig | 32 +++++++++-------- codegen/lower.zig | 26 +++++++------- codegen/normp.zig | 32 +++++++++-------- codegen/numeric.zig | 31 +++++++++-------- codegen/props.zig | 32 +++++++++-------- codegen/scripts.zig | 38 ++++++++++---------- codegen/upper.zig | 31 ++++++++++------- codegen/wbp.zig | 32 +++++++++-------- src/CanonData.zig | 4 +-- src/CaseFolding.zig | 6 ++-- src/CombiningData.zig | 5 +-- src/CompatData.zig | 5 +-- src/DisplayWidth.zig | 8 ++--- src/GeneralCategories.zig | 4 +-- src/Graphemes.zig | 4 +-- src/HangulData.zig | 5 +-- src/LetterCasing.zig | 14 +++----- src/NormPropsData.zig | 5 +-- src/Normalize.zig | 10 +++--- src/Properties.zig | 10 ++---- src/Scripts.zig | 5 +-- src/Words.zig | 4 +-- src/unicode_tests.zig | 88 +++++++++++++++++++++++++---------------------- 33 files changed, 399 insertions(+), 374 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 3e1df95..e585449 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ .name = .zg, - .version = "0.14.1", - .minimum_zig_version = "0.14.0", + .version = "0.15.1", + .minimum_zig_version = "0.15.1", .fingerprint = 0x47df7778dc946aa0, .paths = .{ diff --git a/codegen/canon.zig b/codegen/canon.zig index 28b7f28..89a9070 100644 --- a/codegen/canon.zig +++ b/codegen/canon.zig @@ -1,32 +1,30 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); - + var file_writer = out_file.writer(&write_buf); + var writer = &file_writer.interface; const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterInclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -60,8 +58,12 @@ pub fn main() !void { try writer.writeInt(u8, @intCast(len), endian); for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u16, 0, endian); - try out_comp.flush(); + try writer.flush(); } diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig index 6c912a8..66eb62c 100644 --- a/codegen/case_prop.zig +++ b/codegen/case_prop.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedCoreProperties.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/ccc.zig b/codegen/ccc.zig index a01c8d2..b9b2bc3 100644 --- a/codegen/ccc.zig +++ b/codegen/ccc.zig @@ -21,7 +21,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -34,10 +34,9 @@ pub fn main() !void { // Process DerivedCombiningClass.txt var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); defer cc_file.close(); - var cc_buf = std.io.bufferedReader(cc_file.reader()); - const cc_reader = cc_buf.reader(); + var cc_reader = cc_file.reader(&line_buf); - while (try cc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (cc_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -68,15 +67,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -107,18 +111,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/compat.zig b/codegen/compat.zig index 07616fc..835a636 100644 --- a/codegen/compat.zig +++ b/codegen/compat.zig @@ -1,7 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -9,24 +9,22 @@ pub fn main() !void { // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -55,10 +53,15 @@ pub fn main() !void { } } - try writer.writeInt(u8, @intCast(len), endian); - for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); + try writer.interface.writeInt(u8, @intCast(len), endian); + for (cps[0..len]) |cp| try writer.interface.writeInt(u24, cp, endian); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u16, 0, endian); - try out_comp.flush(); + try writer.interface.writeInt(u16, 0, endian); + try writer.interface.flush(); } diff --git a/codegen/core_props.zig b/codegen/core_props.zig index f60c7a9..19063f6 100644 --- a/codegen/core_props.zig +++ b/codegen/core_props.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedCoreProperties.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -82,15 +81,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -121,18 +124,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var out_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&out_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/dwp.zig b/codegen/dwp.zig index 5e5bf6a..fad08d1 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig @@ -23,7 +23,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -36,10 +36,9 @@ pub fn main() !void { // Process DerivedEastAsianWidth.txt var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); defer deaw_file.close(); - var deaw_buf = std.io.bufferedReader(deaw_file.reader()); - const deaw_reader = deaw_buf.reader(); + var deaw_reader = deaw_file.reader(&line_buf); - while (try deaw_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (deaw_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; // @missing ranges @@ -88,15 +87,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process DerivedGeneralCategory.txt var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); defer dgc_file.close(); - var dgc_buf = std.io.bufferedReader(dgc_file.reader()); - const dgc_reader = dgc_buf.reader(); + var dgc_reader = dgc_file.reader(&line_buf); - while (try dgc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (dgc_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -139,15 +141,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(i4).init(allocator); + var stage2 = std.array_list.Managed(i4).init(allocator); defer stage2.deinit(); var block: Block = [_]i4{0} ** block_size; @@ -227,18 +234,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(i8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(i8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/fold.zig b/codegen/fold.zig index cb73cca..cc381a8 100644 --- a/codegen/fold.zig +++ b/codegen/fold.zig @@ -2,23 +2,21 @@ const std = @import("std"); const builtin = @import("builtin"); const mem = std.mem; -pub fn main() !void { +pub fn main() anyerror!void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer std.debug.assert(gpa.deinit() == .ok); const allocator = gpa.allocator(); + var line_buf: [4096]u8 = undefined; // Process DerivedCoreProperties.txt var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer props_file.close(); - var props_buf = std.io.bufferedReader(props_file.reader()); - const props_reader = props_buf.reader(); + var props_reader = props_file.reader(&line_buf); var props_map = std.AutoHashMap(u21, void).init(allocator); defer props_map.deinit(); - var line_buf: [4096]u8 = undefined; - - props_lines: while (try props_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + props_lines: while (props_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -49,18 +47,21 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator); defer codepoint_mapping.deinit(); // Process CaseFolding.txt var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{}); defer cp_file.close(); - var cp_buf = std.io.bufferedReader(cp_file.reader()); - const cp_reader = cp_buf.reader(); + var cp_reader = cp_file.reader(&line_buf); - while (try cp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (cp_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; var field_it = std.mem.splitScalar(u8, line, ';'); @@ -81,9 +82,14 @@ pub fn main() !void { } try codepoint_mapping.putNoClobber(codepoint, mapping_buf); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var changes_when_casefolded_exceptions = std.ArrayList(u21).init(allocator); + var changes_when_casefolded_exceptions = std.array_list.Managed(u21).init(allocator); defer changes_when_casefolded_exceptions.deinit(); { @@ -221,32 +227,31 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); // Table metadata. - try writer.writeInt(u24, @intCast(codepoint_cutoff), endian); - try writer.writeInt(u24, @intCast(multiple_codepoint_start), endian); + try writer.interface.writeInt(u24, @intCast(codepoint_cutoff), endian); + try writer.interface.writeInt(u24, @intCast(multiple_codepoint_start), endian); // Stage 1 - try writer.writeInt(u16, @intCast(meaningful_stage1.len), endian); - try writer.writeAll(meaningful_stage1); + try writer.interface.writeInt(u16, @intCast(meaningful_stage1.len), endian); + try writer.interface.writeAll(meaningful_stage1); // Stage 2 - try writer.writeInt(u16, @intCast(stage2.len), endian); - try writer.writeAll(stage2); + try writer.interface.writeInt(u16, @intCast(stage2.len), endian); + try writer.interface.writeAll(stage2); // Stage 3 - try writer.writeInt(u16, @intCast(stage3.len), endian); - for (stage3) |offset| try writer.writeInt(i24, offset, endian); + try writer.interface.writeInt(u16, @intCast(stage3.len), endian); + for (stage3) |offset| try writer.interface.writeInt(i24, offset, endian); // Changes when case folded // Min and max - try writer.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); - try writer.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); - try writer.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); - for (changes_when_casefolded_exceptions.items) |cp| try writer.writeInt(u24, cp, endian); + try writer.interface.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); + try writer.interface.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); + try writer.interface.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); + for (changes_when_casefolded_exceptions.items) |cp| try writer.interface.writeInt(u24, cp, endian); - try out_comp.flush(); + try writer.interface.flush(); } } diff --git a/codegen/gbp.zig b/codegen/gbp.zig index 3fc4461..d654cf1 100644 --- a/codegen/gbp.zig +++ b/codegen/gbp.zig @@ -47,7 +47,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -66,10 +66,9 @@ pub fn main() !void { // Process Indic var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer indic_file.close(); - var indic_buf = std.io.bufferedReader(indic_file.reader()); - const indic_reader = indic_buf.reader(); + var indic_reader = indic_file.reader(&line_buf); - while (try indic_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (indic_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "InCB") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -100,15 +99,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process GBP var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{}); defer gbp_file.close(); - var gbp_buf = std.io.bufferedReader(gbp_file.reader()); - const gbp_reader = gbp_buf.reader(); + var gbp_reader = gbp_file.reader(&line_buf); - while (try gbp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (gbp_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -138,15 +140,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process Emoji var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{}); defer emoji_file.close(); - var emoji_buf = std.io.bufferedReader(emoji_file.reader()); - const emoji_reader = emoji_buf.reader(); + var emoji_reader = emoji_file.reader(&line_buf); - while (try emoji_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (emoji_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -170,15 +175,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u16).init(allocator); + var stage2 = std.array_list.Managed(u16).init(allocator); defer stage2.deinit(); var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator); @@ -227,22 +237,21 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u16, i, endian); const props_bytes = stage3.keys(); - try writer.writeInt(u16, @intCast(props_bytes.len), endian); - try writer.writeAll(props_bytes); + try writer.interface.writeInt(u16, @intCast(props_bytes.len), endian); + try writer.interface.writeAll(props_bytes); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/gencat.zig b/codegen/gencat.zig index fe06bd7..7dedb5d 100644 --- a/codegen/gencat.zig +++ b/codegen/gencat.zig @@ -67,10 +67,9 @@ pub fn main() !void { // Process DerivedGeneralCategory.txt var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -101,18 +100,23 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u5).init(allocator); + var stage2 = std.array_list.Managed(u5).init(allocator); defer stage2.deinit(); - var stage3 = std.ArrayList(u5).init(allocator); + var stage3 = std.array_list.Managed(u5).init(allocator); defer stage3.deinit(); var block: Block = [_]u5{0} ** block_size; @@ -151,21 +155,20 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try writer.writeInt(u8, @intCast(stage3.items.len), endian); - for (stage3.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian); + for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/hangul.zig b/codegen/hangul.zig index 2c42bb7..6768b3f 100644 --- a/codegen/hangul.zig +++ b/codegen/hangul.zig @@ -30,7 +30,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -43,10 +43,9 @@ pub fn main() !void { // Process HangulSyllableType.txt var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -77,15 +76,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u3).init(allocator); + var stage2 = std.array_list.Managed(u3).init(allocator); defer stage2.deinit(); var block: Block = [_]u3{0} ** block_size; @@ -116,18 +119,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/lower.zig b/codegen/lower.zig index a053fe3..c11cb0a 100644 --- a/codegen/lower.zig +++ b/codegen/lower.zig @@ -6,27 +6,25 @@ pub fn main() !void { defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -42,16 +40,20 @@ pub fn main() !void { 13 => { // Simple lowercase mapping if (field.len == 0) continue :lines; - try writer.writeInt(i24, cp, endian); + try writer.interface.writeInt(i24, cp, endian); const mapping = try std.fmt.parseInt(i24, field, 16); - try writer.writeInt(i24, mapping - cp, endian); + try writer.interface.writeInt(i24, mapping - cp, endian); }, else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - - try writer.writeInt(u24, 0, endian); - try out_comp.flush(); + try writer.interface.writeInt(u24, 0, endian); + try writer.interface.flush(); } diff --git a/codegen/normp.zig b/codegen/normp.zig index 60dabdc..71a1ae5 100644 --- a/codegen/normp.zig +++ b/codegen/normp.zig @@ -21,7 +21,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -34,10 +34,9 @@ pub fn main() !void { // Process DerivedNormalizationProps.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -78,15 +77,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u3).init(allocator); + var stage2 = std.array_list.Managed(u3).init(allocator); defer stage2.deinit(); var block: Block = [_]u3{0} ** block_size; @@ -117,18 +120,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/numeric.zig b/codegen/numeric.zig index 038ac0a..cf918e8 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedNumericType.txt var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/props.zig b/codegen/props.zig index 24b22e0..6ff0a33 100644 --- a/codegen/props.zig +++ b/codegen/props.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process PropList.txt var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,16 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); - - try out_comp.flush(); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); + try writer.interface.flush(); } diff --git a/codegen/scripts.zig b/codegen/scripts.zig index 530205d..bdd3d9d 100644 --- a/codegen/scripts.zig +++ b/codegen/scripts.zig @@ -195,7 +195,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -208,10 +208,9 @@ pub fn main() !void { // Process DerivedGeneralCategory.txt var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -245,18 +244,22 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); - var stage3 = std.ArrayList(u8).init(allocator); + var stage3 = std.array_list.Managed(u8).init(allocator); defer stage3.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -295,21 +298,20 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try writer.writeInt(u8, @intCast(stage3.items.len), endian); - for (stage3.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian); + for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/upper.zig b/codegen/upper.zig index 5848911..7d96a0e 100644 --- a/codegen/upper.zig +++ b/codegen/upper.zig @@ -1,32 +1,29 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); - + var file_writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -42,16 +39,24 @@ pub fn main() !void { 12 => { // Simple uppercase mapping if (field.len == 0) continue :lines; - try writer.writeInt(i24, cp, endian); + try file_writer.interface.writeInt(i24, cp, endian); const mapping = try std.fmt.parseInt(i24, field, 16); - try writer.writeInt(i24, mapping - cp, endian); + try file_writer.interface.writeInt(i24, mapping - cp, endian); }, else => {}, } } + } else |err| switch (err) { + error.ReadFailed => { + return in_reader.err orelse err; + }, + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u24, 0, endian); - try out_comp.flush(); + try file_writer.interface.writeInt(u24, 0, endian); + try file_writer.interface.flush(); } diff --git a/codegen/wbp.zig b/codegen/wbp.zig index 741103e..b1ed56a 100644 --- a/codegen/wbp.zig +++ b/codegen/wbp.zig @@ -43,7 +43,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -56,10 +56,9 @@ pub fn main() !void { // Process HangulSyllableType.txt var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -90,15 +89,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u5).init(allocator); + var stage2 = std.array_list.Managed(u5).init(allocator); defer stage2.deinit(); var block: Block = [_]u5{0} ** block_size; @@ -129,18 +132,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/src/CanonData.zig b/src/CanonData.zig index 5d2332a..cf9dc8a 100644 --- a/src/CanonData.zig +++ b/src/CanonData.zig @@ -7,11 +7,9 @@ cps: []u21 = undefined, const CanonData = @This(); pub fn init(allocator: mem.Allocator) !CanonData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("canon"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var cdata = CanonData{ diff --git a/src/CaseFolding.zig b/src/CaseFolding.zig index ff41b3e..df86b92 100644 --- a/src/CaseFolding.zig +++ b/src/CaseFolding.zig @@ -48,11 +48,9 @@ fn setupImpl(casefold: *CaseFolding, allocator: Allocator) Allocator.Error!void } inline fn setupImplInner(casefold: *CaseFolding, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("fold"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -123,7 +121,7 @@ pub fn caseFoldAlloc( allocator: Allocator, cps: []const u21, ) Allocator.Error![]const u21 { - var cfcps = std.ArrayList(u21).init(allocator); + var cfcps = std.array_list.Managed(u21).init(allocator); defer cfcps.deinit(); var buf: [3]u21 = undefined; diff --git a/src/CombiningData.zig b/src/CombiningData.zig index fd64a3b..f58e0de 100644 --- a/src/CombiningData.zig +++ b/src/CombiningData.zig @@ -6,11 +6,9 @@ s2: []u8 = undefined, const CombiningData = @This(); pub fn init(allocator: mem.Allocator) !CombiningData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("ccc"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -46,5 +44,4 @@ pub fn isStarter(cbdata: CombiningData, cp: u21) bool { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; diff --git a/src/CompatData.zig b/src/CompatData.zig index 794abca..40ecd12 100644 --- a/src/CompatData.zig +++ b/src/CompatData.zig @@ -6,11 +6,9 @@ cps: []u21 = undefined, const CompatData = @This(); pub fn init(allocator: mem.Allocator) !CompatData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("compat"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var cpdata = CompatData{ @@ -55,6 +53,5 @@ pub fn toNfkd(cpdata: *const CompatData, cp: u21) []u21 { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const magic = @import("magic"); diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig index 3da2d24..82b2649 100644 --- a/src/DisplayWidth.zig +++ b/src/DisplayWidth.zig @@ -39,11 +39,9 @@ pub fn setupWithGraphemes(dw: *DisplayWidth, allocator: Allocator, graphemes: Gr // Sets up the DisplayWidthData, leaving the GraphemeData undefined. pub fn setup(dw: *DisplayWidth, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("dwp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -400,7 +398,7 @@ pub fn wrap( columns: usize, threshold: usize, ) ![]u8 { - var result = ArrayList(u8).init(allocator); + var result = std.array_list.Managed(u8).init(allocator); defer result.deinit(); var line_iter = mem.tokenizeAny(u8, str, "\r\n"); @@ -460,8 +458,6 @@ test "allocation test" { const std = @import("std"); const builtin = @import("builtin"); const options = @import("options"); -const ArrayList = std.ArrayList; -const compress = std.compress; const mem = std.mem; const Allocator = mem.Allocator; const simd = std.simd; diff --git a/src/GeneralCategories.zig b/src/GeneralCategories.zig index 8c1b6a3..eee7e56 100644 --- a/src/GeneralCategories.zig +++ b/src/GeneralCategories.zig @@ -47,11 +47,9 @@ pub fn init(allocator: Allocator) Allocator.Error!GeneralCategories { } pub fn setup(gencat: *GeneralCategories, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("gencat"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/Graphemes.zig b/src/Graphemes.zig index f1c56ed..81d874c 100644 --- a/src/Graphemes.zig +++ b/src/Graphemes.zig @@ -16,11 +16,9 @@ pub fn init(allocator: Allocator) Allocator.Error!Graphemes { } pub fn setup(graphemes: *Graphemes, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("gbp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/HangulData.zig b/src/HangulData.zig index 8c5f3ad..cae8b97 100644 --- a/src/HangulData.zig +++ b/src/HangulData.zig @@ -15,11 +15,9 @@ s2: []u3 = undefined, const Hangul = @This(); pub fn init(allocator: mem.Allocator) !Hangul { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("hangul"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var hangul = Hangul{}; @@ -49,6 +47,5 @@ pub fn syllable(hangul: *const Hangul, cp: u21) Syllable { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const testing = std.testing; diff --git a/src/LetterCasing.zig b/src/LetterCasing.zig index 11a3e96..33096fc 100644 --- a/src/LetterCasing.zig +++ b/src/LetterCasing.zig @@ -22,7 +22,6 @@ pub fn setup(case: *LetterCasing, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const endian = builtin.cpu.arch.endian(); self.case_map = try allocator.alloc([2]u21, 0x110000); @@ -36,8 +35,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Uppercase const upper_bytes = @embedFile("upper"); var upper_fbs = std.io.fixedBufferStream(upper_bytes); - var upper_decomp = decompressor(.raw, upper_fbs.reader()); - var upper_reader = upper_decomp.reader(); + var upper_reader = upper_fbs.reader(); while (true) { const cp = try upper_reader.readInt(i24, endian); @@ -49,8 +47,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Lowercase const lower_bytes = @embedFile("lower"); var lower_fbs = std.io.fixedBufferStream(lower_bytes); - var lower_decomp = decompressor(.raw, lower_fbs.reader()); - var lower_reader = lower_decomp.reader(); + var lower_reader = lower_fbs.reader(); while (true) { const cp = try lower_reader.readInt(i24, endian); @@ -62,8 +59,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Case properties const cp_bytes = @embedFile("case_prop"); var cp_fbs = std.io.fixedBufferStream(cp_bytes); - var cp_decomp = decompressor(.raw, cp_fbs.reader()); - var cp_reader = cp_decomp.reader(); + var cp_reader = cp_fbs.reader(); const stage_1_len: u16 = try cp_reader.readInt(u16, endian); self.prop_s1 = try allocator.alloc(u16, stage_1_len); @@ -122,7 +118,7 @@ pub fn toUpperStr( allocator: mem.Allocator, str: []const u8, ) ![]u8 { - var bytes = std.ArrayList(u8).init(allocator); + var bytes = std.array_list.Managed(u8).init(allocator); defer bytes.deinit(); var iter = CodePointIterator{ .bytes = str }; @@ -180,7 +176,7 @@ pub fn toLowerStr( allocator: mem.Allocator, str: []const u8, ) ![]u8 { - var bytes = std.ArrayList(u8).init(allocator); + var bytes = std.array_list.Managed(u8).init(allocator); defer bytes.deinit(); var iter = CodePointIterator{ .bytes = str }; diff --git a/src/NormPropsData.zig b/src/NormPropsData.zig index ca69569..7b53542 100644 --- a/src/NormPropsData.zig +++ b/src/NormPropsData.zig @@ -6,11 +6,9 @@ s2: []u4 = undefined, const NormProps = @This(); pub fn init(allocator: mem.Allocator) !NormProps { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("normp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var norms = NormProps{}; @@ -50,6 +48,5 @@ pub fn isFcx(norms: *const NormProps, cp: u21) bool { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const testing = std.testing; diff --git a/src/Normalize.zig b/src/Normalize.zig index 989ec29..4a1bae8 100644 --- a/src/Normalize.zig +++ b/src/Normalize.zig @@ -305,7 +305,7 @@ pub fn nfkd(self: Normalize, allocator: Allocator, str: []const u8) Allocator.Er } pub fn nfxdCodePoints(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var cp_iter = CodePointIterator{ .bytes = str }; @@ -332,7 +332,7 @@ fn nfxd(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo const dcps = try self.nfxdCodePoints(allocator, str, form); defer allocator.free(dcps); - var dstr_list = std.ArrayList(u8).init(allocator); + var dstr_list = std.array_list.Managed(u8).init(allocator); defer dstr_list.deinit(); var buf: [4]u8 = undefined; @@ -393,7 +393,7 @@ pub fn nfdCodePoints( allocator: Allocator, cps: []const u21, ) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var dc_buf: [18]u21 = undefined; @@ -418,7 +418,7 @@ pub fn nfkdCodePoints( allocator: Allocator, cps: []const u21, ) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var dc_buf: [18]u21 = undefined; @@ -560,7 +560,7 @@ fn nfxc(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo // If we have no deletions. the code point sequence // has been fully composed. if (deleted == 0) { - var cstr_list = std.ArrayList(u8).init(allocator); + var cstr_list = std.array_list.Managed(u8).init(allocator); defer cstr_list.deinit(); var buf: [4]u8 = undefined; diff --git a/src/Properties.zig b/src/Properties.zig index 73602a0..432d176 100644 --- a/src/Properties.zig +++ b/src/Properties.zig @@ -25,14 +25,12 @@ pub fn setup(props: *Properties, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(props: *Properties, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const endian = builtin.cpu.arch.endian(); // Process DerivedCoreProperties.txt const core_bytes = @embedFile("core_props"); var core_fbs = std.io.fixedBufferStream(core_bytes); - var core_decomp = decompressor(.raw, core_fbs.reader()); - var core_reader = core_decomp.reader(); + var core_reader = core_fbs.reader(); const core_stage_1_len: u16 = try core_reader.readInt(u16, endian); props.core_s1 = try allocator.alloc(u16, core_stage_1_len); @@ -47,8 +45,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void { // Process PropList.txt const props_bytes = @embedFile("props"); var props_fbs = std.io.fixedBufferStream(props_bytes); - var props_decomp = decompressor(.raw, props_fbs.reader()); - var props_reader = props_decomp.reader(); + var props_reader = props_fbs.reader(); const stage_1_len: u16 = try props_reader.readInt(u16, endian); props.props_s1 = try allocator.alloc(u16, stage_1_len); @@ -63,8 +60,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void { // Process DerivedNumericType.txt const num_bytes = @embedFile("numeric"); var num_fbs = std.io.fixedBufferStream(num_bytes); - var num_decomp = decompressor(.raw, num_fbs.reader()); - var num_reader = num_decomp.reader(); + var num_reader = num_fbs.reader(); const num_stage_1_len: u16 = try num_reader.readInt(u16, endian); props.num_s1 = try allocator.alloc(u16, num_stage_1_len); diff --git a/src/Scripts.zig b/src/Scripts.zig index 3bc90bc..719b01f 100644 --- a/src/Scripts.zig +++ b/src/Scripts.zig @@ -196,11 +196,9 @@ pub fn setup(scripts: *Scripts, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(scripts: *Scripts, allocator: mem.Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("scripts"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -250,7 +248,6 @@ test "Allocation failure" { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const Allocator = mem.Allocator; const testing = std.testing; diff --git a/src/Words.zig b/src/Words.zig index 617c34d..ce3203f 100644 --- a/src/Words.zig +++ b/src/Words.zig @@ -605,11 +605,9 @@ const SneakIterator = struct { }; inline fn setupImpl(wb: *Words, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("wbp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index ae177a9..ff49b2a 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig @@ -3,35 +3,34 @@ const dbg_print = false; test "Unicode normalization tests" { var arena = heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); - var allocator = arena.allocator(); + const allocator = arena.allocator(); const n = try Normalize.init(allocator); defer n.deinit(allocator); var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); defer file.close(); - var buf_reader = io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); - var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); + var cp_buf: [4]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |line| { + while (line_iter.next()) |line| { // Iterate over fields. var fields = mem.splitScalar(u8, line, ';'); var field_index: usize = 0; var input: []u8 = undefined; - defer allocator.free(input); - + if (dbg_print) std.debug.print("Line: {s}\n", .{line}); while (fields.next()) |field| : (field_index += 1) { if (field_index == 0) { - var i_buf = std.ArrayList(u8).init(allocator); + var i_buf = std.array_list.Managed(u8).init(allocator); defer i_buf.deinit(); var i_fields = mem.splitScalar(u8, field, ' '); while (i_fields.next()) |s| { + if (dbg_print) std.debug.print("Debug: {s}\n", .{s}); const icp = try fmt.parseInt(u21, s, 16); const len = try unicode.utf8Encode(icp, &cp_buf); try i_buf.appendSlice(cp_buf[0..len]); @@ -41,7 +40,7 @@ test "Unicode normalization tests" { } else if (field_index == 1) { if (dbg_print) debug.print("\n*** {s} ***\n", .{line}); // NFC, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -58,7 +57,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 2) { // NFD, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -75,7 +74,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 3) { // NFKC, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -92,7 +91,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 4) { // NFKD, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -111,6 +110,11 @@ test "Unicode normalization tests" { continue; } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } @@ -118,26 +122,25 @@ test "Segmentation GraphemeIterator" { const allocator = std.testing.allocator; var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{}); defer file.close(); - var buf_reader = std.io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); + var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); const graph = try Graphemes.init(allocator); defer graph.deinit(allocator); - var buf: [4096]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |raw| { + while (line_iter.next()) |raw| { // Clean up. var line = std.mem.trimLeft(u8, raw, "÷ "); if (std.mem.indexOf(u8, line, " ÷\t")) |final| { line = line[0..final]; } // Iterate over fields. - var want = std.ArrayList(Grapheme).init(allocator); + var want = std.array_list.Managed(Grapheme).init(allocator); defer want.deinit(); - var all_bytes = std.ArrayList(u8).init(allocator); + var all_bytes = std.array_list.Managed(u8).init(allocator); defer all_bytes.deinit(); var graphemes = std.mem.splitSequence(u8, line, " ÷ "); @@ -250,6 +253,11 @@ test "Segmentation GraphemeIterator" { } } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } @@ -257,26 +265,25 @@ test "Segmentation Word Iterator" { const allocator = std.testing.allocator; var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{}); defer file.close(); - var buf_reader = std.io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); + var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); const wb = try Words.init(allocator); defer wb.deinit(allocator); - var buf: [4096]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |raw| { + while (line_iter.next()) |raw| { // Clean up. var line = std.mem.trimLeft(u8, raw, "÷ "); if (std.mem.indexOf(u8, line, " ÷\t")) |final| { line = line[0..final]; } // Iterate over fields. - var want = std.ArrayList(Word).init(allocator); + var want = std.array_list.Managed(Word).init(allocator); defer want.deinit(); - var all_bytes = std.ArrayList(u8).init(allocator); + var all_bytes = std.array_list.Managed(u8).init(allocator); defer all_bytes.deinit(); var words = std.mem.splitSequence(u8, line, " ÷ "); @@ -439,26 +446,26 @@ test "Segmentation Word Iterator" { if (idx == 0) break; } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } const IterRead = struct { - read: *Reader, + read: *io.Reader, line: usize = 0, - pub fn next(iter: *IterRead, buf: []u8) !?[]const u8 { - defer iter.line += 1; - const maybe_line = try iter.read.readUntilDelimiterOrEof(buf, '#'); - if (maybe_line) |this_line| { - try iter.read.skipUntilDelimiterOrEof('\n'); - if (this_line.len == 0 or this_line[0] == '@') { - // comment, next line - return iter.next(buf); - } else { - return this_line; - } + pub fn next(iter: *IterRead) anyerror![]const u8 { + iter.line += 1; + const this_line = try iter.read.takeDelimiterExclusive('\n'); + if (this_line.len == 0 or this_line[0] == '@' or this_line[0] == '#') { + // comment, next line + return iter.next(); } else { - return null; + return this_line; } } }; @@ -467,7 +474,6 @@ const std = @import("std"); const fmt = std.fmt; const fs = std.fs; const io = std.io; -const Reader = io.BufferedReader(4096, fs.File.Reader).Reader; const heap = std.heap; const mem = std.mem; const debug = std.debug; -- cgit v1.2.3 From 749197a3f9d25e211615960c02380a3d659b20f9 Mon Sep 17 00:00:00 2001 From: Michael Chaten Date: Sun, 14 Sep 2025 04:11:09 -0700 Subject: Embed data files in scripts rather than relying on filesystem access for easier packaging --- build.zig | 25 +++++++++++++++++++++++++ codegen/canon.zig | 8 ++------ codegen/case_prop.zig | 9 ++------- codegen/ccc.zig | 9 ++------- codegen/compat.zig | 7 ++----- codegen/core_props.zig | 9 ++------- codegen/dwp.zig | 14 ++++---------- codegen/fold.zig | 13 ++++--------- codegen/gbp.zig | 23 +++++++++-------------- codegen/gencat.zig | 9 ++------- codegen/hangul.zig | 9 ++------- codegen/lower.zig | 8 ++------ codegen/normp.zig | 9 ++------- codegen/numeric.zig | 9 ++------- codegen/props.zig | 9 ++------- codegen/scripts.zig | 11 +++-------- codegen/upper.zig | 11 ++--------- codegen/wbp.zig | 10 +++------- src/unicode_tests.zig | 23 ++++++----------------- 19 files changed, 78 insertions(+), 147 deletions(-) diff --git a/build.zig b/build.zig index 67a009a..5678cd1 100644 --- a/build.zig +++ b/build.zig @@ -48,6 +48,9 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + gbp_gen_exe.root_module.addAnonymousImport("DerivedCoreProperties.txt", .{ .root_source_file = b.path("data/unicode/DerivedCoreProperties.txt") }); + gbp_gen_exe.root_module.addAnonymousImport("GraphemeBreakProperty.txt", .{ .root_source_file = b.path("data/unicode/auxiliary/GraphemeBreakProperty.txt") }); + gbp_gen_exe.root_module.addAnonymousImport("emoji-data.txt", .{ .root_source_file = b.path("data/unicode/emoji/emoji-data.txt") }); const run_gbp_gen_exe = b.addRunArtifact(gbp_gen_exe); const gbp_gen_out = run_gbp_gen_exe.addOutputFileArg("gbp.bin.z"); @@ -59,6 +62,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + wbp_gen_exe.root_module.addAnonymousImport("WordBreakProperty.txt", .{ .root_source_file = b.path("data/unicode/auxiliary/WordBreakProperty.txt") }); const run_wbp_gen_exe = b.addRunArtifact(wbp_gen_exe); const wbp_gen_out = run_wbp_gen_exe.addOutputFileArg("wbp.bin.z"); @@ -70,6 +74,8 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + dwp_gen_exe.root_module.addAnonymousImport("DerivedEastAsianWidth.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedEastAsianWidth.txt") }); + dwp_gen_exe.root_module.addAnonymousImport("DerivedGeneralCategory.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedGeneralCategory.txt") }); dwp_gen_exe.root_module.addOptions("options", dwp_options); const run_dwp_gen_exe = b.addRunArtifact(dwp_gen_exe); const dwp_gen_out = run_dwp_gen_exe.addOutputFileArg("dwp.bin.z"); @@ -83,6 +89,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + canon_gen_exe.root_module.addAnonymousImport("UnicodeData.txt", .{ .root_source_file = b.path("data/unicode/UnicodeData.txt") }); const run_canon_gen_exe = b.addRunArtifact(canon_gen_exe); const canon_gen_out = run_canon_gen_exe.addOutputFileArg("canon.bin.z"); @@ -94,6 +101,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + compat_gen_exe.root_module.addAnonymousImport("UnicodeData.txt", .{ .root_source_file = b.path("data/unicode/UnicodeData.txt") }); const run_compat_gen_exe = b.addRunArtifact(compat_gen_exe); const compat_gen_out = run_compat_gen_exe.addOutputFileArg("compat.bin.z"); @@ -105,6 +113,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + hangul_gen_exe.root_module.addAnonymousImport("HangulSyllableType.txt", .{ .root_source_file = b.path("data/unicode/HangulSyllableType.txt") }); const run_hangul_gen_exe = b.addRunArtifact(hangul_gen_exe); const hangul_gen_out = run_hangul_gen_exe.addOutputFileArg("hangul.bin.z"); @@ -116,6 +125,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + normp_gen_exe.root_module.addAnonymousImport("DerivedNormalizationProps.txt", .{ .root_source_file = b.path("data/unicode/DerivedNormalizationProps.txt") }); const run_normp_gen_exe = b.addRunArtifact(normp_gen_exe); const normp_gen_out = run_normp_gen_exe.addOutputFileArg("normp.bin.z"); @@ -127,6 +137,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + ccc_gen_exe.root_module.addAnonymousImport("DerivedCombiningClass.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedCombiningClass.txt") }); const run_ccc_gen_exe = b.addRunArtifact(ccc_gen_exe); const ccc_gen_out = run_ccc_gen_exe.addOutputFileArg("ccc.bin.z"); @@ -138,6 +149,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + gencat_gen_exe.root_module.addAnonymousImport("DerivedGeneralCategory.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedGeneralCategory.txt") }); const run_gencat_gen_exe = b.addRunArtifact(gencat_gen_exe); const gencat_gen_out = run_gencat_gen_exe.addOutputFileArg("gencat.bin.z"); @@ -149,6 +161,8 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + fold_gen_exe.root_module.addAnonymousImport("DerivedCoreProperties.txt", .{ .root_source_file = b.path("data/unicode/DerivedCoreProperties.txt") }); + fold_gen_exe.root_module.addAnonymousImport("CaseFolding.txt", .{ .root_source_file = b.path("data/unicode/CaseFolding.txt") }); const run_fold_gen_exe = b.addRunArtifact(fold_gen_exe); const fold_gen_out = run_fold_gen_exe.addOutputFileArg("fold.bin.z"); @@ -161,6 +175,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + num_gen_exe.root_module.addAnonymousImport("DerivedNumericType.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedNumericType.txt") }); const run_num_gen_exe = b.addRunArtifact(num_gen_exe); const num_gen_out = run_num_gen_exe.addOutputFileArg("numeric.bin.z"); @@ -173,6 +188,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + case_prop_gen_exe.root_module.addAnonymousImport("DerivedCoreProperties.txt", .{ .root_source_file = b.path("data/unicode/DerivedCoreProperties.txt") }); const run_case_prop_gen_exe = b.addRunArtifact(case_prop_gen_exe); const case_prop_gen_out = run_case_prop_gen_exe.addOutputFileArg("case_prop.bin.z"); @@ -185,6 +201,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + upper_gen_exe.root_module.addAnonymousImport("UnicodeData.txt", .{ .root_source_file = b.path("data/unicode/UnicodeData.txt") }); const run_upper_gen_exe = b.addRunArtifact(upper_gen_exe); const upper_gen_out = run_upper_gen_exe.addOutputFileArg("upper.bin.z"); @@ -197,6 +214,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + lower_gen_exe.root_module.addAnonymousImport("UnicodeData.txt", .{ .root_source_file = b.path("data/unicode/UnicodeData.txt") }); const run_lower_gen_exe = b.addRunArtifact(lower_gen_exe); const lower_gen_out = run_lower_gen_exe.addOutputFileArg("lower.bin.z"); @@ -208,6 +226,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + scripts_gen_exe.root_module.addAnonymousImport("Scripts.txt", .{ .root_source_file = b.path("data/unicode/Scripts.txt") }); const run_scripts_gen_exe = b.addRunArtifact(scripts_gen_exe); const scripts_gen_out = run_scripts_gen_exe.addOutputFileArg("scripts.bin.z"); @@ -219,6 +238,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + core_gen_exe.root_module.addAnonymousImport("DerivedCoreProperties.txt", .{ .root_source_file = b.path("data/unicode/DerivedCoreProperties.txt") }); const run_core_gen_exe = b.addRunArtifact(core_gen_exe); const core_gen_out = run_core_gen_exe.addOutputFileArg("core_props.bin.z"); @@ -230,6 +250,8 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + + props_gen_exe.root_module.addAnonymousImport("PropList.txt", .{ .root_source_file = b.path("data/unicode/PropList.txt") }); const run_props_gen_exe = b.addRunArtifact(props_gen_exe); const props_gen_out = run_props_gen_exe.addOutputFileArg("props.bin.z"); @@ -483,6 +505,9 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }), }); + unicode_tests.root_module.addAnonymousImport("GraphemeBreakTest.txt", .{ .root_source_file = b.path("data/unicode/auxiliary/GraphemeBreakTest.txt") }); + unicode_tests.root_module.addAnonymousImport("NormalizationTest.txt", .{ .root_source_file = b.path("data/unicode/NormalizationTest.txt") }); + unicode_tests.root_module.addAnonymousImport("WordBreakTest.txt", .{ .root_source_file = b.path("data/unicode/auxiliary/WordBreakTest.txt") }); unicode_tests.root_module.addImport("Graphemes", graphemes); unicode_tests.root_module.addImport("Normalize", norm); unicode_tests.root_module.addImport("Words", words); diff --git a/codegen/canon.zig b/codegen/canon.zig index 89a9070..34e720f 100644 --- a/codegen/canon.zig +++ b/codegen/canon.zig @@ -6,13 +6,9 @@ pub fn main() anyerror!void { defer arena.deinit(); const allocator = arena.allocator(); - var line_buf: [4096]u8 = undefined; var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt - var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - + var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt")); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); @@ -24,7 +20,7 @@ pub fn main() anyerror!void { var writer = &file_writer.interface; const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.interface.takeDelimiterInclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig index 66eb62c..46b8e65 100644 --- a/codegen/case_prop.zig +++ b/codegen/case_prop.zig @@ -30,14 +30,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedCoreProperties.txt - var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/ccc.zig b/codegen/ccc.zig index b9b2bc3..48d3a9d 100644 --- a/codegen/ccc.zig +++ b/codegen/ccc.zig @@ -29,14 +29,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedCombiningClass.txt - var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); - defer cc_file.close(); - var cc_reader = cc_file.reader(&line_buf); - - while (cc_reader.interface.takeDelimiterExclusive('\n')) |line| { + var cc_reader = std.io.Reader.fixed(@embedFile("DerivedCombiningClass.txt")); + while (cc_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/compat.zig b/codegen/compat.zig index 835a636..0a06c44 100644 --- a/codegen/compat.zig +++ b/codegen/compat.zig @@ -7,12 +7,9 @@ pub fn main() anyerror!void { const allocator = arena.allocator(); // Process UnicodeData.txt - var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); - defer in_file.close(); - var line_buf: [4096]u8 = undefined; var write_buf: [4096]u8 = undefined; - var in_reader = in_file.reader(&line_buf); + var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt")); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); @@ -24,7 +21,7 @@ pub fn main() anyerror!void { const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/core_props.zig b/codegen/core_props.zig index 19063f6..d59a77e 100644 --- a/codegen/core_props.zig +++ b/codegen/core_props.zig @@ -30,14 +30,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedCoreProperties.txt - var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/dwp.zig b/codegen/dwp.zig index fad08d1..8189ad8 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig @@ -31,14 +31,10 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, i4).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedEastAsianWidth.txt - var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); - defer deaw_file.close(); - var deaw_reader = deaw_file.reader(&line_buf); + var deaw_reader = std.io.Reader.fixed(@embedFile("DerivedEastAsianWidth.txt")); - while (deaw_reader.interface.takeDelimiterExclusive('\n')) |line| { + while (deaw_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; // @missing ranges @@ -94,11 +90,9 @@ pub fn main() anyerror!void { }, } // Process DerivedGeneralCategory.txt - var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); - defer dgc_file.close(); - var dgc_reader = dgc_file.reader(&line_buf); + var dgc_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt")); - while (dgc_reader.interface.takeDelimiterExclusive('\n')) |line| { + while (dgc_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/fold.zig b/codegen/fold.zig index cc381a8..d927795 100644 --- a/codegen/fold.zig +++ b/codegen/fold.zig @@ -7,16 +7,13 @@ pub fn main() anyerror!void { defer std.debug.assert(gpa.deinit() == .ok); const allocator = gpa.allocator(); - var line_buf: [4096]u8 = undefined; // Process DerivedCoreProperties.txt - var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); - defer props_file.close(); - var props_reader = props_file.reader(&line_buf); + var props_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); var props_map = std.AutoHashMap(u21, void).init(allocator); defer props_map.deinit(); - props_lines: while (props_reader.interface.takeDelimiterExclusive('\n')) |line| { + props_lines: while (props_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -57,11 +54,9 @@ pub fn main() anyerror!void { defer codepoint_mapping.deinit(); // Process CaseFolding.txt - var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{}); - defer cp_file.close(); - var cp_reader = cp_file.reader(&line_buf); - while (cp_reader.interface.takeDelimiterExclusive('\n')) |line| { + var cp_reader = std.io.Reader.fixed(@embedFile("CaseFolding.txt")); + while (cp_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; var field_it = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/gbp.zig b/codegen/gbp.zig index d654cf1..895aa7a 100644 --- a/codegen/gbp.zig +++ b/codegen/gbp.zig @@ -61,14 +61,11 @@ pub fn main() anyerror!void { var emoji_set = std.AutoHashMap(u21, void).init(allocator); defer emoji_set.deinit(); - var line_buf: [4096]u8 = undefined; - // Process Indic - var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); - defer indic_file.close(); - var indic_reader = indic_file.reader(&line_buf); + const indic_file = @embedFile("DerivedCoreProperties.txt"); + var indic_reader = std.io.Reader.fixed(indic_file); - while (indic_reader.interface.takeDelimiterExclusive('\n')) |line| { + while (indic_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "InCB") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -106,11 +103,10 @@ pub fn main() anyerror!void { }, } // Process GBP - var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{}); - defer gbp_file.close(); - var gbp_reader = gbp_file.reader(&line_buf); - while (gbp_reader.interface.takeDelimiterExclusive('\n')) |line| { + var gbp_reader = std.io.Reader.fixed(@embedFile("GraphemeBreakProperty.txt")); + + while (gbp_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -147,11 +143,10 @@ pub fn main() anyerror!void { }, } // Process Emoji - var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{}); - defer emoji_file.close(); - var emoji_reader = emoji_file.reader(&line_buf); - while (emoji_reader.interface.takeDelimiterExclusive('\n')) |line| { + var emoji_reader = std.io.Reader.fixed(@embedFile("emoji-data.txt")); + + while (emoji_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/gencat.zig b/codegen/gencat.zig index 7dedb5d..79fa072 100644 --- a/codegen/gencat.zig +++ b/codegen/gencat.zig @@ -62,14 +62,9 @@ pub fn main() !void { var flat_map = std.AutoHashMap(u21, u5).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedGeneralCategory.txt - var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/hangul.zig b/codegen/hangul.zig index 6768b3f..64ab11c 100644 --- a/codegen/hangul.zig +++ b/codegen/hangul.zig @@ -38,14 +38,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u3).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process HangulSyllableType.txt - var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("HangulSyllableType.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/lower.zig b/codegen/lower.zig index c11cb0a..987f004 100644 --- a/codegen/lower.zig +++ b/codegen/lower.zig @@ -6,13 +6,9 @@ pub fn main() !void { defer arena.deinit(); const allocator = arena.allocator(); - var line_buf: [4096]u8 = undefined; var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt - var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - + var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt")); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); @@ -24,7 +20,7 @@ pub fn main() !void { const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/normp.zig b/codegen/normp.zig index 71a1ae5..3cdd770 100644 --- a/codegen/normp.zig +++ b/codegen/normp.zig @@ -29,14 +29,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u3).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedNormalizationProps.txt - var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedNormalizationProps.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/numeric.zig b/codegen/numeric.zig index cf918e8..d6b3165 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig @@ -30,14 +30,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedNumericType.txt - var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedNumericType.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/props.zig b/codegen/props.zig index 6ff0a33..ca42987 100644 --- a/codegen/props.zig +++ b/codegen/props.zig @@ -30,14 +30,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process PropList.txt - var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("PropList.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/scripts.zig b/codegen/scripts.zig index bdd3d9d..81511cc 100644 --- a/codegen/scripts.zig +++ b/codegen/scripts.zig @@ -203,14 +203,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - - // Process DerivedGeneralCategory.txt - var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + // Process Scripts.txt + var in_reader = std.io.Reader.fixed(@embedFile("Scripts.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/upper.zig b/codegen/upper.zig index 7d96a0e..108fdbd 100644 --- a/codegen/upper.zig +++ b/codegen/upper.zig @@ -6,13 +6,9 @@ pub fn main() anyerror!void { defer arena.deinit(); const allocator = arena.allocator(); - var line_buf: [4096]u8 = undefined; var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt - var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - + var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt")); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); @@ -23,7 +19,7 @@ pub fn main() anyerror!void { var file_writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -48,9 +44,6 @@ pub fn main() anyerror!void { } } } else |err| switch (err) { - error.ReadFailed => { - return in_reader.err orelse err; - }, error.EndOfStream => {}, else => { return err; diff --git a/codegen/wbp.zig b/codegen/wbp.zig index b1ed56a..dfdc32e 100644 --- a/codegen/wbp.zig +++ b/codegen/wbp.zig @@ -51,14 +51,10 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u5).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process HangulSyllableType.txt - var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + const in_file = @embedFile("WordBreakProperty.txt"); + var in_reader = std.io.Reader.fixed(in_file); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index ff49b2a..875c5f0 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig @@ -8,14 +8,10 @@ test "Unicode normalization tests" { const n = try Normalize.init(allocator); defer n.deinit(allocator); - var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); - defer file.close(); - var buf: [4096]u8 = undefined; - var reader = file.reader(&buf); - + var reader = std.io.Reader.fixed(@embedFile("NormalizationTest.txt")); var cp_buf: [4]u8 = undefined; - var line_iter: IterRead = .{ .read = &reader.interface }; + var line_iter: IterRead = .{ .read = &reader }; while (line_iter.next()) |line| { // Iterate over fields. @@ -120,15 +116,12 @@ test "Unicode normalization tests" { test "Segmentation GraphemeIterator" { const allocator = std.testing.allocator; - var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{}); - defer file.close(); - var buf: [4096]u8 = undefined; - var reader = file.reader(&buf); + var reader = std.io.Reader.fixed(@embedFile("GraphemeBreakTest.txt")); const graph = try Graphemes.init(allocator); defer graph.deinit(allocator); - var line_iter: IterRead = .{ .read = &reader.interface }; + var line_iter: IterRead = .{ .read = &reader }; while (line_iter.next()) |raw| { // Clean up. @@ -263,15 +256,11 @@ test "Segmentation GraphemeIterator" { test "Segmentation Word Iterator" { const allocator = std.testing.allocator; - var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{}); - defer file.close(); - var buf: [4096]u8 = undefined; - var reader = file.reader(&buf); - + var reader = std.io.Reader.fixed(@embedFile("WordBreakTest.txt")); const wb = try Words.init(allocator); defer wb.deinit(allocator); - var line_iter: IterRead = .{ .read = &reader.interface }; + var line_iter: IterRead = .{ .read = &reader }; while (line_iter.next()) |raw| { // Clean up. -- cgit v1.2.3 From 2f02c3b16c073d0bd3d9368a66ce272a574f75a3 Mon Sep 17 00:00:00 2001 From: Jay Date: Sat, 8 Nov 2025 19:59:08 +1100 Subject: Use takeDelimiterInclusive to support Zig 0.15.2 --- codegen/canon.zig | 3 ++- codegen/case_prop.zig | 3 ++- codegen/ccc.zig | 3 ++- codegen/compat.zig | 3 ++- codegen/core_props.zig | 3 ++- codegen/dwp.zig | 6 ++++-- codegen/fold.zig | 6 ++++-- codegen/gbp.zig | 9 ++++++--- codegen/gencat.zig | 3 ++- codegen/hangul.zig | 3 ++- codegen/lower.zig | 3 ++- codegen/normp.zig | 3 ++- codegen/numeric.zig | 3 ++- codegen/props.zig | 3 ++- codegen/scripts.zig | 3 ++- codegen/upper.zig | 3 ++- codegen/wbp.zig | 3 ++- src/unicode_tests.zig | 3 ++- 18 files changed, 44 insertions(+), 22 deletions(-) diff --git a/codegen/canon.zig b/codegen/canon.zig index 34e720f..d95a905 100644 --- a/codegen/canon.zig +++ b/codegen/canon.zig @@ -20,7 +20,8 @@ pub fn main() anyerror!void { var writer = &file_writer.interface; const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.takeDelimiterInclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig index 46b8e65..613f7f6 100644 --- a/codegen/case_prop.zig +++ b/codegen/case_prop.zig @@ -32,7 +32,8 @@ pub fn main() anyerror!void { // Process DerivedCoreProperties.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/ccc.zig b/codegen/ccc.zig index 48d3a9d..4e470ae 100644 --- a/codegen/ccc.zig +++ b/codegen/ccc.zig @@ -31,7 +31,8 @@ pub fn main() anyerror!void { // Process DerivedCombiningClass.txt var cc_reader = std.io.Reader.fixed(@embedFile("DerivedCombiningClass.txt")); - while (cc_reader.takeDelimiterExclusive('\n')) |line| { + while (cc_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/compat.zig b/codegen/compat.zig index 0a06c44..debb83d 100644 --- a/codegen/compat.zig +++ b/codegen/compat.zig @@ -21,7 +21,8 @@ pub fn main() anyerror!void { const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/core_props.zig b/codegen/core_props.zig index d59a77e..6ffdf91 100644 --- a/codegen/core_props.zig +++ b/codegen/core_props.zig @@ -32,7 +32,8 @@ pub fn main() anyerror!void { // Process DerivedCoreProperties.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/dwp.zig b/codegen/dwp.zig index 8189ad8..75ac68e 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig @@ -34,7 +34,8 @@ pub fn main() anyerror!void { // Process DerivedEastAsianWidth.txt var deaw_reader = std.io.Reader.fixed(@embedFile("DerivedEastAsianWidth.txt")); - while (deaw_reader.takeDelimiterExclusive('\n')) |line| { + while (deaw_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; // @missing ranges @@ -92,7 +93,8 @@ pub fn main() anyerror!void { // Process DerivedGeneralCategory.txt var dgc_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt")); - while (dgc_reader.takeDelimiterExclusive('\n')) |line| { + while (dgc_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/fold.zig b/codegen/fold.zig index d927795..366ed79 100644 --- a/codegen/fold.zig +++ b/codegen/fold.zig @@ -13,7 +13,8 @@ pub fn main() anyerror!void { var props_map = std.AutoHashMap(u21, void).init(allocator); defer props_map.deinit(); - props_lines: while (props_reader.takeDelimiterExclusive('\n')) |line| { + props_lines: while (props_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -56,7 +57,8 @@ pub fn main() anyerror!void { // Process CaseFolding.txt var cp_reader = std.io.Reader.fixed(@embedFile("CaseFolding.txt")); - while (cp_reader.takeDelimiterExclusive('\n')) |line| { + while (cp_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; var field_it = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/gbp.zig b/codegen/gbp.zig index 895aa7a..1d06e9a 100644 --- a/codegen/gbp.zig +++ b/codegen/gbp.zig @@ -65,7 +65,8 @@ pub fn main() anyerror!void { const indic_file = @embedFile("DerivedCoreProperties.txt"); var indic_reader = std.io.Reader.fixed(indic_file); - while (indic_reader.takeDelimiterExclusive('\n')) |line| { + while (indic_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "InCB") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -106,7 +107,8 @@ pub fn main() anyerror!void { var gbp_reader = std.io.Reader.fixed(@embedFile("GraphemeBreakProperty.txt")); - while (gbp_reader.takeDelimiterExclusive('\n')) |line| { + while (gbp_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -146,7 +148,8 @@ pub fn main() anyerror!void { var emoji_reader = std.io.Reader.fixed(@embedFile("emoji-data.txt")); - while (emoji_reader.takeDelimiterExclusive('\n')) |line| { + while (emoji_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/gencat.zig b/codegen/gencat.zig index 79fa072..9800f1d 100644 --- a/codegen/gencat.zig +++ b/codegen/gencat.zig @@ -64,7 +64,8 @@ pub fn main() !void { // Process DerivedGeneralCategory.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/hangul.zig b/codegen/hangul.zig index 64ab11c..2e4c175 100644 --- a/codegen/hangul.zig +++ b/codegen/hangul.zig @@ -40,7 +40,8 @@ pub fn main() anyerror!void { // Process HangulSyllableType.txt var in_reader = std.io.Reader.fixed(@embedFile("HangulSyllableType.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/lower.zig b/codegen/lower.zig index 987f004..91f3ef2 100644 --- a/codegen/lower.zig +++ b/codegen/lower.zig @@ -20,7 +20,8 @@ pub fn main() !void { const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/normp.zig b/codegen/normp.zig index 3cdd770..eaf6989 100644 --- a/codegen/normp.zig +++ b/codegen/normp.zig @@ -31,7 +31,8 @@ pub fn main() anyerror!void { // Process DerivedNormalizationProps.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedNormalizationProps.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/numeric.zig b/codegen/numeric.zig index d6b3165..b304349 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig @@ -32,7 +32,8 @@ pub fn main() anyerror!void { // Process DerivedNumericType.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedNumericType.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/props.zig b/codegen/props.zig index ca42987..35c7dfb 100644 --- a/codegen/props.zig +++ b/codegen/props.zig @@ -32,7 +32,8 @@ pub fn main() anyerror!void { // Process PropList.txt var in_reader = std.io.Reader.fixed(@embedFile("PropList.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/scripts.zig b/codegen/scripts.zig index 81511cc..0f0194c 100644 --- a/codegen/scripts.zig +++ b/codegen/scripts.zig @@ -205,7 +205,8 @@ pub fn main() anyerror!void { // Process Scripts.txt var in_reader = std.io.Reader.fixed(@embedFile("Scripts.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/upper.zig b/codegen/upper.zig index 108fdbd..5eb29e7 100644 --- a/codegen/upper.zig +++ b/codegen/upper.zig @@ -19,7 +19,8 @@ pub fn main() anyerror!void { var file_writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/wbp.zig b/codegen/wbp.zig index dfdc32e..33eeea5 100644 --- a/codegen/wbp.zig +++ b/codegen/wbp.zig @@ -54,7 +54,8 @@ pub fn main() anyerror!void { // Process HangulSyllableType.txt const in_file = @embedFile("WordBreakProperty.txt"); var in_reader = std.io.Reader.fixed(in_file); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index 875c5f0..e2a5a96 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig @@ -449,7 +449,8 @@ const IterRead = struct { pub fn next(iter: *IterRead) anyerror![]const u8 { iter.line += 1; - const this_line = try iter.read.takeDelimiterExclusive('\n'); + const took = try iter.read.takeDelimiterInclusive('\n'); + const this_line = std.mem.trimRight(u8, took, "\n"); if (this_line.len == 0 or this_line[0] == '@' or this_line[0] == '#') { // comment, next line return iter.next(); -- cgit v1.2.3 From 0b50c9662813d0a6afd07dbf2472442555497149 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 23 Dec 2025 09:37:20 -0500 Subject: Bump version refs --- README.md | 4 ++-- build.zig.zon | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3abe480..c2db6e2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The Unicode version supported by zg is `16.0.0`. ## Zig Version -The minimum Zig version required is `0.14`. +The minimum Zig version required is `0.15.2`. ## Integrating zg into your Zig Project @@ -19,7 +19,7 @@ You first need to add zg as a dependency in your `build.zig.zon` file. In your Zig project's root directory, run: ```plain -zig fetch --save https://codeberg.org/atman/zg/archive/v0.14.1.tar.gz +zig fetch --save https://codeberg.org/atman/zg/archive/v0.15.2.tar.gz ``` Then instantiate the dependency in your `build.zig`: diff --git a/build.zig.zon b/build.zig.zon index e585449..c160bd9 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ .name = .zg, - .version = "0.15.1", - .minimum_zig_version = "0.15.1", + .version = "0.15.2", + .minimum_zig_version = "0.15.2", .fingerprint = 0x47df7778dc946aa0, .paths = .{ -- cgit v1.2.3 From 041afd58de8525dc0b6f6a9e2b493031dbf4bbee Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 23 Dec 2025 10:22:06 -0500 Subject: Fix #74: Check for characters before popping in wrap --- src/DisplayWidth.zig | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig index 82b2649..d15dbae 100644 --- a/src/DisplayWidth.zig +++ b/src/DisplayWidth.zig @@ -420,8 +420,10 @@ pub fn wrap( } // Remove trailing space and newline. - _ = result.pop(); - _ = result.pop(); + if (result.items[result.items.len - 1] == '\n') + _ = result.pop(); + if (result.items[result.items.len - 1] == ' ') + _ = result.pop(); return try result.toOwnedSlice(); } @@ -438,6 +440,18 @@ test "wrap" { try testing.expectEqualStrings(want, got); } +test "zg/74" { + var debug_alloc = std.heap.DebugAllocator(.{}).init; + const allocator = debug_alloc.allocator(); + defer _ = debug_alloc.deinit(); + const dw = try DisplayWidth.init(allocator); + defer dw.deinit(allocator); + const wrapped = try dw.wrap(allocator, "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam pellentesque pulvinar felis, sit amet commodo ligula feugiat sed. Sed quis malesuada elit, nec eleifend lectus. Sed tincidunt finibus aliquet. Praesent consectetur nibh libero, tempus imperdiet lorem congue eget.", 16, 1); + defer allocator.free(wrapped); + const expected_wrap = "Lorem ipsum dolor \nsit amet, consectetur \nadipiscing elit. \nNullam pellentesque \npulvinar felis, \nsit amet commodo \nligula feugiat \nsed. Sed quis malesuada \nelit, nec eleifend \nlectus. Sed tincidunt \nfinibus aliquet. \nPraesent consectetur \nnibh libero, tempus \nimperdiet lorem \ncongue eget."; + try std.testing.expectEqualStrings(expected_wrap, wrapped); +} + fn testAllocation(allocator: Allocator) !void { { var dw = try DisplayWidth.init(allocator); -- cgit v1.2.3 From 9f725580a2b6c93825edeff27f06951f57bcc237 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 23 Dec 2025 11:04:01 -0500 Subject: Use width 2 when skin tone modifier detected Fix: #82 --- README.md | 2 +- build.zig.zon | 2 +- src/DisplayWidth.zig | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c2db6e2..6ba456f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ You first need to add zg as a dependency in your `build.zig.zon` file. In your Zig project's root directory, run: ```plain -zig fetch --save https://codeberg.org/atman/zg/archive/v0.15.2.tar.gz +zig fetch --save https://codeberg.org/atman/zg/archive/v0.15.3.tar.gz ``` Then instantiate the dependency in your `build.zig`: diff --git a/build.zig.zon b/build.zig.zon index c160bd9..0308457 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = .zg, - .version = "0.15.2", + .version = "0.15.3", .minimum_zig_version = "0.15.2", .fingerprint = 0x47df7778dc946aa0, diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig index d15dbae..9919a55 100644 --- a/src/DisplayWidth.zig +++ b/src/DisplayWidth.zig @@ -126,6 +126,8 @@ pub fn strWidth(dw: DisplayWidth, str: []const u8) usize { // emoji text sequence. if (ncp.code == 0xFE0E) w = 1; if (ncp.code == 0xFE0F) w = 2; + // Skin tones + if (0x1F3FB <= ncp.code and ncp.code <= 0x1F3FF) w = 2; } // Only adding width of first non-zero-width code point. @@ -201,6 +203,9 @@ test "strWidth" { try testing.expectEqual(@as(usize, 9), dw.strWidth("Ẓ̌á̲l͔̝̞̄̑͌g̖̘̘̔̔͢͞͝o̪̔T̢̙̫̈̍͞e̬͈͕͌̏͑x̺̍ṭ̓̓ͅ")); try testing.expectEqual(@as(usize, 17), dw.strWidth("슬라바 우크라이나")); try testing.expectEqual(@as(usize, 1), dw.strWidth("\u{378}")); + + // https://codeberg.org/atman/zg/issues/82 + try testing.expectEqual(@as(usize, 12), dw.strWidth("✍️✍🏻✍🏼✍🏽✍🏾✍🏿")); } /// centers `str` in a new string of width `total_width` (in display cells) using `pad` as padding. -- cgit v1.2.3