From 16df10181dab7834370823efc7cce4a55db570c5 Mon Sep 17 00:00:00 2001 From: Michael Chaten Date: Sat, 13 Sep 2025 07:11:50 -0700 Subject: Update build to 0.15.0 --- build.zig | 176 ++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 90 insertions(+), 86 deletions(-) diff --git a/build.zig b/build.zig index ca0eeef..67a009a 100644 --- a/build.zig +++ b/build.zig @@ -42,27 +42,33 @@ pub fn build(b: *std.Build) void { // Grapheme break const gbp_gen_exe = b.addExecutable(.{ .name = "gbp", - .root_source_file = b.path("codegen/gbp.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/gbp.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_gbp_gen_exe = b.addRunArtifact(gbp_gen_exe); const gbp_gen_out = run_gbp_gen_exe.addOutputFileArg("gbp.bin.z"); const wbp_gen_exe = b.addExecutable(.{ .name = "wbp", - .root_source_file = b.path("codegen/wbp.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/wbp.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_wbp_gen_exe = b.addRunArtifact(wbp_gen_exe); const wbp_gen_out = run_wbp_gen_exe.addOutputFileArg("wbp.bin.z"); const dwp_gen_exe = b.addExecutable(.{ .name = "dwp", - .root_source_file = b.path("codegen/dwp.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/dwp.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); dwp_gen_exe.root_module.addOptions("options", dwp_options); const run_dwp_gen_exe = b.addRunArtifact(dwp_gen_exe); @@ -71,63 +77,77 @@ pub fn build(b: *std.Build) void { // Normalization properties const canon_gen_exe = b.addExecutable(.{ .name = "canon", - .root_source_file = b.path("codegen/canon.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/canon.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_canon_gen_exe = b.addRunArtifact(canon_gen_exe); const canon_gen_out = run_canon_gen_exe.addOutputFileArg("canon.bin.z"); const compat_gen_exe = b.addExecutable(.{ .name = "compat", - .root_source_file = b.path("codegen/compat.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/compat.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_compat_gen_exe = b.addRunArtifact(compat_gen_exe); const compat_gen_out = run_compat_gen_exe.addOutputFileArg("compat.bin.z"); const hangul_gen_exe = b.addExecutable(.{ .name = "hangul", - .root_source_file = b.path("codegen/hangul.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/hangul.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_hangul_gen_exe = b.addRunArtifact(hangul_gen_exe); const hangul_gen_out = run_hangul_gen_exe.addOutputFileArg("hangul.bin.z"); const normp_gen_exe = b.addExecutable(.{ .name = "normp", - .root_source_file = b.path("codegen/normp.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/normp.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_normp_gen_exe = b.addRunArtifact(normp_gen_exe); const normp_gen_out = run_normp_gen_exe.addOutputFileArg("normp.bin.z"); const ccc_gen_exe = b.addExecutable(.{ .name = "ccc", - .root_source_file = b.path("codegen/ccc.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/ccc.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_ccc_gen_exe = b.addRunArtifact(ccc_gen_exe); const ccc_gen_out = run_ccc_gen_exe.addOutputFileArg("ccc.bin.z"); const gencat_gen_exe = b.addExecutable(.{ .name = "gencat", - .root_source_file = b.path("codegen/gencat.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/gencat.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_gencat_gen_exe = b.addRunArtifact(gencat_gen_exe); const gencat_gen_out = run_gencat_gen_exe.addOutputFileArg("gencat.bin.z"); const fold_gen_exe = b.addExecutable(.{ .name = "fold", - .root_source_file = b.path("codegen/fold.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/fold.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_fold_gen_exe = b.addRunArtifact(fold_gen_exe); const fold_gen_out = run_fold_gen_exe.addOutputFileArg("fold.bin.z"); @@ -135,9 +155,11 @@ pub fn build(b: *std.Build) void { // Numeric types const num_gen_exe = b.addExecutable(.{ .name = "numeric", - .root_source_file = b.path("codegen/numeric.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/numeric.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_num_gen_exe = b.addRunArtifact(num_gen_exe); const num_gen_out = run_num_gen_exe.addOutputFileArg("numeric.bin.z"); @@ -145,9 +167,11 @@ pub fn build(b: *std.Build) void { // Letter case properties const case_prop_gen_exe = b.addExecutable(.{ .name = "case_prop", - .root_source_file = b.path("codegen/case_prop.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/case_prop.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_case_prop_gen_exe = b.addRunArtifact(case_prop_gen_exe); const case_prop_gen_out = run_case_prop_gen_exe.addOutputFileArg("case_prop.bin.z"); @@ -155,9 +179,11 @@ pub fn build(b: *std.Build) void { // Uppercase mappings const upper_gen_exe = b.addExecutable(.{ .name = "upper", - .root_source_file = b.path("codegen/upper.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/upper.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_upper_gen_exe = b.addRunArtifact(upper_gen_exe); const upper_gen_out = run_upper_gen_exe.addOutputFileArg("upper.bin.z"); @@ -165,36 +191,44 @@ pub fn build(b: *std.Build) void { // Lowercase mappings const lower_gen_exe = b.addExecutable(.{ .name = "lower", - .root_source_file = b.path("codegen/lower.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/lower.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_lower_gen_exe = b.addRunArtifact(lower_gen_exe); const lower_gen_out = run_lower_gen_exe.addOutputFileArg("lower.bin.z"); const scripts_gen_exe = b.addExecutable(.{ .name = "scripts", - .root_source_file = b.path("codegen/scripts.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/scripts.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_scripts_gen_exe = b.addRunArtifact(scripts_gen_exe); const scripts_gen_out = run_scripts_gen_exe.addOutputFileArg("scripts.bin.z"); const core_gen_exe = b.addExecutable(.{ .name = "core", - .root_source_file = b.path("codegen/core_props.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/core_props.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_core_gen_exe = b.addRunArtifact(core_gen_exe); const core_gen_out = run_core_gen_exe.addOutputFileArg("core_props.bin.z"); const props_gen_exe = b.addExecutable(.{ .name = "props", - .root_source_file = b.path("codegen/props.zig"), - .target = b.graph.host, - .optimize = .Debug, + .root_module = b.createModule(.{ + .root_source_file = b.path("codegen/props.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); const run_props_gen_exe = b.addRunArtifact(props_gen_exe); const props_gen_out = run_props_gen_exe.addOutputFileArg("props.bin.z"); @@ -212,8 +246,6 @@ pub fn build(b: *std.Build) void { const code_point_t = b.addTest(.{ .name = "code_point", .root_module = code_point, - .target = target, - .optimize = optimize, }); const code_point_tr = b.addRunArtifact(code_point_t); @@ -230,8 +262,6 @@ pub fn build(b: *std.Build) void { const grapheme_t = b.addTest(.{ .name = "Graphemes", .root_module = graphemes, - .target = target, - .optimize = optimize, }); const grapheme_tr = b.addRunArtifact(grapheme_t); @@ -247,8 +277,6 @@ pub fn build(b: *std.Build) void { const words_t = b.addTest(.{ .name = "WordBreak", .root_module = words, - .target = target, - .optimize = optimize, }); const words_tr = b.addRunArtifact(words_t); @@ -262,8 +290,6 @@ pub fn build(b: *std.Build) void { const ascii_t = b.addTest(.{ .name = "ascii", .root_module = ascii, - .target = target, - .optimize = optimize, }); const ascii_tr = b.addRunArtifact(ascii_t); @@ -282,8 +308,6 @@ pub fn build(b: *std.Build) void { const display_width_t = b.addTest(.{ .name = "display_width", .root_module = display_width, - .target = target, - .optimize = optimize, }); const display_width_tr = b.addRunArtifact(display_width_t); @@ -298,8 +322,6 @@ pub fn build(b: *std.Build) void { const ccc_data_t = b.addTest(.{ .name = "ccc_data", .root_module = ccc_data, - .target = target, - .optimize = optimize, }); const ccc_data_tr = b.addRunArtifact(ccc_data_t); @@ -314,8 +336,6 @@ pub fn build(b: *std.Build) void { const canon_data_t = b.addTest(.{ .name = "canon_data", .root_module = canon_data, - .target = target, - .optimize = optimize, }); const canon_data_tr = b.addRunArtifact(canon_data_t); @@ -330,8 +350,6 @@ pub fn build(b: *std.Build) void { const compat_data_t = b.addTest(.{ .name = "compat_data", .root_module = compat_data, - .target = target, - .optimize = optimize, }); const compat_data_tr = b.addRunArtifact(compat_data_t); @@ -345,8 +363,6 @@ pub fn build(b: *std.Build) void { const hangul_data_t = b.addTest(.{ .name = "hangul_data", .root_module = hangul_data, - .target = target, - .optimize = optimize, }); const hangul_data_tr = b.addRunArtifact(hangul_data_t); @@ -360,8 +376,6 @@ pub fn build(b: *std.Build) void { const normp_data_t = b.addTest(.{ .name = "normp_data", .root_module = normp_data, - .target = target, - .optimize = optimize, }); const normp_data_tr = b.addRunArtifact(normp_data_t); @@ -381,8 +395,6 @@ pub fn build(b: *std.Build) void { const norm_t = b.addTest(.{ .name = "norm", .root_module = norm, - .target = target, - .optimize = optimize, }); const norm_tr = b.addRunArtifact(norm_t); @@ -397,8 +409,6 @@ pub fn build(b: *std.Build) void { const gencat_t = b.addTest(.{ .name = "gencat", .root_module = gencat, - .target = target, - .optimize = optimize, }); const gencat_tr = b.addRunArtifact(gencat_t); @@ -415,8 +425,6 @@ pub fn build(b: *std.Build) void { const case_fold_t = b.addTest(.{ .name = "case_fold", .root_module = case_fold, - .target = target, - .optimize = optimize, }); const case_fold_tr = b.addRunArtifact(case_fold_t); @@ -434,8 +442,6 @@ pub fn build(b: *std.Build) void { const letter_case_t = b.addTest(.{ .name = "lettercase", .root_module = letter_case, - .target = target, - .optimize = optimize, }); const letter_case_tr = b.addRunArtifact(letter_case_t); @@ -450,8 +456,6 @@ pub fn build(b: *std.Build) void { const scripts_t = b.addTest(.{ .name = "scripts", .root_module = scripts, - .target = target, - .optimize = optimize, }); const scripts_tr = b.addRunArtifact(scripts_t); @@ -468,16 +472,16 @@ pub fn build(b: *std.Build) void { const properties_t = b.addTest(.{ .name = "properties", .root_module = properties, - .target = target, - .optimize = optimize, }); const properties_tr = b.addRunArtifact(properties_t); // Unicode Tests const unicode_tests = b.addTest(.{ - .root_source_file = b.path("src/unicode_tests.zig"), - .target = target, - .optimize = optimize, + .root_module = b.createModule(.{ + .root_source_file = b.path("src/unicode_tests.zig"), + .target = target, + .optimize = optimize, + }), }); unicode_tests.root_module.addImport("Graphemes", graphemes); unicode_tests.root_module.addImport("Normalize", norm); -- cgit v1.2.3 From a3b5e884b12fdaa341010ef41bb9382fa0cd89f8 Mon Sep 17 00:00:00 2001 From: Michael Chaten Date: Sat, 13 Sep 2025 08:38:24 -0700 Subject: Update codebase to Zig 0.15.1. Removes compression support --- build.zig.zon | 4 +-- codegen/canon.zig | 24 +++++++------ codegen/case_prop.zig | 31 +++++++++-------- codegen/ccc.zig | 31 +++++++++-------- codegen/compat.zig | 27 ++++++++------- codegen/core_props.zig | 32 +++++++++-------- codegen/dwp.zig | 42 ++++++++++++---------- codegen/fold.zig | 59 ++++++++++++++++--------------- codegen/gbp.zig | 57 +++++++++++++++++------------- codegen/gencat.zig | 35 ++++++++++--------- codegen/hangul.zig | 32 +++++++++-------- codegen/lower.zig | 26 +++++++------- codegen/normp.zig | 32 +++++++++-------- codegen/numeric.zig | 31 +++++++++-------- codegen/props.zig | 32 +++++++++-------- codegen/scripts.zig | 38 ++++++++++---------- codegen/upper.zig | 31 ++++++++++------- codegen/wbp.zig | 32 +++++++++-------- src/CanonData.zig | 4 +-- src/CaseFolding.zig | 6 ++-- src/CombiningData.zig | 5 +-- src/CompatData.zig | 5 +-- src/DisplayWidth.zig | 8 ++--- src/GeneralCategories.zig | 4 +-- src/Graphemes.zig | 4 +-- src/HangulData.zig | 5 +-- src/LetterCasing.zig | 14 +++----- src/NormPropsData.zig | 5 +-- src/Normalize.zig | 10 +++--- src/Properties.zig | 10 ++---- src/Scripts.zig | 5 +-- src/Words.zig | 4 +-- src/unicode_tests.zig | 88 +++++++++++++++++++++++++---------------------- 33 files changed, 399 insertions(+), 374 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 3e1df95..e585449 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ .name = .zg, - .version = "0.14.1", - .minimum_zig_version = "0.14.0", + .version = "0.15.1", + .minimum_zig_version = "0.15.1", .fingerprint = 0x47df7778dc946aa0, .paths = .{ diff --git a/codegen/canon.zig b/codegen/canon.zig index 28b7f28..89a9070 100644 --- a/codegen/canon.zig +++ b/codegen/canon.zig @@ -1,32 +1,30 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); - + var file_writer = out_file.writer(&write_buf); + var writer = &file_writer.interface; const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterInclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -60,8 +58,12 @@ pub fn main() !void { try writer.writeInt(u8, @intCast(len), endian); for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u16, 0, endian); - try out_comp.flush(); + try writer.flush(); } diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig index 6c912a8..66eb62c 100644 --- a/codegen/case_prop.zig +++ b/codegen/case_prop.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedCoreProperties.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/ccc.zig b/codegen/ccc.zig index a01c8d2..b9b2bc3 100644 --- a/codegen/ccc.zig +++ b/codegen/ccc.zig @@ -21,7 +21,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -34,10 +34,9 @@ pub fn main() !void { // Process DerivedCombiningClass.txt var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); defer cc_file.close(); - var cc_buf = std.io.bufferedReader(cc_file.reader()); - const cc_reader = cc_buf.reader(); + var cc_reader = cc_file.reader(&line_buf); - while (try cc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (cc_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -68,15 +67,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -107,18 +111,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/compat.zig b/codegen/compat.zig index 07616fc..835a636 100644 --- a/codegen/compat.zig +++ b/codegen/compat.zig @@ -1,7 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -9,24 +9,22 @@ pub fn main() !void { // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -55,10 +53,15 @@ pub fn main() !void { } } - try writer.writeInt(u8, @intCast(len), endian); - for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); + try writer.interface.writeInt(u8, @intCast(len), endian); + for (cps[0..len]) |cp| try writer.interface.writeInt(u24, cp, endian); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u16, 0, endian); - try out_comp.flush(); + try writer.interface.writeInt(u16, 0, endian); + try writer.interface.flush(); } diff --git a/codegen/core_props.zig b/codegen/core_props.zig index f60c7a9..19063f6 100644 --- a/codegen/core_props.zig +++ b/codegen/core_props.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedCoreProperties.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -82,15 +81,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -121,18 +124,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var out_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&out_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/dwp.zig b/codegen/dwp.zig index 5e5bf6a..fad08d1 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig @@ -23,7 +23,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -36,10 +36,9 @@ pub fn main() !void { // Process DerivedEastAsianWidth.txt var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); defer deaw_file.close(); - var deaw_buf = std.io.bufferedReader(deaw_file.reader()); - const deaw_reader = deaw_buf.reader(); + var deaw_reader = deaw_file.reader(&line_buf); - while (try deaw_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (deaw_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; // @missing ranges @@ -88,15 +87,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process DerivedGeneralCategory.txt var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); defer dgc_file.close(); - var dgc_buf = std.io.bufferedReader(dgc_file.reader()); - const dgc_reader = dgc_buf.reader(); + var dgc_reader = dgc_file.reader(&line_buf); - while (try dgc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (dgc_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -139,15 +141,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(i4).init(allocator); + var stage2 = std.array_list.Managed(i4).init(allocator); defer stage2.deinit(); var block: Block = [_]i4{0} ** block_size; @@ -227,18 +234,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(i8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(i8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/fold.zig b/codegen/fold.zig index cb73cca..cc381a8 100644 --- a/codegen/fold.zig +++ b/codegen/fold.zig @@ -2,23 +2,21 @@ const std = @import("std"); const builtin = @import("builtin"); const mem = std.mem; -pub fn main() !void { +pub fn main() anyerror!void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer std.debug.assert(gpa.deinit() == .ok); const allocator = gpa.allocator(); + var line_buf: [4096]u8 = undefined; // Process DerivedCoreProperties.txt var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer props_file.close(); - var props_buf = std.io.bufferedReader(props_file.reader()); - const props_reader = props_buf.reader(); + var props_reader = props_file.reader(&line_buf); var props_map = std.AutoHashMap(u21, void).init(allocator); defer props_map.deinit(); - var line_buf: [4096]u8 = undefined; - - props_lines: while (try props_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + props_lines: while (props_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -49,18 +47,21 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator); defer codepoint_mapping.deinit(); // Process CaseFolding.txt var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{}); defer cp_file.close(); - var cp_buf = std.io.bufferedReader(cp_file.reader()); - const cp_reader = cp_buf.reader(); + var cp_reader = cp_file.reader(&line_buf); - while (try cp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (cp_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; var field_it = std.mem.splitScalar(u8, line, ';'); @@ -81,9 +82,14 @@ pub fn main() !void { } try codepoint_mapping.putNoClobber(codepoint, mapping_buf); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var changes_when_casefolded_exceptions = std.ArrayList(u21).init(allocator); + var changes_when_casefolded_exceptions = std.array_list.Managed(u21).init(allocator); defer changes_when_casefolded_exceptions.deinit(); { @@ -221,32 +227,31 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); // Table metadata. - try writer.writeInt(u24, @intCast(codepoint_cutoff), endian); - try writer.writeInt(u24, @intCast(multiple_codepoint_start), endian); + try writer.interface.writeInt(u24, @intCast(codepoint_cutoff), endian); + try writer.interface.writeInt(u24, @intCast(multiple_codepoint_start), endian); // Stage 1 - try writer.writeInt(u16, @intCast(meaningful_stage1.len), endian); - try writer.writeAll(meaningful_stage1); + try writer.interface.writeInt(u16, @intCast(meaningful_stage1.len), endian); + try writer.interface.writeAll(meaningful_stage1); // Stage 2 - try writer.writeInt(u16, @intCast(stage2.len), endian); - try writer.writeAll(stage2); + try writer.interface.writeInt(u16, @intCast(stage2.len), endian); + try writer.interface.writeAll(stage2); // Stage 3 - try writer.writeInt(u16, @intCast(stage3.len), endian); - for (stage3) |offset| try writer.writeInt(i24, offset, endian); + try writer.interface.writeInt(u16, @intCast(stage3.len), endian); + for (stage3) |offset| try writer.interface.writeInt(i24, offset, endian); // Changes when case folded // Min and max - try writer.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); - try writer.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); - try writer.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); - for (changes_when_casefolded_exceptions.items) |cp| try writer.writeInt(u24, cp, endian); + try writer.interface.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); + try writer.interface.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); + try writer.interface.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); + for (changes_when_casefolded_exceptions.items) |cp| try writer.interface.writeInt(u24, cp, endian); - try out_comp.flush(); + try writer.interface.flush(); } } diff --git a/codegen/gbp.zig b/codegen/gbp.zig index 3fc4461..d654cf1 100644 --- a/codegen/gbp.zig +++ b/codegen/gbp.zig @@ -47,7 +47,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -66,10 +66,9 @@ pub fn main() !void { // Process Indic var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer indic_file.close(); - var indic_buf = std.io.bufferedReader(indic_file.reader()); - const indic_reader = indic_buf.reader(); + var indic_reader = indic_file.reader(&line_buf); - while (try indic_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (indic_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "InCB") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -100,15 +99,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process GBP var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{}); defer gbp_file.close(); - var gbp_buf = std.io.bufferedReader(gbp_file.reader()); - const gbp_reader = gbp_buf.reader(); + var gbp_reader = gbp_file.reader(&line_buf); - while (try gbp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (gbp_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -138,15 +140,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process Emoji var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{}); defer emoji_file.close(); - var emoji_buf = std.io.bufferedReader(emoji_file.reader()); - const emoji_reader = emoji_buf.reader(); + var emoji_reader = emoji_file.reader(&line_buf); - while (try emoji_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (emoji_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -170,15 +175,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u16).init(allocator); + var stage2 = std.array_list.Managed(u16).init(allocator); defer stage2.deinit(); var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator); @@ -227,22 +237,21 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u16, i, endian); const props_bytes = stage3.keys(); - try writer.writeInt(u16, @intCast(props_bytes.len), endian); - try writer.writeAll(props_bytes); + try writer.interface.writeInt(u16, @intCast(props_bytes.len), endian); + try writer.interface.writeAll(props_bytes); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/gencat.zig b/codegen/gencat.zig index fe06bd7..7dedb5d 100644 --- a/codegen/gencat.zig +++ b/codegen/gencat.zig @@ -67,10 +67,9 @@ pub fn main() !void { // Process DerivedGeneralCategory.txt var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -101,18 +100,23 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u5).init(allocator); + var stage2 = std.array_list.Managed(u5).init(allocator); defer stage2.deinit(); - var stage3 = std.ArrayList(u5).init(allocator); + var stage3 = std.array_list.Managed(u5).init(allocator); defer stage3.deinit(); var block: Block = [_]u5{0} ** block_size; @@ -151,21 +155,20 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try writer.writeInt(u8, @intCast(stage3.items.len), endian); - for (stage3.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian); + for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/hangul.zig b/codegen/hangul.zig index 2c42bb7..6768b3f 100644 --- a/codegen/hangul.zig +++ b/codegen/hangul.zig @@ -30,7 +30,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -43,10 +43,9 @@ pub fn main() !void { // Process HangulSyllableType.txt var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -77,15 +76,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u3).init(allocator); + var stage2 = std.array_list.Managed(u3).init(allocator); defer stage2.deinit(); var block: Block = [_]u3{0} ** block_size; @@ -116,18 +119,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/lower.zig b/codegen/lower.zig index a053fe3..c11cb0a 100644 --- a/codegen/lower.zig +++ b/codegen/lower.zig @@ -6,27 +6,25 @@ pub fn main() !void { defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -42,16 +40,20 @@ pub fn main() !void { 13 => { // Simple lowercase mapping if (field.len == 0) continue :lines; - try writer.writeInt(i24, cp, endian); + try writer.interface.writeInt(i24, cp, endian); const mapping = try std.fmt.parseInt(i24, field, 16); - try writer.writeInt(i24, mapping - cp, endian); + try writer.interface.writeInt(i24, mapping - cp, endian); }, else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - - try writer.writeInt(u24, 0, endian); - try out_comp.flush(); + try writer.interface.writeInt(u24, 0, endian); + try writer.interface.flush(); } diff --git a/codegen/normp.zig b/codegen/normp.zig index 60dabdc..71a1ae5 100644 --- a/codegen/normp.zig +++ b/codegen/normp.zig @@ -21,7 +21,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -34,10 +34,9 @@ pub fn main() !void { // Process DerivedNormalizationProps.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -78,15 +77,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u3).init(allocator); + var stage2 = std.array_list.Managed(u3).init(allocator); defer stage2.deinit(); var block: Block = [_]u3{0} ** block_size; @@ -117,18 +120,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/numeric.zig b/codegen/numeric.zig index 038ac0a..cf918e8 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedNumericType.txt var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/props.zig b/codegen/props.zig index 24b22e0..6ff0a33 100644 --- a/codegen/props.zig +++ b/codegen/props.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process PropList.txt var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,16 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); - - try out_comp.flush(); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); + try writer.interface.flush(); } diff --git a/codegen/scripts.zig b/codegen/scripts.zig index 530205d..bdd3d9d 100644 --- a/codegen/scripts.zig +++ b/codegen/scripts.zig @@ -195,7 +195,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -208,10 +208,9 @@ pub fn main() !void { // Process DerivedGeneralCategory.txt var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -245,18 +244,22 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); - var stage3 = std.ArrayList(u8).init(allocator); + var stage3 = std.array_list.Managed(u8).init(allocator); defer stage3.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -295,21 +298,20 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try writer.writeInt(u8, @intCast(stage3.items.len), endian); - for (stage3.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian); + for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/upper.zig b/codegen/upper.zig index 5848911..7d96a0e 100644 --- a/codegen/upper.zig +++ b/codegen/upper.zig @@ -1,32 +1,29 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); - + var file_writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -42,16 +39,24 @@ pub fn main() !void { 12 => { // Simple uppercase mapping if (field.len == 0) continue :lines; - try writer.writeInt(i24, cp, endian); + try file_writer.interface.writeInt(i24, cp, endian); const mapping = try std.fmt.parseInt(i24, field, 16); - try writer.writeInt(i24, mapping - cp, endian); + try file_writer.interface.writeInt(i24, mapping - cp, endian); }, else => {}, } } + } else |err| switch (err) { + error.ReadFailed => { + return in_reader.err orelse err; + }, + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u24, 0, endian); - try out_comp.flush(); + try file_writer.interface.writeInt(u24, 0, endian); + try file_writer.interface.flush(); } diff --git a/codegen/wbp.zig b/codegen/wbp.zig index 741103e..b1ed56a 100644 --- a/codegen/wbp.zig +++ b/codegen/wbp.zig @@ -43,7 +43,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -56,10 +56,9 @@ pub fn main() !void { // Process HangulSyllableType.txt var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -90,15 +89,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u5).init(allocator); + var stage2 = std.array_list.Managed(u5).init(allocator); defer stage2.deinit(); var block: Block = [_]u5{0} ** block_size; @@ -129,18 +132,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/src/CanonData.zig b/src/CanonData.zig index 5d2332a..cf9dc8a 100644 --- a/src/CanonData.zig +++ b/src/CanonData.zig @@ -7,11 +7,9 @@ cps: []u21 = undefined, const CanonData = @This(); pub fn init(allocator: mem.Allocator) !CanonData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("canon"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var cdata = CanonData{ diff --git a/src/CaseFolding.zig b/src/CaseFolding.zig index ff41b3e..df86b92 100644 --- a/src/CaseFolding.zig +++ b/src/CaseFolding.zig @@ -48,11 +48,9 @@ fn setupImpl(casefold: *CaseFolding, allocator: Allocator) Allocator.Error!void } inline fn setupImplInner(casefold: *CaseFolding, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("fold"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -123,7 +121,7 @@ pub fn caseFoldAlloc( allocator: Allocator, cps: []const u21, ) Allocator.Error![]const u21 { - var cfcps = std.ArrayList(u21).init(allocator); + var cfcps = std.array_list.Managed(u21).init(allocator); defer cfcps.deinit(); var buf: [3]u21 = undefined; diff --git a/src/CombiningData.zig b/src/CombiningData.zig index fd64a3b..f58e0de 100644 --- a/src/CombiningData.zig +++ b/src/CombiningData.zig @@ -6,11 +6,9 @@ s2: []u8 = undefined, const CombiningData = @This(); pub fn init(allocator: mem.Allocator) !CombiningData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("ccc"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -46,5 +44,4 @@ pub fn isStarter(cbdata: CombiningData, cp: u21) bool { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; diff --git a/src/CompatData.zig b/src/CompatData.zig index 794abca..40ecd12 100644 --- a/src/CompatData.zig +++ b/src/CompatData.zig @@ -6,11 +6,9 @@ cps: []u21 = undefined, const CompatData = @This(); pub fn init(allocator: mem.Allocator) !CompatData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("compat"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var cpdata = CompatData{ @@ -55,6 +53,5 @@ pub fn toNfkd(cpdata: *const CompatData, cp: u21) []u21 { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const magic = @import("magic"); diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig index 3da2d24..82b2649 100644 --- a/src/DisplayWidth.zig +++ b/src/DisplayWidth.zig @@ -39,11 +39,9 @@ pub fn setupWithGraphemes(dw: *DisplayWidth, allocator: Allocator, graphemes: Gr // Sets up the DisplayWidthData, leaving the GraphemeData undefined. pub fn setup(dw: *DisplayWidth, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("dwp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -400,7 +398,7 @@ pub fn wrap( columns: usize, threshold: usize, ) ![]u8 { - var result = ArrayList(u8).init(allocator); + var result = std.array_list.Managed(u8).init(allocator); defer result.deinit(); var line_iter = mem.tokenizeAny(u8, str, "\r\n"); @@ -460,8 +458,6 @@ test "allocation test" { const std = @import("std"); const builtin = @import("builtin"); const options = @import("options"); -const ArrayList = std.ArrayList; -const compress = std.compress; const mem = std.mem; const Allocator = mem.Allocator; const simd = std.simd; diff --git a/src/GeneralCategories.zig b/src/GeneralCategories.zig index 8c1b6a3..eee7e56 100644 --- a/src/GeneralCategories.zig +++ b/src/GeneralCategories.zig @@ -47,11 +47,9 @@ pub fn init(allocator: Allocator) Allocator.Error!GeneralCategories { } pub fn setup(gencat: *GeneralCategories, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("gencat"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/Graphemes.zig b/src/Graphemes.zig index f1c56ed..81d874c 100644 --- a/src/Graphemes.zig +++ b/src/Graphemes.zig @@ -16,11 +16,9 @@ pub fn init(allocator: Allocator) Allocator.Error!Graphemes { } pub fn setup(graphemes: *Graphemes, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("gbp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/HangulData.zig b/src/HangulData.zig index 8c5f3ad..cae8b97 100644 --- a/src/HangulData.zig +++ b/src/HangulData.zig @@ -15,11 +15,9 @@ s2: []u3 = undefined, const Hangul = @This(); pub fn init(allocator: mem.Allocator) !Hangul { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("hangul"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var hangul = Hangul{}; @@ -49,6 +47,5 @@ pub fn syllable(hangul: *const Hangul, cp: u21) Syllable { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const testing = std.testing; diff --git a/src/LetterCasing.zig b/src/LetterCasing.zig index 11a3e96..33096fc 100644 --- a/src/LetterCasing.zig +++ b/src/LetterCasing.zig @@ -22,7 +22,6 @@ pub fn setup(case: *LetterCasing, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const endian = builtin.cpu.arch.endian(); self.case_map = try allocator.alloc([2]u21, 0x110000); @@ -36,8 +35,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Uppercase const upper_bytes = @embedFile("upper"); var upper_fbs = std.io.fixedBufferStream(upper_bytes); - var upper_decomp = decompressor(.raw, upper_fbs.reader()); - var upper_reader = upper_decomp.reader(); + var upper_reader = upper_fbs.reader(); while (true) { const cp = try upper_reader.readInt(i24, endian); @@ -49,8 +47,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Lowercase const lower_bytes = @embedFile("lower"); var lower_fbs = std.io.fixedBufferStream(lower_bytes); - var lower_decomp = decompressor(.raw, lower_fbs.reader()); - var lower_reader = lower_decomp.reader(); + var lower_reader = lower_fbs.reader(); while (true) { const cp = try lower_reader.readInt(i24, endian); @@ -62,8 +59,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Case properties const cp_bytes = @embedFile("case_prop"); var cp_fbs = std.io.fixedBufferStream(cp_bytes); - var cp_decomp = decompressor(.raw, cp_fbs.reader()); - var cp_reader = cp_decomp.reader(); + var cp_reader = cp_fbs.reader(); const stage_1_len: u16 = try cp_reader.readInt(u16, endian); self.prop_s1 = try allocator.alloc(u16, stage_1_len); @@ -122,7 +118,7 @@ pub fn toUpperStr( allocator: mem.Allocator, str: []const u8, ) ![]u8 { - var bytes = std.ArrayList(u8).init(allocator); + var bytes = std.array_list.Managed(u8).init(allocator); defer bytes.deinit(); var iter = CodePointIterator{ .bytes = str }; @@ -180,7 +176,7 @@ pub fn toLowerStr( allocator: mem.Allocator, str: []const u8, ) ![]u8 { - var bytes = std.ArrayList(u8).init(allocator); + var bytes = std.array_list.Managed(u8).init(allocator); defer bytes.deinit(); var iter = CodePointIterator{ .bytes = str }; diff --git a/src/NormPropsData.zig b/src/NormPropsData.zig index ca69569..7b53542 100644 --- a/src/NormPropsData.zig +++ b/src/NormPropsData.zig @@ -6,11 +6,9 @@ s2: []u4 = undefined, const NormProps = @This(); pub fn init(allocator: mem.Allocator) !NormProps { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("normp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var norms = NormProps{}; @@ -50,6 +48,5 @@ pub fn isFcx(norms: *const NormProps, cp: u21) bool { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const testing = std.testing; diff --git a/src/Normalize.zig b/src/Normalize.zig index 989ec29..4a1bae8 100644 --- a/src/Normalize.zig +++ b/src/Normalize.zig @@ -305,7 +305,7 @@ pub fn nfkd(self: Normalize, allocator: Allocator, str: []const u8) Allocator.Er } pub fn nfxdCodePoints(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var cp_iter = CodePointIterator{ .bytes = str }; @@ -332,7 +332,7 @@ fn nfxd(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo const dcps = try self.nfxdCodePoints(allocator, str, form); defer allocator.free(dcps); - var dstr_list = std.ArrayList(u8).init(allocator); + var dstr_list = std.array_list.Managed(u8).init(allocator); defer dstr_list.deinit(); var buf: [4]u8 = undefined; @@ -393,7 +393,7 @@ pub fn nfdCodePoints( allocator: Allocator, cps: []const u21, ) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var dc_buf: [18]u21 = undefined; @@ -418,7 +418,7 @@ pub fn nfkdCodePoints( allocator: Allocator, cps: []const u21, ) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var dc_buf: [18]u21 = undefined; @@ -560,7 +560,7 @@ fn nfxc(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo // If we have no deletions. the code point sequence // has been fully composed. if (deleted == 0) { - var cstr_list = std.ArrayList(u8).init(allocator); + var cstr_list = std.array_list.Managed(u8).init(allocator); defer cstr_list.deinit(); var buf: [4]u8 = undefined; diff --git a/src/Properties.zig b/src/Properties.zig index 73602a0..432d176 100644 --- a/src/Properties.zig +++ b/src/Properties.zig @@ -25,14 +25,12 @@ pub fn setup(props: *Properties, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(props: *Properties, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const endian = builtin.cpu.arch.endian(); // Process DerivedCoreProperties.txt const core_bytes = @embedFile("core_props"); var core_fbs = std.io.fixedBufferStream(core_bytes); - var core_decomp = decompressor(.raw, core_fbs.reader()); - var core_reader = core_decomp.reader(); + var core_reader = core_fbs.reader(); const core_stage_1_len: u16 = try core_reader.readInt(u16, endian); props.core_s1 = try allocator.alloc(u16, core_stage_1_len); @@ -47,8 +45,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void { // Process PropList.txt const props_bytes = @embedFile("props"); var props_fbs = std.io.fixedBufferStream(props_bytes); - var props_decomp = decompressor(.raw, props_fbs.reader()); - var props_reader = props_decomp.reader(); + var props_reader = props_fbs.reader(); const stage_1_len: u16 = try props_reader.readInt(u16, endian); props.props_s1 = try allocator.alloc(u16, stage_1_len); @@ -63,8 +60,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void { // Process DerivedNumericType.txt const num_bytes = @embedFile("numeric"); var num_fbs = std.io.fixedBufferStream(num_bytes); - var num_decomp = decompressor(.raw, num_fbs.reader()); - var num_reader = num_decomp.reader(); + var num_reader = num_fbs.reader(); const num_stage_1_len: u16 = try num_reader.readInt(u16, endian); props.num_s1 = try allocator.alloc(u16, num_stage_1_len); diff --git a/src/Scripts.zig b/src/Scripts.zig index 3bc90bc..719b01f 100644 --- a/src/Scripts.zig +++ b/src/Scripts.zig @@ -196,11 +196,9 @@ pub fn setup(scripts: *Scripts, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(scripts: *Scripts, allocator: mem.Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("scripts"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -250,7 +248,6 @@ test "Allocation failure" { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const Allocator = mem.Allocator; const testing = std.testing; diff --git a/src/Words.zig b/src/Words.zig index 617c34d..ce3203f 100644 --- a/src/Words.zig +++ b/src/Words.zig @@ -605,11 +605,9 @@ const SneakIterator = struct { }; inline fn setupImpl(wb: *Words, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("wbp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index ae177a9..ff49b2a 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig @@ -3,35 +3,34 @@ const dbg_print = false; test "Unicode normalization tests" { var arena = heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); - var allocator = arena.allocator(); + const allocator = arena.allocator(); const n = try Normalize.init(allocator); defer n.deinit(allocator); var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); defer file.close(); - var buf_reader = io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); - var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); + var cp_buf: [4]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |line| { + while (line_iter.next()) |line| { // Iterate over fields. var fields = mem.splitScalar(u8, line, ';'); var field_index: usize = 0; var input: []u8 = undefined; - defer allocator.free(input); - + if (dbg_print) std.debug.print("Line: {s}\n", .{line}); while (fields.next()) |field| : (field_index += 1) { if (field_index == 0) { - var i_buf = std.ArrayList(u8).init(allocator); + var i_buf = std.array_list.Managed(u8).init(allocator); defer i_buf.deinit(); var i_fields = mem.splitScalar(u8, field, ' '); while (i_fields.next()) |s| { + if (dbg_print) std.debug.print("Debug: {s}\n", .{s}); const icp = try fmt.parseInt(u21, s, 16); const len = try unicode.utf8Encode(icp, &cp_buf); try i_buf.appendSlice(cp_buf[0..len]); @@ -41,7 +40,7 @@ test "Unicode normalization tests" { } else if (field_index == 1) { if (dbg_print) debug.print("\n*** {s} ***\n", .{line}); // NFC, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -58,7 +57,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 2) { // NFD, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -75,7 +74,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 3) { // NFKC, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -92,7 +91,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 4) { // NFKD, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -111,6 +110,11 @@ test "Unicode normalization tests" { continue; } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } @@ -118,26 +122,25 @@ test "Segmentation GraphemeIterator" { const allocator = std.testing.allocator; var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{}); defer file.close(); - var buf_reader = std.io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); + var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); const graph = try Graphemes.init(allocator); defer graph.deinit(allocator); - var buf: [4096]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |raw| { + while (line_iter.next()) |raw| { // Clean up. var line = std.mem.trimLeft(u8, raw, "÷ "); if (std.mem.indexOf(u8, line, " ÷\t")) |final| { line = line[0..final]; } // Iterate over fields. - var want = std.ArrayList(Grapheme).init(allocator); + var want = std.array_list.Managed(Grapheme).init(allocator); defer want.deinit(); - var all_bytes = std.ArrayList(u8).init(allocator); + var all_bytes = std.array_list.Managed(u8).init(allocator); defer all_bytes.deinit(); var graphemes = std.mem.splitSequence(u8, line, " ÷ "); @@ -250,6 +253,11 @@ test "Segmentation GraphemeIterator" { } } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } @@ -257,26 +265,25 @@ test "Segmentation Word Iterator" { const allocator = std.testing.allocator; var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{}); defer file.close(); - var buf_reader = std.io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); + var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); const wb = try Words.init(allocator); defer wb.deinit(allocator); - var buf: [4096]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |raw| { + while (line_iter.next()) |raw| { // Clean up. var line = std.mem.trimLeft(u8, raw, "÷ "); if (std.mem.indexOf(u8, line, " ÷\t")) |final| { line = line[0..final]; } // Iterate over fields. - var want = std.ArrayList(Word).init(allocator); + var want = std.array_list.Managed(Word).init(allocator); defer want.deinit(); - var all_bytes = std.ArrayList(u8).init(allocator); + var all_bytes = std.array_list.Managed(u8).init(allocator); defer all_bytes.deinit(); var words = std.mem.splitSequence(u8, line, " ÷ "); @@ -439,26 +446,26 @@ test "Segmentation Word Iterator" { if (idx == 0) break; } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } const IterRead = struct { - read: *Reader, + read: *io.Reader, line: usize = 0, - pub fn next(iter: *IterRead, buf: []u8) !?[]const u8 { - defer iter.line += 1; - const maybe_line = try iter.read.readUntilDelimiterOrEof(buf, '#'); - if (maybe_line) |this_line| { - try iter.read.skipUntilDelimiterOrEof('\n'); - if (this_line.len == 0 or this_line[0] == '@') { - // comment, next line - return iter.next(buf); - } else { - return this_line; - } + pub fn next(iter: *IterRead) anyerror![]const u8 { + iter.line += 1; + const this_line = try iter.read.takeDelimiterExclusive('\n'); + if (this_line.len == 0 or this_line[0] == '@' or this_line[0] == '#') { + // comment, next line + return iter.next(); } else { - return null; + return this_line; } } }; @@ -467,7 +474,6 @@ const std = @import("std"); const fmt = std.fmt; const fs = std.fs; const io = std.io; -const Reader = io.BufferedReader(4096, fs.File.Reader).Reader; const heap = std.heap; const mem = std.mem; const debug = std.debug; -- cgit v1.2.3 From 749197a3f9d25e211615960c02380a3d659b20f9 Mon Sep 17 00:00:00 2001 From: Michael Chaten Date: Sun, 14 Sep 2025 04:11:09 -0700 Subject: Embed data files in scripts rather than relying on filesystem access for easier packaging --- build.zig | 25 +++++++++++++++++++++++++ codegen/canon.zig | 8 ++------ codegen/case_prop.zig | 9 ++------- codegen/ccc.zig | 9 ++------- codegen/compat.zig | 7 ++----- codegen/core_props.zig | 9 ++------- codegen/dwp.zig | 14 ++++---------- codegen/fold.zig | 13 ++++--------- codegen/gbp.zig | 23 +++++++++-------------- codegen/gencat.zig | 9 ++------- codegen/hangul.zig | 9 ++------- codegen/lower.zig | 8 ++------ codegen/normp.zig | 9 ++------- codegen/numeric.zig | 9 ++------- codegen/props.zig | 9 ++------- codegen/scripts.zig | 11 +++-------- codegen/upper.zig | 11 ++--------- codegen/wbp.zig | 10 +++------- src/unicode_tests.zig | 23 ++++++----------------- 19 files changed, 78 insertions(+), 147 deletions(-) diff --git a/build.zig b/build.zig index 67a009a..5678cd1 100644 --- a/build.zig +++ b/build.zig @@ -48,6 +48,9 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + gbp_gen_exe.root_module.addAnonymousImport("DerivedCoreProperties.txt", .{ .root_source_file = b.path("data/unicode/DerivedCoreProperties.txt") }); + gbp_gen_exe.root_module.addAnonymousImport("GraphemeBreakProperty.txt", .{ .root_source_file = b.path("data/unicode/auxiliary/GraphemeBreakProperty.txt") }); + gbp_gen_exe.root_module.addAnonymousImport("emoji-data.txt", .{ .root_source_file = b.path("data/unicode/emoji/emoji-data.txt") }); const run_gbp_gen_exe = b.addRunArtifact(gbp_gen_exe); const gbp_gen_out = run_gbp_gen_exe.addOutputFileArg("gbp.bin.z"); @@ -59,6 +62,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + wbp_gen_exe.root_module.addAnonymousImport("WordBreakProperty.txt", .{ .root_source_file = b.path("data/unicode/auxiliary/WordBreakProperty.txt") }); const run_wbp_gen_exe = b.addRunArtifact(wbp_gen_exe); const wbp_gen_out = run_wbp_gen_exe.addOutputFileArg("wbp.bin.z"); @@ -70,6 +74,8 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + dwp_gen_exe.root_module.addAnonymousImport("DerivedEastAsianWidth.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedEastAsianWidth.txt") }); + dwp_gen_exe.root_module.addAnonymousImport("DerivedGeneralCategory.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedGeneralCategory.txt") }); dwp_gen_exe.root_module.addOptions("options", dwp_options); const run_dwp_gen_exe = b.addRunArtifact(dwp_gen_exe); const dwp_gen_out = run_dwp_gen_exe.addOutputFileArg("dwp.bin.z"); @@ -83,6 +89,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + canon_gen_exe.root_module.addAnonymousImport("UnicodeData.txt", .{ .root_source_file = b.path("data/unicode/UnicodeData.txt") }); const run_canon_gen_exe = b.addRunArtifact(canon_gen_exe); const canon_gen_out = run_canon_gen_exe.addOutputFileArg("canon.bin.z"); @@ -94,6 +101,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + compat_gen_exe.root_module.addAnonymousImport("UnicodeData.txt", .{ .root_source_file = b.path("data/unicode/UnicodeData.txt") }); const run_compat_gen_exe = b.addRunArtifact(compat_gen_exe); const compat_gen_out = run_compat_gen_exe.addOutputFileArg("compat.bin.z"); @@ -105,6 +113,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + hangul_gen_exe.root_module.addAnonymousImport("HangulSyllableType.txt", .{ .root_source_file = b.path("data/unicode/HangulSyllableType.txt") }); const run_hangul_gen_exe = b.addRunArtifact(hangul_gen_exe); const hangul_gen_out = run_hangul_gen_exe.addOutputFileArg("hangul.bin.z"); @@ -116,6 +125,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + normp_gen_exe.root_module.addAnonymousImport("DerivedNormalizationProps.txt", .{ .root_source_file = b.path("data/unicode/DerivedNormalizationProps.txt") }); const run_normp_gen_exe = b.addRunArtifact(normp_gen_exe); const normp_gen_out = run_normp_gen_exe.addOutputFileArg("normp.bin.z"); @@ -127,6 +137,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + ccc_gen_exe.root_module.addAnonymousImport("DerivedCombiningClass.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedCombiningClass.txt") }); const run_ccc_gen_exe = b.addRunArtifact(ccc_gen_exe); const ccc_gen_out = run_ccc_gen_exe.addOutputFileArg("ccc.bin.z"); @@ -138,6 +149,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + gencat_gen_exe.root_module.addAnonymousImport("DerivedGeneralCategory.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedGeneralCategory.txt") }); const run_gencat_gen_exe = b.addRunArtifact(gencat_gen_exe); const gencat_gen_out = run_gencat_gen_exe.addOutputFileArg("gencat.bin.z"); @@ -149,6 +161,8 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + fold_gen_exe.root_module.addAnonymousImport("DerivedCoreProperties.txt", .{ .root_source_file = b.path("data/unicode/DerivedCoreProperties.txt") }); + fold_gen_exe.root_module.addAnonymousImport("CaseFolding.txt", .{ .root_source_file = b.path("data/unicode/CaseFolding.txt") }); const run_fold_gen_exe = b.addRunArtifact(fold_gen_exe); const fold_gen_out = run_fold_gen_exe.addOutputFileArg("fold.bin.z"); @@ -161,6 +175,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + num_gen_exe.root_module.addAnonymousImport("DerivedNumericType.txt", .{ .root_source_file = b.path("data/unicode/extracted/DerivedNumericType.txt") }); const run_num_gen_exe = b.addRunArtifact(num_gen_exe); const num_gen_out = run_num_gen_exe.addOutputFileArg("numeric.bin.z"); @@ -173,6 +188,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + case_prop_gen_exe.root_module.addAnonymousImport("DerivedCoreProperties.txt", .{ .root_source_file = b.path("data/unicode/DerivedCoreProperties.txt") }); const run_case_prop_gen_exe = b.addRunArtifact(case_prop_gen_exe); const case_prop_gen_out = run_case_prop_gen_exe.addOutputFileArg("case_prop.bin.z"); @@ -185,6 +201,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + upper_gen_exe.root_module.addAnonymousImport("UnicodeData.txt", .{ .root_source_file = b.path("data/unicode/UnicodeData.txt") }); const run_upper_gen_exe = b.addRunArtifact(upper_gen_exe); const upper_gen_out = run_upper_gen_exe.addOutputFileArg("upper.bin.z"); @@ -197,6 +214,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + lower_gen_exe.root_module.addAnonymousImport("UnicodeData.txt", .{ .root_source_file = b.path("data/unicode/UnicodeData.txt") }); const run_lower_gen_exe = b.addRunArtifact(lower_gen_exe); const lower_gen_out = run_lower_gen_exe.addOutputFileArg("lower.bin.z"); @@ -208,6 +226,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + scripts_gen_exe.root_module.addAnonymousImport("Scripts.txt", .{ .root_source_file = b.path("data/unicode/Scripts.txt") }); const run_scripts_gen_exe = b.addRunArtifact(scripts_gen_exe); const scripts_gen_out = run_scripts_gen_exe.addOutputFileArg("scripts.bin.z"); @@ -219,6 +238,7 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + core_gen_exe.root_module.addAnonymousImport("DerivedCoreProperties.txt", .{ .root_source_file = b.path("data/unicode/DerivedCoreProperties.txt") }); const run_core_gen_exe = b.addRunArtifact(core_gen_exe); const core_gen_out = run_core_gen_exe.addOutputFileArg("core_props.bin.z"); @@ -230,6 +250,8 @@ pub fn build(b: *std.Build) void { .optimize = .Debug, }), }); + + props_gen_exe.root_module.addAnonymousImport("PropList.txt", .{ .root_source_file = b.path("data/unicode/PropList.txt") }); const run_props_gen_exe = b.addRunArtifact(props_gen_exe); const props_gen_out = run_props_gen_exe.addOutputFileArg("props.bin.z"); @@ -483,6 +505,9 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }), }); + unicode_tests.root_module.addAnonymousImport("GraphemeBreakTest.txt", .{ .root_source_file = b.path("data/unicode/auxiliary/GraphemeBreakTest.txt") }); + unicode_tests.root_module.addAnonymousImport("NormalizationTest.txt", .{ .root_source_file = b.path("data/unicode/NormalizationTest.txt") }); + unicode_tests.root_module.addAnonymousImport("WordBreakTest.txt", .{ .root_source_file = b.path("data/unicode/auxiliary/WordBreakTest.txt") }); unicode_tests.root_module.addImport("Graphemes", graphemes); unicode_tests.root_module.addImport("Normalize", norm); unicode_tests.root_module.addImport("Words", words); diff --git a/codegen/canon.zig b/codegen/canon.zig index 89a9070..34e720f 100644 --- a/codegen/canon.zig +++ b/codegen/canon.zig @@ -6,13 +6,9 @@ pub fn main() anyerror!void { defer arena.deinit(); const allocator = arena.allocator(); - var line_buf: [4096]u8 = undefined; var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt - var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - + var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt")); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); @@ -24,7 +20,7 @@ pub fn main() anyerror!void { var writer = &file_writer.interface; const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.interface.takeDelimiterInclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig index 66eb62c..46b8e65 100644 --- a/codegen/case_prop.zig +++ b/codegen/case_prop.zig @@ -30,14 +30,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedCoreProperties.txt - var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/ccc.zig b/codegen/ccc.zig index b9b2bc3..48d3a9d 100644 --- a/codegen/ccc.zig +++ b/codegen/ccc.zig @@ -29,14 +29,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedCombiningClass.txt - var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); - defer cc_file.close(); - var cc_reader = cc_file.reader(&line_buf); - - while (cc_reader.interface.takeDelimiterExclusive('\n')) |line| { + var cc_reader = std.io.Reader.fixed(@embedFile("DerivedCombiningClass.txt")); + while (cc_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/compat.zig b/codegen/compat.zig index 835a636..0a06c44 100644 --- a/codegen/compat.zig +++ b/codegen/compat.zig @@ -7,12 +7,9 @@ pub fn main() anyerror!void { const allocator = arena.allocator(); // Process UnicodeData.txt - var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); - defer in_file.close(); - var line_buf: [4096]u8 = undefined; var write_buf: [4096]u8 = undefined; - var in_reader = in_file.reader(&line_buf); + var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt")); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); @@ -24,7 +21,7 @@ pub fn main() anyerror!void { const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/core_props.zig b/codegen/core_props.zig index 19063f6..d59a77e 100644 --- a/codegen/core_props.zig +++ b/codegen/core_props.zig @@ -30,14 +30,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedCoreProperties.txt - var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/dwp.zig b/codegen/dwp.zig index fad08d1..8189ad8 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig @@ -31,14 +31,10 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, i4).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedEastAsianWidth.txt - var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); - defer deaw_file.close(); - var deaw_reader = deaw_file.reader(&line_buf); + var deaw_reader = std.io.Reader.fixed(@embedFile("DerivedEastAsianWidth.txt")); - while (deaw_reader.interface.takeDelimiterExclusive('\n')) |line| { + while (deaw_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; // @missing ranges @@ -94,11 +90,9 @@ pub fn main() anyerror!void { }, } // Process DerivedGeneralCategory.txt - var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); - defer dgc_file.close(); - var dgc_reader = dgc_file.reader(&line_buf); + var dgc_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt")); - while (dgc_reader.interface.takeDelimiterExclusive('\n')) |line| { + while (dgc_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/fold.zig b/codegen/fold.zig index cc381a8..d927795 100644 --- a/codegen/fold.zig +++ b/codegen/fold.zig @@ -7,16 +7,13 @@ pub fn main() anyerror!void { defer std.debug.assert(gpa.deinit() == .ok); const allocator = gpa.allocator(); - var line_buf: [4096]u8 = undefined; // Process DerivedCoreProperties.txt - var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); - defer props_file.close(); - var props_reader = props_file.reader(&line_buf); + var props_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); var props_map = std.AutoHashMap(u21, void).init(allocator); defer props_map.deinit(); - props_lines: while (props_reader.interface.takeDelimiterExclusive('\n')) |line| { + props_lines: while (props_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -57,11 +54,9 @@ pub fn main() anyerror!void { defer codepoint_mapping.deinit(); // Process CaseFolding.txt - var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{}); - defer cp_file.close(); - var cp_reader = cp_file.reader(&line_buf); - while (cp_reader.interface.takeDelimiterExclusive('\n')) |line| { + var cp_reader = std.io.Reader.fixed(@embedFile("CaseFolding.txt")); + while (cp_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; var field_it = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/gbp.zig b/codegen/gbp.zig index d654cf1..895aa7a 100644 --- a/codegen/gbp.zig +++ b/codegen/gbp.zig @@ -61,14 +61,11 @@ pub fn main() anyerror!void { var emoji_set = std.AutoHashMap(u21, void).init(allocator); defer emoji_set.deinit(); - var line_buf: [4096]u8 = undefined; - // Process Indic - var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); - defer indic_file.close(); - var indic_reader = indic_file.reader(&line_buf); + const indic_file = @embedFile("DerivedCoreProperties.txt"); + var indic_reader = std.io.Reader.fixed(indic_file); - while (indic_reader.interface.takeDelimiterExclusive('\n')) |line| { + while (indic_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "InCB") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -106,11 +103,10 @@ pub fn main() anyerror!void { }, } // Process GBP - var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{}); - defer gbp_file.close(); - var gbp_reader = gbp_file.reader(&line_buf); - while (gbp_reader.interface.takeDelimiterExclusive('\n')) |line| { + var gbp_reader = std.io.Reader.fixed(@embedFile("GraphemeBreakProperty.txt")); + + while (gbp_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -147,11 +143,10 @@ pub fn main() anyerror!void { }, } // Process Emoji - var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{}); - defer emoji_file.close(); - var emoji_reader = emoji_file.reader(&line_buf); - while (emoji_reader.interface.takeDelimiterExclusive('\n')) |line| { + var emoji_reader = std.io.Reader.fixed(@embedFile("emoji-data.txt")); + + while (emoji_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/gencat.zig b/codegen/gencat.zig index 7dedb5d..79fa072 100644 --- a/codegen/gencat.zig +++ b/codegen/gencat.zig @@ -62,14 +62,9 @@ pub fn main() !void { var flat_map = std.AutoHashMap(u21, u5).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedGeneralCategory.txt - var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/hangul.zig b/codegen/hangul.zig index 6768b3f..64ab11c 100644 --- a/codegen/hangul.zig +++ b/codegen/hangul.zig @@ -38,14 +38,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u3).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process HangulSyllableType.txt - var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("HangulSyllableType.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/lower.zig b/codegen/lower.zig index c11cb0a..987f004 100644 --- a/codegen/lower.zig +++ b/codegen/lower.zig @@ -6,13 +6,9 @@ pub fn main() !void { defer arena.deinit(); const allocator = arena.allocator(); - var line_buf: [4096]u8 = undefined; var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt - var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - + var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt")); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); @@ -24,7 +20,7 @@ pub fn main() !void { const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/normp.zig b/codegen/normp.zig index 71a1ae5..3cdd770 100644 --- a/codegen/normp.zig +++ b/codegen/normp.zig @@ -29,14 +29,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u3).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedNormalizationProps.txt - var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedNormalizationProps.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/numeric.zig b/codegen/numeric.zig index cf918e8..d6b3165 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig @@ -30,14 +30,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process DerivedNumericType.txt - var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("DerivedNumericType.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/props.zig b/codegen/props.zig index 6ff0a33..ca42987 100644 --- a/codegen/props.zig +++ b/codegen/props.zig @@ -30,14 +30,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process PropList.txt - var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + var in_reader = std.io.Reader.fixed(@embedFile("PropList.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/scripts.zig b/codegen/scripts.zig index bdd3d9d..81511cc 100644 --- a/codegen/scripts.zig +++ b/codegen/scripts.zig @@ -203,14 +203,9 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u8).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - - // Process DerivedGeneralCategory.txt - var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + // Process Scripts.txt + var in_reader = std.io.Reader.fixed(@embedFile("Scripts.txt")); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/upper.zig b/codegen/upper.zig index 7d96a0e..108fdbd 100644 --- a/codegen/upper.zig +++ b/codegen/upper.zig @@ -6,13 +6,9 @@ pub fn main() anyerror!void { defer arena.deinit(); const allocator = arena.allocator(); - var line_buf: [4096]u8 = undefined; var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt - var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - + var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt")); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); @@ -23,7 +19,7 @@ pub fn main() anyerror!void { var file_writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -48,9 +44,6 @@ pub fn main() anyerror!void { } } } else |err| switch (err) { - error.ReadFailed => { - return in_reader.err orelse err; - }, error.EndOfStream => {}, else => { return err; diff --git a/codegen/wbp.zig b/codegen/wbp.zig index b1ed56a..dfdc32e 100644 --- a/codegen/wbp.zig +++ b/codegen/wbp.zig @@ -51,14 +51,10 @@ pub fn main() anyerror!void { var flat_map = std.AutoHashMap(u21, u5).init(allocator); defer flat_map.deinit(); - var line_buf: [4096]u8 = undefined; - // Process HangulSyllableType.txt - var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); - defer in_file.close(); - var in_reader = in_file.reader(&line_buf); - - while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { + const in_file = @embedFile("WordBreakProperty.txt"); + var in_reader = std.io.Reader.fixed(in_file); + while (in_reader.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index ff49b2a..875c5f0 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig @@ -8,14 +8,10 @@ test "Unicode normalization tests" { const n = try Normalize.init(allocator); defer n.deinit(allocator); - var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); - defer file.close(); - var buf: [4096]u8 = undefined; - var reader = file.reader(&buf); - + var reader = std.io.Reader.fixed(@embedFile("NormalizationTest.txt")); var cp_buf: [4]u8 = undefined; - var line_iter: IterRead = .{ .read = &reader.interface }; + var line_iter: IterRead = .{ .read = &reader }; while (line_iter.next()) |line| { // Iterate over fields. @@ -120,15 +116,12 @@ test "Unicode normalization tests" { test "Segmentation GraphemeIterator" { const allocator = std.testing.allocator; - var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{}); - defer file.close(); - var buf: [4096]u8 = undefined; - var reader = file.reader(&buf); + var reader = std.io.Reader.fixed(@embedFile("GraphemeBreakTest.txt")); const graph = try Graphemes.init(allocator); defer graph.deinit(allocator); - var line_iter: IterRead = .{ .read = &reader.interface }; + var line_iter: IterRead = .{ .read = &reader }; while (line_iter.next()) |raw| { // Clean up. @@ -263,15 +256,11 @@ test "Segmentation GraphemeIterator" { test "Segmentation Word Iterator" { const allocator = std.testing.allocator; - var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{}); - defer file.close(); - var buf: [4096]u8 = undefined; - var reader = file.reader(&buf); - + var reader = std.io.Reader.fixed(@embedFile("WordBreakTest.txt")); const wb = try Words.init(allocator); defer wb.deinit(allocator); - var line_iter: IterRead = .{ .read = &reader.interface }; + var line_iter: IterRead = .{ .read = &reader }; while (line_iter.next()) |raw| { // Clean up. -- cgit v1.2.3 From 2f02c3b16c073d0bd3d9368a66ce272a574f75a3 Mon Sep 17 00:00:00 2001 From: Jay Date: Sat, 8 Nov 2025 19:59:08 +1100 Subject: Use takeDelimiterInclusive to support Zig 0.15.2 --- codegen/canon.zig | 3 ++- codegen/case_prop.zig | 3 ++- codegen/ccc.zig | 3 ++- codegen/compat.zig | 3 ++- codegen/core_props.zig | 3 ++- codegen/dwp.zig | 6 ++++-- codegen/fold.zig | 6 ++++-- codegen/gbp.zig | 9 ++++++--- codegen/gencat.zig | 3 ++- codegen/hangul.zig | 3 ++- codegen/lower.zig | 3 ++- codegen/normp.zig | 3 ++- codegen/numeric.zig | 3 ++- codegen/props.zig | 3 ++- codegen/scripts.zig | 3 ++- codegen/upper.zig | 3 ++- codegen/wbp.zig | 3 ++- src/unicode_tests.zig | 3 ++- 18 files changed, 44 insertions(+), 22 deletions(-) diff --git a/codegen/canon.zig b/codegen/canon.zig index 34e720f..d95a905 100644 --- a/codegen/canon.zig +++ b/codegen/canon.zig @@ -20,7 +20,8 @@ pub fn main() anyerror!void { var writer = &file_writer.interface; const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.takeDelimiterInclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig index 46b8e65..613f7f6 100644 --- a/codegen/case_prop.zig +++ b/codegen/case_prop.zig @@ -32,7 +32,8 @@ pub fn main() anyerror!void { // Process DerivedCoreProperties.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/ccc.zig b/codegen/ccc.zig index 48d3a9d..4e470ae 100644 --- a/codegen/ccc.zig +++ b/codegen/ccc.zig @@ -31,7 +31,8 @@ pub fn main() anyerror!void { // Process DerivedCombiningClass.txt var cc_reader = std.io.Reader.fixed(@embedFile("DerivedCombiningClass.txt")); - while (cc_reader.takeDelimiterExclusive('\n')) |line| { + while (cc_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/compat.zig b/codegen/compat.zig index 0a06c44..debb83d 100644 --- a/codegen/compat.zig +++ b/codegen/compat.zig @@ -21,7 +21,8 @@ pub fn main() anyerror!void { const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/core_props.zig b/codegen/core_props.zig index d59a77e..6ffdf91 100644 --- a/codegen/core_props.zig +++ b/codegen/core_props.zig @@ -32,7 +32,8 @@ pub fn main() anyerror!void { // Process DerivedCoreProperties.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/dwp.zig b/codegen/dwp.zig index 8189ad8..75ac68e 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig @@ -34,7 +34,8 @@ pub fn main() anyerror!void { // Process DerivedEastAsianWidth.txt var deaw_reader = std.io.Reader.fixed(@embedFile("DerivedEastAsianWidth.txt")); - while (deaw_reader.takeDelimiterExclusive('\n')) |line| { + while (deaw_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; // @missing ranges @@ -92,7 +93,8 @@ pub fn main() anyerror!void { // Process DerivedGeneralCategory.txt var dgc_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt")); - while (dgc_reader.takeDelimiterExclusive('\n')) |line| { + while (dgc_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/fold.zig b/codegen/fold.zig index d927795..366ed79 100644 --- a/codegen/fold.zig +++ b/codegen/fold.zig @@ -13,7 +13,8 @@ pub fn main() anyerror!void { var props_map = std.AutoHashMap(u21, void).init(allocator); defer props_map.deinit(); - props_lines: while (props_reader.takeDelimiterExclusive('\n')) |line| { + props_lines: while (props_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -56,7 +57,8 @@ pub fn main() anyerror!void { // Process CaseFolding.txt var cp_reader = std.io.Reader.fixed(@embedFile("CaseFolding.txt")); - while (cp_reader.takeDelimiterExclusive('\n')) |line| { + while (cp_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; var field_it = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/gbp.zig b/codegen/gbp.zig index 895aa7a..1d06e9a 100644 --- a/codegen/gbp.zig +++ b/codegen/gbp.zig @@ -65,7 +65,8 @@ pub fn main() anyerror!void { const indic_file = @embedFile("DerivedCoreProperties.txt"); var indic_reader = std.io.Reader.fixed(indic_file); - while (indic_reader.takeDelimiterExclusive('\n')) |line| { + while (indic_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "InCB") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -106,7 +107,8 @@ pub fn main() anyerror!void { var gbp_reader = std.io.Reader.fixed(@embedFile("GraphemeBreakProperty.txt")); - while (gbp_reader.takeDelimiterExclusive('\n')) |line| { + while (gbp_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -146,7 +148,8 @@ pub fn main() anyerror!void { var emoji_reader = std.io.Reader.fixed(@embedFile("emoji-data.txt")); - while (emoji_reader.takeDelimiterExclusive('\n')) |line| { + while (emoji_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/gencat.zig b/codegen/gencat.zig index 79fa072..9800f1d 100644 --- a/codegen/gencat.zig +++ b/codegen/gencat.zig @@ -64,7 +64,8 @@ pub fn main() !void { // Process DerivedGeneralCategory.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/hangul.zig b/codegen/hangul.zig index 64ab11c..2e4c175 100644 --- a/codegen/hangul.zig +++ b/codegen/hangul.zig @@ -40,7 +40,8 @@ pub fn main() anyerror!void { // Process HangulSyllableType.txt var in_reader = std.io.Reader.fixed(@embedFile("HangulSyllableType.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/lower.zig b/codegen/lower.zig index 987f004..91f3ef2 100644 --- a/codegen/lower.zig +++ b/codegen/lower.zig @@ -20,7 +20,8 @@ pub fn main() !void { const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/normp.zig b/codegen/normp.zig index 3cdd770..eaf6989 100644 --- a/codegen/normp.zig +++ b/codegen/normp.zig @@ -31,7 +31,8 @@ pub fn main() anyerror!void { // Process DerivedNormalizationProps.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedNormalizationProps.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/numeric.zig b/codegen/numeric.zig index d6b3165..b304349 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig @@ -32,7 +32,8 @@ pub fn main() anyerror!void { // Process DerivedNumericType.txt var in_reader = std.io.Reader.fixed(@embedFile("DerivedNumericType.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/props.zig b/codegen/props.zig index ca42987..35c7dfb 100644 --- a/codegen/props.zig +++ b/codegen/props.zig @@ -32,7 +32,8 @@ pub fn main() anyerror!void { // Process PropList.txt var in_reader = std.io.Reader.fixed(@embedFile("PropList.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/scripts.zig b/codegen/scripts.zig index 81511cc..0f0194c 100644 --- a/codegen/scripts.zig +++ b/codegen/scripts.zig @@ -205,7 +205,8 @@ pub fn main() anyerror!void { // Process Scripts.txt var in_reader = std.io.Reader.fixed(@embedFile("Scripts.txt")); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/codegen/upper.zig b/codegen/upper.zig index 108fdbd..5eb29e7 100644 --- a/codegen/upper.zig +++ b/codegen/upper.zig @@ -19,7 +19,8 @@ pub fn main() anyerror!void { var file_writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - lines: while (in_reader.takeDelimiterExclusive('\n')) |line| { + lines: while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); diff --git a/codegen/wbp.zig b/codegen/wbp.zig index dfdc32e..33eeea5 100644 --- a/codegen/wbp.zig +++ b/codegen/wbp.zig @@ -54,7 +54,8 @@ pub fn main() anyerror!void { // Process HangulSyllableType.txt const in_file = @embedFile("WordBreakProperty.txt"); var in_reader = std.io.Reader.fixed(in_file); - while (in_reader.takeDelimiterExclusive('\n')) |line| { + while (in_reader.takeDelimiterInclusive('\n')) |took| { + const line = std.mem.trimRight(u8, took, "\n"); if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index 875c5f0..e2a5a96 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig @@ -449,7 +449,8 @@ const IterRead = struct { pub fn next(iter: *IterRead) anyerror![]const u8 { iter.line += 1; - const this_line = try iter.read.takeDelimiterExclusive('\n'); + const took = try iter.read.takeDelimiterInclusive('\n'); + const this_line = std.mem.trimRight(u8, took, "\n"); if (this_line.len == 0 or this_line[0] == '@' or this_line[0] == '#') { // comment, next line return iter.next(); -- cgit v1.2.3