From a3b5e884b12fdaa341010ef41bb9382fa0cd89f8 Mon Sep 17 00:00:00 2001 From: Michael Chaten Date: Sat, 13 Sep 2025 08:38:24 -0700 Subject: Update codebase to Zig 0.15.1. Removes compression support --- build.zig.zon | 4 +-- codegen/canon.zig | 24 +++++++------ codegen/case_prop.zig | 31 +++++++++-------- codegen/ccc.zig | 31 +++++++++-------- codegen/compat.zig | 27 ++++++++------- codegen/core_props.zig | 32 +++++++++-------- codegen/dwp.zig | 42 ++++++++++++---------- codegen/fold.zig | 59 ++++++++++++++++--------------- codegen/gbp.zig | 57 +++++++++++++++++------------- codegen/gencat.zig | 35 ++++++++++--------- codegen/hangul.zig | 32 +++++++++-------- codegen/lower.zig | 26 +++++++------- codegen/normp.zig | 32 +++++++++-------- codegen/numeric.zig | 31 +++++++++-------- codegen/props.zig | 32 +++++++++-------- codegen/scripts.zig | 38 ++++++++++---------- codegen/upper.zig | 31 ++++++++++------- codegen/wbp.zig | 32 +++++++++-------- src/CanonData.zig | 4 +-- src/CaseFolding.zig | 6 ++-- src/CombiningData.zig | 5 +-- src/CompatData.zig | 5 +-- src/DisplayWidth.zig | 8 ++--- src/GeneralCategories.zig | 4 +-- src/Graphemes.zig | 4 +-- src/HangulData.zig | 5 +-- src/LetterCasing.zig | 14 +++----- src/NormPropsData.zig | 5 +-- src/Normalize.zig | 10 +++--- src/Properties.zig | 10 ++---- src/Scripts.zig | 5 +-- src/Words.zig | 4 +-- src/unicode_tests.zig | 88 +++++++++++++++++++++++++---------------------- 33 files changed, 399 insertions(+), 374 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 3e1df95..e585449 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ .name = .zg, - .version = "0.14.1", - .minimum_zig_version = "0.14.0", + .version = "0.15.1", + .minimum_zig_version = "0.15.1", .fingerprint = 0x47df7778dc946aa0, .paths = .{ diff --git a/codegen/canon.zig b/codegen/canon.zig index 28b7f28..89a9070 100644 --- a/codegen/canon.zig +++ b/codegen/canon.zig @@ -1,32 +1,30 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); - + var file_writer = out_file.writer(&write_buf); + var writer = &file_writer.interface; const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterInclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -60,8 +58,12 @@ pub fn main() !void { try writer.writeInt(u8, @intCast(len), endian); for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u16, 0, endian); - try out_comp.flush(); + try writer.flush(); } diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig index 6c912a8..66eb62c 100644 --- a/codegen/case_prop.zig +++ b/codegen/case_prop.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedCoreProperties.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/ccc.zig b/codegen/ccc.zig index a01c8d2..b9b2bc3 100644 --- a/codegen/ccc.zig +++ b/codegen/ccc.zig @@ -21,7 +21,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -34,10 +34,9 @@ pub fn main() !void { // Process DerivedCombiningClass.txt var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); defer cc_file.close(); - var cc_buf = std.io.bufferedReader(cc_file.reader()); - const cc_reader = cc_buf.reader(); + var cc_reader = cc_file.reader(&line_buf); - while (try cc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (cc_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -68,15 +67,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -107,18 +111,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/compat.zig b/codegen/compat.zig index 07616fc..835a636 100644 --- a/codegen/compat.zig +++ b/codegen/compat.zig @@ -1,7 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -9,24 +9,22 @@ pub fn main() !void { // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -55,10 +53,15 @@ pub fn main() !void { } } - try writer.writeInt(u8, @intCast(len), endian); - for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); + try writer.interface.writeInt(u8, @intCast(len), endian); + for (cps[0..len]) |cp| try writer.interface.writeInt(u24, cp, endian); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u16, 0, endian); - try out_comp.flush(); + try writer.interface.writeInt(u16, 0, endian); + try writer.interface.flush(); } diff --git a/codegen/core_props.zig b/codegen/core_props.zig index f60c7a9..19063f6 100644 --- a/codegen/core_props.zig +++ b/codegen/core_props.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedCoreProperties.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -82,15 +81,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -121,18 +124,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var out_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&out_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/dwp.zig b/codegen/dwp.zig index 5e5bf6a..fad08d1 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig @@ -23,7 +23,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -36,10 +36,9 @@ pub fn main() !void { // Process DerivedEastAsianWidth.txt var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); defer deaw_file.close(); - var deaw_buf = std.io.bufferedReader(deaw_file.reader()); - const deaw_reader = deaw_buf.reader(); + var deaw_reader = deaw_file.reader(&line_buf); - while (try deaw_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (deaw_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; // @missing ranges @@ -88,15 +87,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process DerivedGeneralCategory.txt var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); defer dgc_file.close(); - var dgc_buf = std.io.bufferedReader(dgc_file.reader()); - const dgc_reader = dgc_buf.reader(); + var dgc_reader = dgc_file.reader(&line_buf); - while (try dgc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (dgc_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -139,15 +141,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(i4).init(allocator); + var stage2 = std.array_list.Managed(i4).init(allocator); defer stage2.deinit(); var block: Block = [_]i4{0} ** block_size; @@ -227,18 +234,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(i8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(i8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/fold.zig b/codegen/fold.zig index cb73cca..cc381a8 100644 --- a/codegen/fold.zig +++ b/codegen/fold.zig @@ -2,23 +2,21 @@ const std = @import("std"); const builtin = @import("builtin"); const mem = std.mem; -pub fn main() !void { +pub fn main() anyerror!void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer std.debug.assert(gpa.deinit() == .ok); const allocator = gpa.allocator(); + var line_buf: [4096]u8 = undefined; // Process DerivedCoreProperties.txt var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer props_file.close(); - var props_buf = std.io.bufferedReader(props_file.reader()); - const props_reader = props_buf.reader(); + var props_reader = props_file.reader(&line_buf); var props_map = std.AutoHashMap(u21, void).init(allocator); defer props_map.deinit(); - var line_buf: [4096]u8 = undefined; - - props_lines: while (try props_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + props_lines: while (props_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -49,18 +47,21 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator); defer codepoint_mapping.deinit(); // Process CaseFolding.txt var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{}); defer cp_file.close(); - var cp_buf = std.io.bufferedReader(cp_file.reader()); - const cp_reader = cp_buf.reader(); + var cp_reader = cp_file.reader(&line_buf); - while (try cp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (cp_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; var field_it = std.mem.splitScalar(u8, line, ';'); @@ -81,9 +82,14 @@ pub fn main() !void { } try codepoint_mapping.putNoClobber(codepoint, mapping_buf); + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var changes_when_casefolded_exceptions = std.ArrayList(u21).init(allocator); + var changes_when_casefolded_exceptions = std.array_list.Managed(u21).init(allocator); defer changes_when_casefolded_exceptions.deinit(); { @@ -221,32 +227,31 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); // Table metadata. - try writer.writeInt(u24, @intCast(codepoint_cutoff), endian); - try writer.writeInt(u24, @intCast(multiple_codepoint_start), endian); + try writer.interface.writeInt(u24, @intCast(codepoint_cutoff), endian); + try writer.interface.writeInt(u24, @intCast(multiple_codepoint_start), endian); // Stage 1 - try writer.writeInt(u16, @intCast(meaningful_stage1.len), endian); - try writer.writeAll(meaningful_stage1); + try writer.interface.writeInt(u16, @intCast(meaningful_stage1.len), endian); + try writer.interface.writeAll(meaningful_stage1); // Stage 2 - try writer.writeInt(u16, @intCast(stage2.len), endian); - try writer.writeAll(stage2); + try writer.interface.writeInt(u16, @intCast(stage2.len), endian); + try writer.interface.writeAll(stage2); // Stage 3 - try writer.writeInt(u16, @intCast(stage3.len), endian); - for (stage3) |offset| try writer.writeInt(i24, offset, endian); + try writer.interface.writeInt(u16, @intCast(stage3.len), endian); + for (stage3) |offset| try writer.interface.writeInt(i24, offset, endian); // Changes when case folded // Min and max - try writer.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); - try writer.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); - try writer.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); - for (changes_when_casefolded_exceptions.items) |cp| try writer.writeInt(u24, cp, endian); + try writer.interface.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); + try writer.interface.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); + try writer.interface.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); + for (changes_when_casefolded_exceptions.items) |cp| try writer.interface.writeInt(u24, cp, endian); - try out_comp.flush(); + try writer.interface.flush(); } } diff --git a/codegen/gbp.zig b/codegen/gbp.zig index 3fc4461..d654cf1 100644 --- a/codegen/gbp.zig +++ b/codegen/gbp.zig @@ -47,7 +47,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -66,10 +66,9 @@ pub fn main() !void { // Process Indic var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); defer indic_file.close(); - var indic_buf = std.io.bufferedReader(indic_file.reader()); - const indic_reader = indic_buf.reader(); + var indic_reader = indic_file.reader(&line_buf); - while (try indic_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (indic_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "InCB") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -100,15 +99,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process GBP var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{}); defer gbp_file.close(); - var gbp_buf = std.io.bufferedReader(gbp_file.reader()); - const gbp_reader = gbp_buf.reader(); + var gbp_reader = gbp_file.reader(&line_buf); - while (try gbp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (gbp_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -138,15 +140,18 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - // Process Emoji var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{}); defer emoji_file.close(); - var emoji_buf = std.io.bufferedReader(emoji_file.reader()); - const emoji_reader = emoji_buf.reader(); + var emoji_reader = emoji_file.reader(&line_buf); - while (try emoji_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (emoji_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -170,15 +175,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u16).init(allocator); + var stage2 = std.array_list.Managed(u16).init(allocator); defer stage2.deinit(); var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator); @@ -227,22 +237,21 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u16, i, endian); const props_bytes = stage3.keys(); - try writer.writeInt(u16, @intCast(props_bytes.len), endian); - try writer.writeAll(props_bytes); + try writer.interface.writeInt(u16, @intCast(props_bytes.len), endian); + try writer.interface.writeAll(props_bytes); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/gencat.zig b/codegen/gencat.zig index fe06bd7..7dedb5d 100644 --- a/codegen/gencat.zig +++ b/codegen/gencat.zig @@ -67,10 +67,9 @@ pub fn main() !void { // Process DerivedGeneralCategory.txt var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -101,18 +100,23 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u5).init(allocator); + var stage2 = std.array_list.Managed(u5).init(allocator); defer stage2.deinit(); - var stage3 = std.ArrayList(u5).init(allocator); + var stage3 = std.array_list.Managed(u5).init(allocator); defer stage3.deinit(); var block: Block = [_]u5{0} ** block_size; @@ -151,21 +155,20 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try writer.writeInt(u8, @intCast(stage3.items.len), endian); - for (stage3.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian); + for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/hangul.zig b/codegen/hangul.zig index 2c42bb7..6768b3f 100644 --- a/codegen/hangul.zig +++ b/codegen/hangul.zig @@ -30,7 +30,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -43,10 +43,9 @@ pub fn main() !void { // Process HangulSyllableType.txt var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -77,15 +76,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u3).init(allocator); + var stage2 = std.array_list.Managed(u3).init(allocator); defer stage2.deinit(); var block: Block = [_]u3{0} ** block_size; @@ -116,18 +119,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/lower.zig b/codegen/lower.zig index a053fe3..c11cb0a 100644 --- a/codegen/lower.zig +++ b/codegen/lower.zig @@ -6,27 +6,25 @@ pub fn main() !void { defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -42,16 +40,20 @@ pub fn main() !void { 13 => { // Simple lowercase mapping if (field.len == 0) continue :lines; - try writer.writeInt(i24, cp, endian); + try writer.interface.writeInt(i24, cp, endian); const mapping = try std.fmt.parseInt(i24, field, 16); - try writer.writeInt(i24, mapping - cp, endian); + try writer.interface.writeInt(i24, mapping - cp, endian); }, else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - - try writer.writeInt(u24, 0, endian); - try out_comp.flush(); + try writer.interface.writeInt(u24, 0, endian); + try writer.interface.flush(); } diff --git a/codegen/normp.zig b/codegen/normp.zig index 60dabdc..71a1ae5 100644 --- a/codegen/normp.zig +++ b/codegen/normp.zig @@ -21,7 +21,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -34,10 +34,9 @@ pub fn main() !void { // Process DerivedNormalizationProps.txt var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -78,15 +77,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u3).init(allocator); + var stage2 = std.array_list.Managed(u3).init(allocator); defer stage2.deinit(); var block: Block = [_]u3{0} ** block_size; @@ -117,18 +120,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/numeric.zig b/codegen/numeric.zig index 038ac0a..cf918e8 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process DerivedNumericType.txt var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/props.zig b/codegen/props.zig index 24b22e0..6ff0a33 100644 --- a/codegen/props.zig +++ b/codegen/props.zig @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -35,10 +35,9 @@ pub fn main() !void { // Process PropList.txt var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -79,15 +78,20 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -118,18 +122,16 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - try writer.writeAll(stage2.items); - - try out_comp.flush(); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + try writer.interface.writeAll(stage2.items); + try writer.interface.flush(); } diff --git a/codegen/scripts.zig b/codegen/scripts.zig index 530205d..bdd3d9d 100644 --- a/codegen/scripts.zig +++ b/codegen/scripts.zig @@ -195,7 +195,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -208,10 +208,9 @@ pub fn main() !void { // Process DerivedGeneralCategory.txt var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -245,18 +244,22 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u8).init(allocator); + var stage2 = std.array_list.Managed(u8).init(allocator); defer stage2.deinit(); - var stage3 = std.ArrayList(u8).init(allocator); + var stage3 = std.array_list.Managed(u8).init(allocator); defer stage3.deinit(); var block: Block = [_]u8{0} ** block_size; @@ -295,21 +298,20 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try writer.writeInt(u8, @intCast(stage3.items.len), endian); - for (stage3.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian); + for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/codegen/upper.zig b/codegen/upper.zig index 5848911..7d96a0e 100644 --- a/codegen/upper.zig +++ b/codegen/upper.zig @@ -1,32 +1,29 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); + var line_buf: [4096]u8 = undefined; + var write_buf: [4096]u8 = undefined; // Process UnicodeData.txt var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); var args_iter = try std.process.argsWithAllocator(allocator); defer args_iter.deinit(); _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); - + var file_writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - var line_buf: [4096]u8 = undefined; - lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0) continue; var field_iter = std.mem.splitScalar(u8, line, ';'); @@ -42,16 +39,24 @@ pub fn main() !void { 12 => { // Simple uppercase mapping if (field.len == 0) continue :lines; - try writer.writeInt(i24, cp, endian); + try file_writer.interface.writeInt(i24, cp, endian); const mapping = try std.fmt.parseInt(i24, field, 16); - try writer.writeInt(i24, mapping - cp, endian); + try file_writer.interface.writeInt(i24, mapping - cp, endian); }, else => {}, } } + } else |err| switch (err) { + error.ReadFailed => { + return in_reader.err orelse err; + }, + error.EndOfStream => {}, + else => { + return err; + }, } - try writer.writeInt(u24, 0, endian); - try out_comp.flush(); + try file_writer.interface.writeInt(u24, 0, endian); + try file_writer.interface.flush(); } diff --git a/codegen/wbp.zig b/codegen/wbp.zig index 741103e..b1ed56a 100644 --- a/codegen/wbp.zig +++ b/codegen/wbp.zig @@ -43,7 +43,7 @@ const BlockMap = std.HashMap( std.hash_map.default_max_load_percentage, ); -pub fn main() !void { +pub fn main() anyerror!void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -56,10 +56,9 @@ pub fn main() !void { // Process HangulSyllableType.txt var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); defer in_file.close(); - var in_buf = std.io.bufferedReader(in_file.reader()); - const in_reader = in_buf.reader(); + var in_reader = in_file.reader(&line_buf); - while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { + while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { if (line.len == 0 or line[0] == '#') continue; const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; @@ -90,15 +89,19 @@ pub fn main() !void { else => {}, } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } - var blocks_map = BlockMap.init(allocator); defer blocks_map.deinit(); - var stage1 = std.ArrayList(u16).init(allocator); + var stage1 = std.array_list.Managed(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(u5).init(allocator); + var stage2 = std.array_list.Managed(u5).init(allocator); defer stage2.deinit(); var block: Block = [_]u5{0} ** block_size; @@ -129,18 +132,17 @@ pub fn main() !void { _ = args_iter.skip(); const output_path = args_iter.next() orelse @panic("No output file arg!"); - const compressor = std.compress.flate.deflate.compressor; + var write_buf: [4096]u8 = undefined; var out_file = try std.fs.cwd().createFile(output_path, .{}); defer out_file.close(); - var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); - const writer = out_comp.writer(); + var writer = out_file.writer(&write_buf); const endian = builtin.cpu.arch.endian(); - try writer.writeInt(u16, @intCast(stage1.items.len), endian); - for (stage1.items) |i| try writer.writeInt(u16, i, endian); + try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); + for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); - try writer.writeInt(u16, @intCast(stage2.items.len), endian); - for (stage2.items) |i| try writer.writeInt(u8, i, endian); + try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); + for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); - try out_comp.flush(); + try writer.interface.flush(); } diff --git a/src/CanonData.zig b/src/CanonData.zig index 5d2332a..cf9dc8a 100644 --- a/src/CanonData.zig +++ b/src/CanonData.zig @@ -7,11 +7,9 @@ cps: []u21 = undefined, const CanonData = @This(); pub fn init(allocator: mem.Allocator) !CanonData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("canon"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var cdata = CanonData{ diff --git a/src/CaseFolding.zig b/src/CaseFolding.zig index ff41b3e..df86b92 100644 --- a/src/CaseFolding.zig +++ b/src/CaseFolding.zig @@ -48,11 +48,9 @@ fn setupImpl(casefold: *CaseFolding, allocator: Allocator) Allocator.Error!void } inline fn setupImplInner(casefold: *CaseFolding, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("fold"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -123,7 +121,7 @@ pub fn caseFoldAlloc( allocator: Allocator, cps: []const u21, ) Allocator.Error![]const u21 { - var cfcps = std.ArrayList(u21).init(allocator); + var cfcps = std.array_list.Managed(u21).init(allocator); defer cfcps.deinit(); var buf: [3]u21 = undefined; diff --git a/src/CombiningData.zig b/src/CombiningData.zig index fd64a3b..f58e0de 100644 --- a/src/CombiningData.zig +++ b/src/CombiningData.zig @@ -6,11 +6,9 @@ s2: []u8 = undefined, const CombiningData = @This(); pub fn init(allocator: mem.Allocator) !CombiningData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("ccc"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -46,5 +44,4 @@ pub fn isStarter(cbdata: CombiningData, cp: u21) bool { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; diff --git a/src/CompatData.zig b/src/CompatData.zig index 794abca..40ecd12 100644 --- a/src/CompatData.zig +++ b/src/CompatData.zig @@ -6,11 +6,9 @@ cps: []u21 = undefined, const CompatData = @This(); pub fn init(allocator: mem.Allocator) !CompatData { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("compat"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var cpdata = CompatData{ @@ -55,6 +53,5 @@ pub fn toNfkd(cpdata: *const CompatData, cp: u21) []u21 { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const magic = @import("magic"); diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig index 3da2d24..82b2649 100644 --- a/src/DisplayWidth.zig +++ b/src/DisplayWidth.zig @@ -39,11 +39,9 @@ pub fn setupWithGraphemes(dw: *DisplayWidth, allocator: Allocator, graphemes: Gr // Sets up the DisplayWidthData, leaving the GraphemeData undefined. pub fn setup(dw: *DisplayWidth, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("dwp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -400,7 +398,7 @@ pub fn wrap( columns: usize, threshold: usize, ) ![]u8 { - var result = ArrayList(u8).init(allocator); + var result = std.array_list.Managed(u8).init(allocator); defer result.deinit(); var line_iter = mem.tokenizeAny(u8, str, "\r\n"); @@ -460,8 +458,6 @@ test "allocation test" { const std = @import("std"); const builtin = @import("builtin"); const options = @import("options"); -const ArrayList = std.ArrayList; -const compress = std.compress; const mem = std.mem; const Allocator = mem.Allocator; const simd = std.simd; diff --git a/src/GeneralCategories.zig b/src/GeneralCategories.zig index 8c1b6a3..eee7e56 100644 --- a/src/GeneralCategories.zig +++ b/src/GeneralCategories.zig @@ -47,11 +47,9 @@ pub fn init(allocator: Allocator) Allocator.Error!GeneralCategories { } pub fn setup(gencat: *GeneralCategories, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("gencat"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/Graphemes.zig b/src/Graphemes.zig index f1c56ed..81d874c 100644 --- a/src/Graphemes.zig +++ b/src/Graphemes.zig @@ -16,11 +16,9 @@ pub fn init(allocator: Allocator) Allocator.Error!Graphemes { } pub fn setup(graphemes: *Graphemes, allocator: Allocator) Allocator.Error!void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("gbp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/HangulData.zig b/src/HangulData.zig index 8c5f3ad..cae8b97 100644 --- a/src/HangulData.zig +++ b/src/HangulData.zig @@ -15,11 +15,9 @@ s2: []u3 = undefined, const Hangul = @This(); pub fn init(allocator: mem.Allocator) !Hangul { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("hangul"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var hangul = Hangul{}; @@ -49,6 +47,5 @@ pub fn syllable(hangul: *const Hangul, cp: u21) Syllable { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const testing = std.testing; diff --git a/src/LetterCasing.zig b/src/LetterCasing.zig index 11a3e96..33096fc 100644 --- a/src/LetterCasing.zig +++ b/src/LetterCasing.zig @@ -22,7 +22,6 @@ pub fn setup(case: *LetterCasing, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const endian = builtin.cpu.arch.endian(); self.case_map = try allocator.alloc([2]u21, 0x110000); @@ -36,8 +35,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Uppercase const upper_bytes = @embedFile("upper"); var upper_fbs = std.io.fixedBufferStream(upper_bytes); - var upper_decomp = decompressor(.raw, upper_fbs.reader()); - var upper_reader = upper_decomp.reader(); + var upper_reader = upper_fbs.reader(); while (true) { const cp = try upper_reader.readInt(i24, endian); @@ -49,8 +47,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Lowercase const lower_bytes = @embedFile("lower"); var lower_fbs = std.io.fixedBufferStream(lower_bytes); - var lower_decomp = decompressor(.raw, lower_fbs.reader()); - var lower_reader = lower_decomp.reader(); + var lower_reader = lower_fbs.reader(); while (true) { const cp = try lower_reader.readInt(i24, endian); @@ -62,8 +59,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { // Case properties const cp_bytes = @embedFile("case_prop"); var cp_fbs = std.io.fixedBufferStream(cp_bytes); - var cp_decomp = decompressor(.raw, cp_fbs.reader()); - var cp_reader = cp_decomp.reader(); + var cp_reader = cp_fbs.reader(); const stage_1_len: u16 = try cp_reader.readInt(u16, endian); self.prop_s1 = try allocator.alloc(u16, stage_1_len); @@ -122,7 +118,7 @@ pub fn toUpperStr( allocator: mem.Allocator, str: []const u8, ) ![]u8 { - var bytes = std.ArrayList(u8).init(allocator); + var bytes = std.array_list.Managed(u8).init(allocator); defer bytes.deinit(); var iter = CodePointIterator{ .bytes = str }; @@ -180,7 +176,7 @@ pub fn toLowerStr( allocator: mem.Allocator, str: []const u8, ) ![]u8 { - var bytes = std.ArrayList(u8).init(allocator); + var bytes = std.array_list.Managed(u8).init(allocator); defer bytes.deinit(); var iter = CodePointIterator{ .bytes = str }; diff --git a/src/NormPropsData.zig b/src/NormPropsData.zig index ca69569..7b53542 100644 --- a/src/NormPropsData.zig +++ b/src/NormPropsData.zig @@ -6,11 +6,9 @@ s2: []u4 = undefined, const NormProps = @This(); pub fn init(allocator: mem.Allocator) !NormProps { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("normp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); var norms = NormProps{}; @@ -50,6 +48,5 @@ pub fn isFcx(norms: *const NormProps, cp: u21) bool { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const testing = std.testing; diff --git a/src/Normalize.zig b/src/Normalize.zig index 989ec29..4a1bae8 100644 --- a/src/Normalize.zig +++ b/src/Normalize.zig @@ -305,7 +305,7 @@ pub fn nfkd(self: Normalize, allocator: Allocator, str: []const u8) Allocator.Er } pub fn nfxdCodePoints(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var cp_iter = CodePointIterator{ .bytes = str }; @@ -332,7 +332,7 @@ fn nfxd(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo const dcps = try self.nfxdCodePoints(allocator, str, form); defer allocator.free(dcps); - var dstr_list = std.ArrayList(u8).init(allocator); + var dstr_list = std.array_list.Managed(u8).init(allocator); defer dstr_list.deinit(); var buf: [4]u8 = undefined; @@ -393,7 +393,7 @@ pub fn nfdCodePoints( allocator: Allocator, cps: []const u21, ) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var dc_buf: [18]u21 = undefined; @@ -418,7 +418,7 @@ pub fn nfkdCodePoints( allocator: Allocator, cps: []const u21, ) Allocator.Error![]u21 { - var dcp_list = std.ArrayList(u21).init(allocator); + var dcp_list = std.array_list.Managed(u21).init(allocator); defer dcp_list.deinit(); var dc_buf: [18]u21 = undefined; @@ -560,7 +560,7 @@ fn nfxc(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo // If we have no deletions. the code point sequence // has been fully composed. if (deleted == 0) { - var cstr_list = std.ArrayList(u8).init(allocator); + var cstr_list = std.array_list.Managed(u8).init(allocator); defer cstr_list.deinit(); var buf: [4]u8 = undefined; diff --git a/src/Properties.zig b/src/Properties.zig index 73602a0..432d176 100644 --- a/src/Properties.zig +++ b/src/Properties.zig @@ -25,14 +25,12 @@ pub fn setup(props: *Properties, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(props: *Properties, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const endian = builtin.cpu.arch.endian(); // Process DerivedCoreProperties.txt const core_bytes = @embedFile("core_props"); var core_fbs = std.io.fixedBufferStream(core_bytes); - var core_decomp = decompressor(.raw, core_fbs.reader()); - var core_reader = core_decomp.reader(); + var core_reader = core_fbs.reader(); const core_stage_1_len: u16 = try core_reader.readInt(u16, endian); props.core_s1 = try allocator.alloc(u16, core_stage_1_len); @@ -47,8 +45,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void { // Process PropList.txt const props_bytes = @embedFile("props"); var props_fbs = std.io.fixedBufferStream(props_bytes); - var props_decomp = decompressor(.raw, props_fbs.reader()); - var props_reader = props_decomp.reader(); + var props_reader = props_fbs.reader(); const stage_1_len: u16 = try props_reader.readInt(u16, endian); props.props_s1 = try allocator.alloc(u16, stage_1_len); @@ -63,8 +60,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void { // Process DerivedNumericType.txt const num_bytes = @embedFile("numeric"); var num_fbs = std.io.fixedBufferStream(num_bytes); - var num_decomp = decompressor(.raw, num_fbs.reader()); - var num_reader = num_decomp.reader(); + var num_reader = num_fbs.reader(); const num_stage_1_len: u16 = try num_reader.readInt(u16, endian); props.num_s1 = try allocator.alloc(u16, num_stage_1_len); diff --git a/src/Scripts.zig b/src/Scripts.zig index 3bc90bc..719b01f 100644 --- a/src/Scripts.zig +++ b/src/Scripts.zig @@ -196,11 +196,9 @@ pub fn setup(scripts: *Scripts, allocator: Allocator) Allocator.Error!void { } inline fn setupInner(scripts: *Scripts, allocator: mem.Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("scripts"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); @@ -250,7 +248,6 @@ test "Allocation failure" { const std = @import("std"); const builtin = @import("builtin"); -const compress = std.compress; const mem = std.mem; const Allocator = mem.Allocator; const testing = std.testing; diff --git a/src/Words.zig b/src/Words.zig index 617c34d..ce3203f 100644 --- a/src/Words.zig +++ b/src/Words.zig @@ -605,11 +605,9 @@ const SneakIterator = struct { }; inline fn setupImpl(wb: *Words, allocator: Allocator) !void { - const decompressor = compress.flate.inflate.decompressor; const in_bytes = @embedFile("wbp"); var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); + var reader = in_fbs.reader(); const endian = builtin.cpu.arch.endian(); diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index ae177a9..ff49b2a 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig @@ -3,35 +3,34 @@ const dbg_print = false; test "Unicode normalization tests" { var arena = heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); - var allocator = arena.allocator(); + const allocator = arena.allocator(); const n = try Normalize.init(allocator); defer n.deinit(allocator); var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); defer file.close(); - var buf_reader = io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); - var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); + var cp_buf: [4]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |line| { + while (line_iter.next()) |line| { // Iterate over fields. var fields = mem.splitScalar(u8, line, ';'); var field_index: usize = 0; var input: []u8 = undefined; - defer allocator.free(input); - + if (dbg_print) std.debug.print("Line: {s}\n", .{line}); while (fields.next()) |field| : (field_index += 1) { if (field_index == 0) { - var i_buf = std.ArrayList(u8).init(allocator); + var i_buf = std.array_list.Managed(u8).init(allocator); defer i_buf.deinit(); var i_fields = mem.splitScalar(u8, field, ' '); while (i_fields.next()) |s| { + if (dbg_print) std.debug.print("Debug: {s}\n", .{s}); const icp = try fmt.parseInt(u21, s, 16); const len = try unicode.utf8Encode(icp, &cp_buf); try i_buf.appendSlice(cp_buf[0..len]); @@ -41,7 +40,7 @@ test "Unicode normalization tests" { } else if (field_index == 1) { if (dbg_print) debug.print("\n*** {s} ***\n", .{line}); // NFC, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -58,7 +57,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 2) { // NFD, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -75,7 +74,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 3) { // NFKC, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -92,7 +91,7 @@ test "Unicode normalization tests" { try testing.expectEqualStrings(want, got.slice); } else if (field_index == 4) { // NFKD, time to test. - var w_buf = std.ArrayList(u8).init(allocator); + var w_buf = std.array_list.Managed(u8).init(allocator); defer w_buf.deinit(); var w_fields = mem.splitScalar(u8, field, ' '); @@ -111,6 +110,11 @@ test "Unicode normalization tests" { continue; } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } @@ -118,26 +122,25 @@ test "Segmentation GraphemeIterator" { const allocator = std.testing.allocator; var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{}); defer file.close(); - var buf_reader = std.io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); + var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); const graph = try Graphemes.init(allocator); defer graph.deinit(allocator); - var buf: [4096]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |raw| { + while (line_iter.next()) |raw| { // Clean up. var line = std.mem.trimLeft(u8, raw, "÷ "); if (std.mem.indexOf(u8, line, " ÷\t")) |final| { line = line[0..final]; } // Iterate over fields. - var want = std.ArrayList(Grapheme).init(allocator); + var want = std.array_list.Managed(Grapheme).init(allocator); defer want.deinit(); - var all_bytes = std.ArrayList(u8).init(allocator); + var all_bytes = std.array_list.Managed(u8).init(allocator); defer all_bytes.deinit(); var graphemes = std.mem.splitSequence(u8, line, " ÷ "); @@ -250,6 +253,11 @@ test "Segmentation GraphemeIterator" { } } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } @@ -257,26 +265,25 @@ test "Segmentation Word Iterator" { const allocator = std.testing.allocator; var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{}); defer file.close(); - var buf_reader = std.io.bufferedReader(file.reader()); - var input_stream = buf_reader.reader(); + var buf: [4096]u8 = undefined; + var reader = file.reader(&buf); const wb = try Words.init(allocator); defer wb.deinit(allocator); - var buf: [4096]u8 = undefined; - var line_iter: IterRead = .{ .read = &input_stream }; + var line_iter: IterRead = .{ .read = &reader.interface }; - while (try line_iter.next(&buf)) |raw| { + while (line_iter.next()) |raw| { // Clean up. var line = std.mem.trimLeft(u8, raw, "÷ "); if (std.mem.indexOf(u8, line, " ÷\t")) |final| { line = line[0..final]; } // Iterate over fields. - var want = std.ArrayList(Word).init(allocator); + var want = std.array_list.Managed(Word).init(allocator); defer want.deinit(); - var all_bytes = std.ArrayList(u8).init(allocator); + var all_bytes = std.array_list.Managed(u8).init(allocator); defer all_bytes.deinit(); var words = std.mem.splitSequence(u8, line, " ÷ "); @@ -439,26 +446,26 @@ test "Segmentation Word Iterator" { if (idx == 0) break; } } + } else |err| switch (err) { + error.EndOfStream => {}, + else => { + return err; + }, } } const IterRead = struct { - read: *Reader, + read: *io.Reader, line: usize = 0, - pub fn next(iter: *IterRead, buf: []u8) !?[]const u8 { - defer iter.line += 1; - const maybe_line = try iter.read.readUntilDelimiterOrEof(buf, '#'); - if (maybe_line) |this_line| { - try iter.read.skipUntilDelimiterOrEof('\n'); - if (this_line.len == 0 or this_line[0] == '@') { - // comment, next line - return iter.next(buf); - } else { - return this_line; - } + pub fn next(iter: *IterRead) anyerror![]const u8 { + iter.line += 1; + const this_line = try iter.read.takeDelimiterExclusive('\n'); + if (this_line.len == 0 or this_line[0] == '@' or this_line[0] == '#') { + // comment, next line + return iter.next(); } else { - return null; + return this_line; } } }; @@ -467,7 +474,6 @@ const std = @import("std"); const fmt = std.fmt; const fs = std.fs; const io = std.io; -const Reader = io.BufferedReader(4096, fs.File.Reader).Reader; const heap = std.heap; const mem = std.mem; const debug = std.debug; -- cgit v1.2.3