diff options
| author | 2025-09-13 08:38:24 -0700 | |
|---|---|---|
| committer | 2025-09-14 01:48:41 -0700 | |
| commit | a3b5e884b12fdaa341010ef41bb9382fa0cd89f8 (patch) | |
| tree | 8bdd22c1c54da0b56ef6b9fe0dc4c2ca667b71a3 | |
| parent | Update build to 0.15.0 (diff) | |
| download | zg-a3b5e884b12fdaa341010ef41bb9382fa0cd89f8.tar.gz zg-a3b5e884b12fdaa341010ef41bb9382fa0cd89f8.tar.xz zg-a3b5e884b12fdaa341010ef41bb9382fa0cd89f8.zip | |
Update codebase to Zig 0.15.1.
Removes compression support
33 files changed, 399 insertions, 374 deletions
diff --git a/build.zig.zon b/build.zig.zon index 3e1df95..e585449 100644 --- a/build.zig.zon +++ b/build.zig.zon | |||
| @@ -1,7 +1,7 @@ | |||
| 1 | .{ | 1 | .{ |
| 2 | .name = .zg, | 2 | .name = .zg, |
| 3 | .version = "0.14.1", | 3 | .version = "0.15.1", |
| 4 | .minimum_zig_version = "0.14.0", | 4 | .minimum_zig_version = "0.15.1", |
| 5 | .fingerprint = 0x47df7778dc946aa0, | 5 | .fingerprint = 0x47df7778dc946aa0, |
| 6 | 6 | ||
| 7 | .paths = .{ | 7 | .paths = .{ |
diff --git a/codegen/canon.zig b/codegen/canon.zig index 28b7f28..89a9070 100644 --- a/codegen/canon.zig +++ b/codegen/canon.zig | |||
| @@ -1,32 +1,30 @@ | |||
| 1 | const std = @import("std"); | 1 | const std = @import("std"); |
| 2 | const builtin = @import("builtin"); | 2 | const builtin = @import("builtin"); |
| 3 | 3 | ||
| 4 | pub fn main() !void { | 4 | pub fn main() anyerror!void { |
| 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 6 | defer arena.deinit(); | 6 | defer arena.deinit(); |
| 7 | const allocator = arena.allocator(); | 7 | const allocator = arena.allocator(); |
| 8 | 8 | ||
| 9 | var line_buf: [4096]u8 = undefined; | ||
| 10 | var write_buf: [4096]u8 = undefined; | ||
| 9 | // Process UnicodeData.txt | 11 | // Process UnicodeData.txt |
| 10 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); | 12 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); |
| 11 | defer in_file.close(); | 13 | defer in_file.close(); |
| 12 | var in_buf = std.io.bufferedReader(in_file.reader()); | 14 | var in_reader = in_file.reader(&line_buf); |
| 13 | const in_reader = in_buf.reader(); | ||
| 14 | 15 | ||
| 15 | var args_iter = try std.process.argsWithAllocator(allocator); | 16 | var args_iter = try std.process.argsWithAllocator(allocator); |
| 16 | defer args_iter.deinit(); | 17 | defer args_iter.deinit(); |
| 17 | _ = args_iter.skip(); | 18 | _ = args_iter.skip(); |
| 18 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 19 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 19 | 20 | ||
| 20 | const compressor = std.compress.flate.deflate.compressor; | ||
| 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 22 | defer out_file.close(); | 22 | defer out_file.close(); |
| 23 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 23 | var file_writer = out_file.writer(&write_buf); |
| 24 | const writer = out_comp.writer(); | 24 | var writer = &file_writer.interface; |
| 25 | |||
| 26 | const endian = builtin.cpu.arch.endian(); | 25 | const endian = builtin.cpu.arch.endian(); |
| 27 | var line_buf: [4096]u8 = undefined; | ||
| 28 | 26 | ||
| 29 | lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 27 | lines: while (in_reader.interface.takeDelimiterInclusive('\n')) |line| { |
| 30 | if (line.len == 0) continue; | 28 | if (line.len == 0) continue; |
| 31 | 29 | ||
| 32 | var field_iter = std.mem.splitScalar(u8, line, ';'); | 30 | var field_iter = std.mem.splitScalar(u8, line, ';'); |
| @@ -60,8 +58,12 @@ pub fn main() !void { | |||
| 60 | 58 | ||
| 61 | try writer.writeInt(u8, @intCast(len), endian); | 59 | try writer.writeInt(u8, @intCast(len), endian); |
| 62 | for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); | 60 | for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); |
| 61 | } else |err| switch (err) { | ||
| 62 | error.EndOfStream => {}, | ||
| 63 | else => { | ||
| 64 | return err; | ||
| 65 | }, | ||
| 63 | } | 66 | } |
| 64 | |||
| 65 | try writer.writeInt(u16, 0, endian); | 67 | try writer.writeInt(u16, 0, endian); |
| 66 | try out_comp.flush(); | 68 | try writer.flush(); |
| 67 | } | 69 | } |
diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig index 6c912a8..66eb62c 100644 --- a/codegen/case_prop.zig +++ b/codegen/case_prop.zig | |||
| @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( | |||
| 22 | std.hash_map.default_max_load_percentage, | 22 | std.hash_map.default_max_load_percentage, |
| 23 | ); | 23 | ); |
| 24 | 24 | ||
| 25 | pub fn main() !void { | 25 | pub fn main() anyerror!void { |
| 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 27 | defer arena.deinit(); | 27 | defer arena.deinit(); |
| 28 | const allocator = arena.allocator(); | 28 | const allocator = arena.allocator(); |
| @@ -35,10 +35,9 @@ pub fn main() !void { | |||
| 35 | // Process DerivedCoreProperties.txt | 35 | // Process DerivedCoreProperties.txt |
| 36 | var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); | 36 | var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); |
| 37 | defer in_file.close(); | 37 | defer in_file.close(); |
| 38 | var in_buf = std.io.bufferedReader(in_file.reader()); | 38 | var in_reader = in_file.reader(&line_buf); |
| 39 | const in_reader = in_buf.reader(); | ||
| 40 | 39 | ||
| 41 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 40 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 42 | if (line.len == 0 or line[0] == '#') continue; | 41 | if (line.len == 0 or line[0] == '#') continue; |
| 43 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 42 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| 44 | 43 | ||
| @@ -79,15 +78,20 @@ pub fn main() !void { | |||
| 79 | else => {}, | 78 | else => {}, |
| 80 | } | 79 | } |
| 81 | } | 80 | } |
| 81 | } else |err| switch (err) { | ||
| 82 | error.EndOfStream => {}, | ||
| 83 | else => { | ||
| 84 | return err; | ||
| 85 | }, | ||
| 82 | } | 86 | } |
| 83 | 87 | ||
| 84 | var blocks_map = BlockMap.init(allocator); | 88 | var blocks_map = BlockMap.init(allocator); |
| 85 | defer blocks_map.deinit(); | 89 | defer blocks_map.deinit(); |
| 86 | 90 | ||
| 87 | var stage1 = std.ArrayList(u16).init(allocator); | 91 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 88 | defer stage1.deinit(); | 92 | defer stage1.deinit(); |
| 89 | 93 | ||
| 90 | var stage2 = std.ArrayList(u8).init(allocator); | 94 | var stage2 = std.array_list.Managed(u8).init(allocator); |
| 91 | defer stage2.deinit(); | 95 | defer stage2.deinit(); |
| 92 | 96 | ||
| 93 | var block: Block = [_]u8{0} ** block_size; | 97 | var block: Block = [_]u8{0} ** block_size; |
| @@ -118,18 +122,17 @@ pub fn main() !void { | |||
| 118 | _ = args_iter.skip(); | 122 | _ = args_iter.skip(); |
| 119 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 123 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 120 | 124 | ||
| 121 | const compressor = std.compress.flate.deflate.compressor; | 125 | var write_buf: [4096]u8 = undefined; |
| 122 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 126 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 123 | defer out_file.close(); | 127 | defer out_file.close(); |
| 124 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 128 | var writer = out_file.writer(&write_buf); |
| 125 | const writer = out_comp.writer(); | ||
| 126 | 129 | ||
| 127 | const endian = builtin.cpu.arch.endian(); | 130 | const endian = builtin.cpu.arch.endian(); |
| 128 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 131 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 129 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 132 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 130 | 133 | ||
| 131 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 134 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 132 | try writer.writeAll(stage2.items); | 135 | try writer.interface.writeAll(stage2.items); |
| 133 | 136 | ||
| 134 | try out_comp.flush(); | 137 | try writer.interface.flush(); |
| 135 | } | 138 | } |
diff --git a/codegen/ccc.zig b/codegen/ccc.zig index a01c8d2..b9b2bc3 100644 --- a/codegen/ccc.zig +++ b/codegen/ccc.zig | |||
| @@ -21,7 +21,7 @@ const BlockMap = std.HashMap( | |||
| 21 | std.hash_map.default_max_load_percentage, | 21 | std.hash_map.default_max_load_percentage, |
| 22 | ); | 22 | ); |
| 23 | 23 | ||
| 24 | pub fn main() !void { | 24 | pub fn main() anyerror!void { |
| 25 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 25 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 26 | defer arena.deinit(); | 26 | defer arena.deinit(); |
| 27 | const allocator = arena.allocator(); | 27 | const allocator = arena.allocator(); |
| @@ -34,10 +34,9 @@ pub fn main() !void { | |||
| 34 | // Process DerivedCombiningClass.txt | 34 | // Process DerivedCombiningClass.txt |
| 35 | var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); | 35 | var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); |
| 36 | defer cc_file.close(); | 36 | defer cc_file.close(); |
| 37 | var cc_buf = std.io.bufferedReader(cc_file.reader()); | 37 | var cc_reader = cc_file.reader(&line_buf); |
| 38 | const cc_reader = cc_buf.reader(); | ||
| 39 | 38 | ||
| 40 | while (try cc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 39 | while (cc_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 41 | if (line.len == 0 or line[0] == '#') continue; | 40 | if (line.len == 0 or line[0] == '#') continue; |
| 42 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 41 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| 43 | 42 | ||
| @@ -68,15 +67,20 @@ pub fn main() !void { | |||
| 68 | else => {}, | 67 | else => {}, |
| 69 | } | 68 | } |
| 70 | } | 69 | } |
| 70 | } else |err| switch (err) { | ||
| 71 | error.EndOfStream => {}, | ||
| 72 | else => { | ||
| 73 | return err; | ||
| 74 | }, | ||
| 71 | } | 75 | } |
| 72 | 76 | ||
| 73 | var blocks_map = BlockMap.init(allocator); | 77 | var blocks_map = BlockMap.init(allocator); |
| 74 | defer blocks_map.deinit(); | 78 | defer blocks_map.deinit(); |
| 75 | 79 | ||
| 76 | var stage1 = std.ArrayList(u16).init(allocator); | 80 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 77 | defer stage1.deinit(); | 81 | defer stage1.deinit(); |
| 78 | 82 | ||
| 79 | var stage2 = std.ArrayList(u8).init(allocator); | 83 | var stage2 = std.array_list.Managed(u8).init(allocator); |
| 80 | defer stage2.deinit(); | 84 | defer stage2.deinit(); |
| 81 | 85 | ||
| 82 | var block: Block = [_]u8{0} ** block_size; | 86 | var block: Block = [_]u8{0} ** block_size; |
| @@ -107,18 +111,17 @@ pub fn main() !void { | |||
| 107 | _ = args_iter.skip(); | 111 | _ = args_iter.skip(); |
| 108 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 112 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 109 | 113 | ||
| 110 | const compressor = std.compress.flate.deflate.compressor; | 114 | var write_buf: [4096]u8 = undefined; |
| 111 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 115 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 112 | defer out_file.close(); | 116 | defer out_file.close(); |
| 113 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 117 | var writer = out_file.writer(&write_buf); |
| 114 | const writer = out_comp.writer(); | ||
| 115 | 118 | ||
| 116 | const endian = builtin.cpu.arch.endian(); | 119 | const endian = builtin.cpu.arch.endian(); |
| 117 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 120 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 118 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 121 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 119 | 122 | ||
| 120 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 123 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 121 | try writer.writeAll(stage2.items); | 124 | try writer.interface.writeAll(stage2.items); |
| 122 | 125 | ||
| 123 | try out_comp.flush(); | 126 | try writer.interface.flush(); |
| 124 | } | 127 | } |
diff --git a/codegen/compat.zig b/codegen/compat.zig index 07616fc..835a636 100644 --- a/codegen/compat.zig +++ b/codegen/compat.zig | |||
| @@ -1,7 +1,7 @@ | |||
| 1 | const std = @import("std"); | 1 | const std = @import("std"); |
| 2 | const builtin = @import("builtin"); | 2 | const builtin = @import("builtin"); |
| 3 | 3 | ||
| 4 | pub fn main() !void { | 4 | pub fn main() anyerror!void { |
| 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 6 | defer arena.deinit(); | 6 | defer arena.deinit(); |
| 7 | const allocator = arena.allocator(); | 7 | const allocator = arena.allocator(); |
| @@ -9,24 +9,22 @@ pub fn main() !void { | |||
| 9 | // Process UnicodeData.txt | 9 | // Process UnicodeData.txt |
| 10 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); | 10 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); |
| 11 | defer in_file.close(); | 11 | defer in_file.close(); |
| 12 | var in_buf = std.io.bufferedReader(in_file.reader()); | 12 | var line_buf: [4096]u8 = undefined; |
| 13 | const in_reader = in_buf.reader(); | 13 | var write_buf: [4096]u8 = undefined; |
| 14 | var in_reader = in_file.reader(&line_buf); | ||
| 14 | 15 | ||
| 15 | var args_iter = try std.process.argsWithAllocator(allocator); | 16 | var args_iter = try std.process.argsWithAllocator(allocator); |
| 16 | defer args_iter.deinit(); | 17 | defer args_iter.deinit(); |
| 17 | _ = args_iter.skip(); | 18 | _ = args_iter.skip(); |
| 18 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 19 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 19 | 20 | ||
| 20 | const compressor = std.compress.flate.deflate.compressor; | ||
| 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 22 | defer out_file.close(); | 22 | defer out_file.close(); |
| 23 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 23 | var writer = out_file.writer(&write_buf); |
| 24 | const writer = out_comp.writer(); | ||
| 25 | 24 | ||
| 26 | const endian = builtin.cpu.arch.endian(); | 25 | const endian = builtin.cpu.arch.endian(); |
| 27 | var line_buf: [4096]u8 = undefined; | ||
| 28 | 26 | ||
| 29 | lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 27 | lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 30 | if (line.len == 0) continue; | 28 | if (line.len == 0) continue; |
| 31 | 29 | ||
| 32 | var field_iter = std.mem.splitScalar(u8, line, ';'); | 30 | var field_iter = std.mem.splitScalar(u8, line, ';'); |
| @@ -55,10 +53,15 @@ pub fn main() !void { | |||
| 55 | } | 53 | } |
| 56 | } | 54 | } |
| 57 | 55 | ||
| 58 | try writer.writeInt(u8, @intCast(len), endian); | 56 | try writer.interface.writeInt(u8, @intCast(len), endian); |
| 59 | for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); | 57 | for (cps[0..len]) |cp| try writer.interface.writeInt(u24, cp, endian); |
| 58 | } else |err| switch (err) { | ||
| 59 | error.EndOfStream => {}, | ||
| 60 | else => { | ||
| 61 | return err; | ||
| 62 | }, | ||
| 60 | } | 63 | } |
| 61 | 64 | ||
| 62 | try writer.writeInt(u16, 0, endian); | 65 | try writer.interface.writeInt(u16, 0, endian); |
| 63 | try out_comp.flush(); | 66 | try writer.interface.flush(); |
| 64 | } | 67 | } |
diff --git a/codegen/core_props.zig b/codegen/core_props.zig index f60c7a9..19063f6 100644 --- a/codegen/core_props.zig +++ b/codegen/core_props.zig | |||
| @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( | |||
| 22 | std.hash_map.default_max_load_percentage, | 22 | std.hash_map.default_max_load_percentage, |
| 23 | ); | 23 | ); |
| 24 | 24 | ||
| 25 | pub fn main() !void { | 25 | pub fn main() anyerror!void { |
| 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 27 | defer arena.deinit(); | 27 | defer arena.deinit(); |
| 28 | const allocator = arena.allocator(); | 28 | const allocator = arena.allocator(); |
| @@ -35,10 +35,9 @@ pub fn main() !void { | |||
| 35 | // Process DerivedCoreProperties.txt | 35 | // Process DerivedCoreProperties.txt |
| 36 | var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); | 36 | var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); |
| 37 | defer in_file.close(); | 37 | defer in_file.close(); |
| 38 | var in_buf = std.io.bufferedReader(in_file.reader()); | 38 | var in_reader = in_file.reader(&line_buf); |
| 39 | const in_reader = in_buf.reader(); | ||
| 40 | 39 | ||
| 41 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 40 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 42 | if (line.len == 0 or line[0] == '#') continue; | 41 | if (line.len == 0 or line[0] == '#') continue; |
| 43 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 42 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| 44 | 43 | ||
| @@ -82,15 +81,19 @@ pub fn main() !void { | |||
| 82 | else => {}, | 81 | else => {}, |
| 83 | } | 82 | } |
| 84 | } | 83 | } |
| 84 | } else |err| switch (err) { | ||
| 85 | error.EndOfStream => {}, | ||
| 86 | else => { | ||
| 87 | return err; | ||
| 88 | }, | ||
| 85 | } | 89 | } |
| 86 | |||
| 87 | var blocks_map = BlockMap.init(allocator); | 90 | var blocks_map = BlockMap.init(allocator); |
| 88 | defer blocks_map.deinit(); | 91 | defer blocks_map.deinit(); |
| 89 | 92 | ||
| 90 | var stage1 = std.ArrayList(u16).init(allocator); | 93 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 91 | defer stage1.deinit(); | 94 | defer stage1.deinit(); |
| 92 | 95 | ||
| 93 | var stage2 = std.ArrayList(u8).init(allocator); | 96 | var stage2 = std.array_list.Managed(u8).init(allocator); |
| 94 | defer stage2.deinit(); | 97 | defer stage2.deinit(); |
| 95 | 98 | ||
| 96 | var block: Block = [_]u8{0} ** block_size; | 99 | var block: Block = [_]u8{0} ** block_size; |
| @@ -121,18 +124,17 @@ pub fn main() !void { | |||
| 121 | _ = args_iter.skip(); | 124 | _ = args_iter.skip(); |
| 122 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 125 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 123 | 126 | ||
| 124 | const compressor = std.compress.flate.deflate.compressor; | 127 | var out_buf: [4096]u8 = undefined; |
| 125 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 128 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 126 | defer out_file.close(); | 129 | defer out_file.close(); |
| 127 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 130 | var writer = out_file.writer(&out_buf); |
| 128 | const writer = out_comp.writer(); | ||
| 129 | 131 | ||
| 130 | const endian = builtin.cpu.arch.endian(); | 132 | const endian = builtin.cpu.arch.endian(); |
| 131 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 133 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 132 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 134 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 133 | 135 | ||
| 134 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 136 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 135 | try writer.writeAll(stage2.items); | 137 | try writer.interface.writeAll(stage2.items); |
| 136 | 138 | ||
| 137 | try out_comp.flush(); | 139 | try writer.interface.flush(); |
| 138 | } | 140 | } |
diff --git a/codegen/dwp.zig b/codegen/dwp.zig index 5e5bf6a..fad08d1 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig | |||
| @@ -23,7 +23,7 @@ const BlockMap = std.HashMap( | |||
| 23 | std.hash_map.default_max_load_percentage, | 23 | std.hash_map.default_max_load_percentage, |
| 24 | ); | 24 | ); |
| 25 | 25 | ||
| 26 | pub fn main() !void { | 26 | pub fn main() anyerror!void { |
| 27 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 27 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 28 | defer arena.deinit(); | 28 | defer arena.deinit(); |
| 29 | const allocator = arena.allocator(); | 29 | const allocator = arena.allocator(); |
| @@ -36,10 +36,9 @@ pub fn main() !void { | |||
| 36 | // Process DerivedEastAsianWidth.txt | 36 | // Process DerivedEastAsianWidth.txt |
| 37 | var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); | 37 | var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); |
| 38 | defer deaw_file.close(); | 38 | defer deaw_file.close(); |
| 39 | var deaw_buf = std.io.bufferedReader(deaw_file.reader()); | 39 | var deaw_reader = deaw_file.reader(&line_buf); |
| 40 | const deaw_reader = deaw_buf.reader(); | ||
| 41 | 40 | ||
| 42 | while (try deaw_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 41 | while (deaw_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 43 | if (line.len == 0) continue; | 42 | if (line.len == 0) continue; |
| 44 | 43 | ||
| 45 | // @missing ranges | 44 | // @missing ranges |
| @@ -88,15 +87,18 @@ pub fn main() !void { | |||
| 88 | else => {}, | 87 | else => {}, |
| 89 | } | 88 | } |
| 90 | } | 89 | } |
| 90 | } else |err| switch (err) { | ||
| 91 | error.EndOfStream => {}, | ||
| 92 | else => { | ||
| 93 | return err; | ||
| 94 | }, | ||
| 91 | } | 95 | } |
| 92 | |||
| 93 | // Process DerivedGeneralCategory.txt | 96 | // Process DerivedGeneralCategory.txt |
| 94 | var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); | 97 | var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); |
| 95 | defer dgc_file.close(); | 98 | defer dgc_file.close(); |
| 96 | var dgc_buf = std.io.bufferedReader(dgc_file.reader()); | 99 | var dgc_reader = dgc_file.reader(&line_buf); |
| 97 | const dgc_reader = dgc_buf.reader(); | ||
| 98 | 100 | ||
| 99 | while (try dgc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 101 | while (dgc_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 100 | if (line.len == 0 or line[0] == '#') continue; | 102 | if (line.len == 0 or line[0] == '#') continue; |
| 101 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 103 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| 102 | 104 | ||
| @@ -139,15 +141,20 @@ pub fn main() !void { | |||
| 139 | else => {}, | 141 | else => {}, |
| 140 | } | 142 | } |
| 141 | } | 143 | } |
| 144 | } else |err| switch (err) { | ||
| 145 | error.EndOfStream => {}, | ||
| 146 | else => { | ||
| 147 | return err; | ||
| 148 | }, | ||
| 142 | } | 149 | } |
| 143 | 150 | ||
| 144 | var blocks_map = BlockMap.init(allocator); | 151 | var blocks_map = BlockMap.init(allocator); |
| 145 | defer blocks_map.deinit(); | 152 | defer blocks_map.deinit(); |
| 146 | 153 | ||
| 147 | var stage1 = std.ArrayList(u16).init(allocator); | 154 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 148 | defer stage1.deinit(); | 155 | defer stage1.deinit(); |
| 149 | 156 | ||
| 150 | var stage2 = std.ArrayList(i4).init(allocator); | 157 | var stage2 = std.array_list.Managed(i4).init(allocator); |
| 151 | defer stage2.deinit(); | 158 | defer stage2.deinit(); |
| 152 | 159 | ||
| 153 | var block: Block = [_]i4{0} ** block_size; | 160 | var block: Block = [_]i4{0} ** block_size; |
| @@ -227,18 +234,17 @@ pub fn main() !void { | |||
| 227 | _ = args_iter.skip(); | 234 | _ = args_iter.skip(); |
| 228 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 235 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 229 | 236 | ||
| 230 | const compressor = std.compress.flate.deflate.compressor; | 237 | var write_buf: [4096]u8 = undefined; |
| 231 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 238 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 232 | defer out_file.close(); | 239 | defer out_file.close(); |
| 233 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 240 | var writer = out_file.writer(&write_buf); |
| 234 | const writer = out_comp.writer(); | ||
| 235 | 241 | ||
| 236 | const endian = builtin.cpu.arch.endian(); | 242 | const endian = builtin.cpu.arch.endian(); |
| 237 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 243 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 238 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 244 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 239 | 245 | ||
| 240 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 246 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 241 | for (stage2.items) |i| try writer.writeInt(i8, i, endian); | 247 | for (stage2.items) |i| try writer.interface.writeInt(i8, i, endian); |
| 242 | 248 | ||
| 243 | try out_comp.flush(); | 249 | try writer.interface.flush(); |
| 244 | } | 250 | } |
diff --git a/codegen/fold.zig b/codegen/fold.zig index cb73cca..cc381a8 100644 --- a/codegen/fold.zig +++ b/codegen/fold.zig | |||
| @@ -2,23 +2,21 @@ const std = @import("std"); | |||
| 2 | const builtin = @import("builtin"); | 2 | const builtin = @import("builtin"); |
| 3 | const mem = std.mem; | 3 | const mem = std.mem; |
| 4 | 4 | ||
| 5 | pub fn main() !void { | 5 | pub fn main() anyerror!void { |
| 6 | var gpa = std.heap.GeneralPurposeAllocator(.{}){}; | 6 | var gpa = std.heap.GeneralPurposeAllocator(.{}){}; |
| 7 | defer std.debug.assert(gpa.deinit() == .ok); | 7 | defer std.debug.assert(gpa.deinit() == .ok); |
| 8 | const allocator = gpa.allocator(); | 8 | const allocator = gpa.allocator(); |
| 9 | 9 | ||
| 10 | var line_buf: [4096]u8 = undefined; | ||
| 10 | // Process DerivedCoreProperties.txt | 11 | // Process DerivedCoreProperties.txt |
| 11 | var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); | 12 | var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); |
| 12 | defer props_file.close(); | 13 | defer props_file.close(); |
| 13 | var props_buf = std.io.bufferedReader(props_file.reader()); | 14 | var props_reader = props_file.reader(&line_buf); |
| 14 | const props_reader = props_buf.reader(); | ||
| 15 | 15 | ||
| 16 | var props_map = std.AutoHashMap(u21, void).init(allocator); | 16 | var props_map = std.AutoHashMap(u21, void).init(allocator); |
| 17 | defer props_map.deinit(); | 17 | defer props_map.deinit(); |
| 18 | 18 | ||
| 19 | var line_buf: [4096]u8 = undefined; | 19 | props_lines: while (props_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 20 | |||
| 21 | props_lines: while (try props_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | ||
| 22 | if (line.len == 0 or line[0] == '#') continue; | 20 | if (line.len == 0 or line[0] == '#') continue; |
| 23 | 21 | ||
| 24 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 22 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| @@ -49,18 +47,21 @@ pub fn main() !void { | |||
| 49 | else => {}, | 47 | else => {}, |
| 50 | } | 48 | } |
| 51 | } | 49 | } |
| 50 | } else |err| switch (err) { | ||
| 51 | error.EndOfStream => {}, | ||
| 52 | else => { | ||
| 53 | return err; | ||
| 54 | }, | ||
| 52 | } | 55 | } |
| 53 | |||
| 54 | var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator); | 56 | var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator); |
| 55 | defer codepoint_mapping.deinit(); | 57 | defer codepoint_mapping.deinit(); |
| 56 | 58 | ||
| 57 | // Process CaseFolding.txt | 59 | // Process CaseFolding.txt |
| 58 | var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{}); | 60 | var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{}); |
| 59 | defer cp_file.close(); | 61 | defer cp_file.close(); |
| 60 | var cp_buf = std.io.bufferedReader(cp_file.reader()); | 62 | var cp_reader = cp_file.reader(&line_buf); |
| 61 | const cp_reader = cp_buf.reader(); | ||
| 62 | 63 | ||
| 63 | while (try cp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 64 | while (cp_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 64 | if (line.len == 0 or line[0] == '#') continue; | 65 | if (line.len == 0 or line[0] == '#') continue; |
| 65 | 66 | ||
| 66 | var field_it = std.mem.splitScalar(u8, line, ';'); | 67 | var field_it = std.mem.splitScalar(u8, line, ';'); |
| @@ -81,9 +82,14 @@ pub fn main() !void { | |||
| 81 | } | 82 | } |
| 82 | 83 | ||
| 83 | try codepoint_mapping.putNoClobber(codepoint, mapping_buf); | 84 | try codepoint_mapping.putNoClobber(codepoint, mapping_buf); |
| 85 | } else |err| switch (err) { | ||
| 86 | error.EndOfStream => {}, | ||
| 87 | else => { | ||
| 88 | return err; | ||
| 89 | }, | ||
| 84 | } | 90 | } |
| 85 | 91 | ||
| 86 | var changes_when_casefolded_exceptions = std.ArrayList(u21).init(allocator); | 92 | var changes_when_casefolded_exceptions = std.array_list.Managed(u21).init(allocator); |
| 87 | defer changes_when_casefolded_exceptions.deinit(); | 93 | defer changes_when_casefolded_exceptions.deinit(); |
| 88 | 94 | ||
| 89 | { | 95 | { |
| @@ -221,32 +227,31 @@ pub fn main() !void { | |||
| 221 | _ = args_iter.skip(); | 227 | _ = args_iter.skip(); |
| 222 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 228 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 223 | 229 | ||
| 224 | const compressor = std.compress.flate.deflate.compressor; | 230 | var write_buf: [4096]u8 = undefined; |
| 225 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 231 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 226 | defer out_file.close(); | 232 | defer out_file.close(); |
| 227 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 233 | var writer = out_file.writer(&write_buf); |
| 228 | const writer = out_comp.writer(); | ||
| 229 | 234 | ||
| 230 | const endian = builtin.cpu.arch.endian(); | 235 | const endian = builtin.cpu.arch.endian(); |
| 231 | // Table metadata. | 236 | // Table metadata. |
| 232 | try writer.writeInt(u24, @intCast(codepoint_cutoff), endian); | 237 | try writer.interface.writeInt(u24, @intCast(codepoint_cutoff), endian); |
| 233 | try writer.writeInt(u24, @intCast(multiple_codepoint_start), endian); | 238 | try writer.interface.writeInt(u24, @intCast(multiple_codepoint_start), endian); |
| 234 | // Stage 1 | 239 | // Stage 1 |
| 235 | try writer.writeInt(u16, @intCast(meaningful_stage1.len), endian); | 240 | try writer.interface.writeInt(u16, @intCast(meaningful_stage1.len), endian); |
| 236 | try writer.writeAll(meaningful_stage1); | 241 | try writer.interface.writeAll(meaningful_stage1); |
| 237 | // Stage 2 | 242 | // Stage 2 |
| 238 | try writer.writeInt(u16, @intCast(stage2.len), endian); | 243 | try writer.interface.writeInt(u16, @intCast(stage2.len), endian); |
| 239 | try writer.writeAll(stage2); | 244 | try writer.interface.writeAll(stage2); |
| 240 | // Stage 3 | 245 | // Stage 3 |
| 241 | try writer.writeInt(u16, @intCast(stage3.len), endian); | 246 | try writer.interface.writeInt(u16, @intCast(stage3.len), endian); |
| 242 | for (stage3) |offset| try writer.writeInt(i24, offset, endian); | 247 | for (stage3) |offset| try writer.interface.writeInt(i24, offset, endian); |
| 243 | // Changes when case folded | 248 | // Changes when case folded |
| 244 | // Min and max | 249 | // Min and max |
| 245 | try writer.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); | 250 | try writer.interface.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); |
| 246 | try writer.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); | 251 | try writer.interface.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); |
| 247 | try writer.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); | 252 | try writer.interface.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); |
| 248 | for (changes_when_casefolded_exceptions.items) |cp| try writer.writeInt(u24, cp, endian); | 253 | for (changes_when_casefolded_exceptions.items) |cp| try writer.interface.writeInt(u24, cp, endian); |
| 249 | 254 | ||
| 250 | try out_comp.flush(); | 255 | try writer.interface.flush(); |
| 251 | } | 256 | } |
| 252 | } | 257 | } |
diff --git a/codegen/gbp.zig b/codegen/gbp.zig index 3fc4461..d654cf1 100644 --- a/codegen/gbp.zig +++ b/codegen/gbp.zig | |||
| @@ -47,7 +47,7 @@ const BlockMap = std.HashMap( | |||
| 47 | std.hash_map.default_max_load_percentage, | 47 | std.hash_map.default_max_load_percentage, |
| 48 | ); | 48 | ); |
| 49 | 49 | ||
| 50 | pub fn main() !void { | 50 | pub fn main() anyerror!void { |
| 51 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 51 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 52 | defer arena.deinit(); | 52 | defer arena.deinit(); |
| 53 | const allocator = arena.allocator(); | 53 | const allocator = arena.allocator(); |
| @@ -66,10 +66,9 @@ pub fn main() !void { | |||
| 66 | // Process Indic | 66 | // Process Indic |
| 67 | var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); | 67 | var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); |
| 68 | defer indic_file.close(); | 68 | defer indic_file.close(); |
| 69 | var indic_buf = std.io.bufferedReader(indic_file.reader()); | 69 | var indic_reader = indic_file.reader(&line_buf); |
| 70 | const indic_reader = indic_buf.reader(); | ||
| 71 | 70 | ||
| 72 | while (try indic_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 71 | while (indic_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 73 | if (line.len == 0 or line[0] == '#') continue; | 72 | if (line.len == 0 or line[0] == '#') continue; |
| 74 | if (std.mem.indexOf(u8, line, "InCB") == null) continue; | 73 | if (std.mem.indexOf(u8, line, "InCB") == null) continue; |
| 75 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 74 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| @@ -100,15 +99,18 @@ pub fn main() !void { | |||
| 100 | else => {}, | 99 | else => {}, |
| 101 | } | 100 | } |
| 102 | } | 101 | } |
| 102 | } else |err| switch (err) { | ||
| 103 | error.EndOfStream => {}, | ||
| 104 | else => { | ||
| 105 | return err; | ||
| 106 | }, | ||
| 103 | } | 107 | } |
| 104 | |||
| 105 | // Process GBP | 108 | // Process GBP |
| 106 | var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{}); | 109 | var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{}); |
| 107 | defer gbp_file.close(); | 110 | defer gbp_file.close(); |
| 108 | var gbp_buf = std.io.bufferedReader(gbp_file.reader()); | 111 | var gbp_reader = gbp_file.reader(&line_buf); |
| 109 | const gbp_reader = gbp_buf.reader(); | ||
| 110 | 112 | ||
| 111 | while (try gbp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 113 | while (gbp_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 112 | if (line.len == 0 or line[0] == '#') continue; | 114 | if (line.len == 0 or line[0] == '#') continue; |
| 113 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 115 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| 114 | 116 | ||
| @@ -138,15 +140,18 @@ pub fn main() !void { | |||
| 138 | else => {}, | 140 | else => {}, |
| 139 | } | 141 | } |
| 140 | } | 142 | } |
| 143 | } else |err| switch (err) { | ||
| 144 | error.EndOfStream => {}, | ||
| 145 | else => { | ||
| 146 | return err; | ||
| 147 | }, | ||
| 141 | } | 148 | } |
| 142 | |||
| 143 | // Process Emoji | 149 | // Process Emoji |
| 144 | var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{}); | 150 | var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{}); |
| 145 | defer emoji_file.close(); | 151 | defer emoji_file.close(); |
| 146 | var emoji_buf = std.io.bufferedReader(emoji_file.reader()); | 152 | var emoji_reader = emoji_file.reader(&line_buf); |
| 147 | const emoji_reader = emoji_buf.reader(); | ||
| 148 | 153 | ||
| 149 | while (try emoji_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 154 | while (emoji_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 150 | if (line.len == 0 or line[0] == '#') continue; | 155 | if (line.len == 0 or line[0] == '#') continue; |
| 151 | if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; | 156 | if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; |
| 152 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 157 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| @@ -170,15 +175,20 @@ pub fn main() !void { | |||
| 170 | else => {}, | 175 | else => {}, |
| 171 | } | 176 | } |
| 172 | } | 177 | } |
| 178 | } else |err| switch (err) { | ||
| 179 | error.EndOfStream => {}, | ||
| 180 | else => { | ||
| 181 | return err; | ||
| 182 | }, | ||
| 173 | } | 183 | } |
| 174 | 184 | ||
| 175 | var blocks_map = BlockMap.init(allocator); | 185 | var blocks_map = BlockMap.init(allocator); |
| 176 | defer blocks_map.deinit(); | 186 | defer blocks_map.deinit(); |
| 177 | 187 | ||
| 178 | var stage1 = std.ArrayList(u16).init(allocator); | 188 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 179 | defer stage1.deinit(); | 189 | defer stage1.deinit(); |
| 180 | 190 | ||
| 181 | var stage2 = std.ArrayList(u16).init(allocator); | 191 | var stage2 = std.array_list.Managed(u16).init(allocator); |
| 182 | defer stage2.deinit(); | 192 | defer stage2.deinit(); |
| 183 | 193 | ||
| 184 | var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator); | 194 | var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator); |
| @@ -227,22 +237,21 @@ pub fn main() !void { | |||
| 227 | _ = args_iter.skip(); | 237 | _ = args_iter.skip(); |
| 228 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 238 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 229 | 239 | ||
| 230 | const compressor = std.compress.flate.deflate.compressor; | 240 | var write_buf: [4096]u8 = undefined; |
| 231 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 241 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 232 | defer out_file.close(); | 242 | defer out_file.close(); |
| 233 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 243 | var writer = out_file.writer(&write_buf); |
| 234 | const writer = out_comp.writer(); | ||
| 235 | 244 | ||
| 236 | const endian = builtin.cpu.arch.endian(); | 245 | const endian = builtin.cpu.arch.endian(); |
| 237 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 246 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 238 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 247 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 239 | 248 | ||
| 240 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 249 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 241 | for (stage2.items) |i| try writer.writeInt(u16, i, endian); | 250 | for (stage2.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 242 | 251 | ||
| 243 | const props_bytes = stage3.keys(); | 252 | const props_bytes = stage3.keys(); |
| 244 | try writer.writeInt(u16, @intCast(props_bytes.len), endian); | 253 | try writer.interface.writeInt(u16, @intCast(props_bytes.len), endian); |
| 245 | try writer.writeAll(props_bytes); | 254 | try writer.interface.writeAll(props_bytes); |
| 246 | 255 | ||
| 247 | try out_comp.flush(); | 256 | try writer.interface.flush(); |
| 248 | } | 257 | } |
diff --git a/codegen/gencat.zig b/codegen/gencat.zig index fe06bd7..7dedb5d 100644 --- a/codegen/gencat.zig +++ b/codegen/gencat.zig | |||
| @@ -67,10 +67,9 @@ pub fn main() !void { | |||
| 67 | // Process DerivedGeneralCategory.txt | 67 | // Process DerivedGeneralCategory.txt |
| 68 | var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); | 68 | var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); |
| 69 | defer in_file.close(); | 69 | defer in_file.close(); |
| 70 | var in_buf = std.io.bufferedReader(in_file.reader()); | 70 | var in_reader = in_file.reader(&line_buf); |
| 71 | const in_reader = in_buf.reader(); | ||
| 72 | 71 | ||
| 73 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 72 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 74 | if (line.len == 0 or line[0] == '#') continue; | 73 | if (line.len == 0 or line[0] == '#') continue; |
| 75 | 74 | ||
| 76 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 75 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| @@ -101,18 +100,23 @@ pub fn main() !void { | |||
| 101 | else => {}, | 100 | else => {}, |
| 102 | } | 101 | } |
| 103 | } | 102 | } |
| 103 | } else |err| switch (err) { | ||
| 104 | error.EndOfStream => {}, | ||
| 105 | else => { | ||
| 106 | return err; | ||
| 107 | }, | ||
| 104 | } | 108 | } |
| 105 | 109 | ||
| 106 | var blocks_map = BlockMap.init(allocator); | 110 | var blocks_map = BlockMap.init(allocator); |
| 107 | defer blocks_map.deinit(); | 111 | defer blocks_map.deinit(); |
| 108 | 112 | ||
| 109 | var stage1 = std.ArrayList(u16).init(allocator); | 113 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 110 | defer stage1.deinit(); | 114 | defer stage1.deinit(); |
| 111 | 115 | ||
| 112 | var stage2 = std.ArrayList(u5).init(allocator); | 116 | var stage2 = std.array_list.Managed(u5).init(allocator); |
| 113 | defer stage2.deinit(); | 117 | defer stage2.deinit(); |
| 114 | 118 | ||
| 115 | var stage3 = std.ArrayList(u5).init(allocator); | 119 | var stage3 = std.array_list.Managed(u5).init(allocator); |
| 116 | defer stage3.deinit(); | 120 | defer stage3.deinit(); |
| 117 | 121 | ||
| 118 | var block: Block = [_]u5{0} ** block_size; | 122 | var block: Block = [_]u5{0} ** block_size; |
| @@ -151,21 +155,20 @@ pub fn main() !void { | |||
| 151 | _ = args_iter.skip(); | 155 | _ = args_iter.skip(); |
| 152 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 156 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 153 | 157 | ||
| 154 | const compressor = std.compress.flate.deflate.compressor; | 158 | var write_buf: [4096]u8 = undefined; |
| 155 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 159 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 156 | defer out_file.close(); | 160 | defer out_file.close(); |
| 157 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 161 | var writer = out_file.writer(&write_buf); |
| 158 | const writer = out_comp.writer(); | ||
| 159 | 162 | ||
| 160 | const endian = builtin.cpu.arch.endian(); | 163 | const endian = builtin.cpu.arch.endian(); |
| 161 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 164 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 162 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 165 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 163 | 166 | ||
| 164 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 167 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 165 | for (stage2.items) |i| try writer.writeInt(u8, i, endian); | 168 | for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); |
| 166 | 169 | ||
| 167 | try writer.writeInt(u8, @intCast(stage3.items.len), endian); | 170 | try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian); |
| 168 | for (stage3.items) |i| try writer.writeInt(u8, i, endian); | 171 | for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian); |
| 169 | 172 | ||
| 170 | try out_comp.flush(); | 173 | try writer.interface.flush(); |
| 171 | } | 174 | } |
diff --git a/codegen/hangul.zig b/codegen/hangul.zig index 2c42bb7..6768b3f 100644 --- a/codegen/hangul.zig +++ b/codegen/hangul.zig | |||
| @@ -30,7 +30,7 @@ const BlockMap = std.HashMap( | |||
| 30 | std.hash_map.default_max_load_percentage, | 30 | std.hash_map.default_max_load_percentage, |
| 31 | ); | 31 | ); |
| 32 | 32 | ||
| 33 | pub fn main() !void { | 33 | pub fn main() anyerror!void { |
| 34 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 34 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 35 | defer arena.deinit(); | 35 | defer arena.deinit(); |
| 36 | const allocator = arena.allocator(); | 36 | const allocator = arena.allocator(); |
| @@ -43,10 +43,9 @@ pub fn main() !void { | |||
| 43 | // Process HangulSyllableType.txt | 43 | // Process HangulSyllableType.txt |
| 44 | var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); | 44 | var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); |
| 45 | defer in_file.close(); | 45 | defer in_file.close(); |
| 46 | var in_buf = std.io.bufferedReader(in_file.reader()); | 46 | var in_reader = in_file.reader(&line_buf); |
| 47 | const in_reader = in_buf.reader(); | ||
| 48 | 47 | ||
| 49 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 48 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 50 | if (line.len == 0 or line[0] == '#') continue; | 49 | if (line.len == 0 or line[0] == '#') continue; |
| 51 | 50 | ||
| 52 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 51 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| @@ -77,15 +76,19 @@ pub fn main() !void { | |||
| 77 | else => {}, | 76 | else => {}, |
| 78 | } | 77 | } |
| 79 | } | 78 | } |
| 79 | } else |err| switch (err) { | ||
| 80 | error.EndOfStream => {}, | ||
| 81 | else => { | ||
| 82 | return err; | ||
| 83 | }, | ||
| 80 | } | 84 | } |
| 81 | |||
| 82 | var blocks_map = BlockMap.init(allocator); | 85 | var blocks_map = BlockMap.init(allocator); |
| 83 | defer blocks_map.deinit(); | 86 | defer blocks_map.deinit(); |
| 84 | 87 | ||
| 85 | var stage1 = std.ArrayList(u16).init(allocator); | 88 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 86 | defer stage1.deinit(); | 89 | defer stage1.deinit(); |
| 87 | 90 | ||
| 88 | var stage2 = std.ArrayList(u3).init(allocator); | 91 | var stage2 = std.array_list.Managed(u3).init(allocator); |
| 89 | defer stage2.deinit(); | 92 | defer stage2.deinit(); |
| 90 | 93 | ||
| 91 | var block: Block = [_]u3{0} ** block_size; | 94 | var block: Block = [_]u3{0} ** block_size; |
| @@ -116,18 +119,17 @@ pub fn main() !void { | |||
| 116 | _ = args_iter.skip(); | 119 | _ = args_iter.skip(); |
| 117 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 120 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 118 | 121 | ||
| 119 | const compressor = std.compress.flate.deflate.compressor; | 122 | var write_buf: [4096]u8 = undefined; |
| 120 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 123 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 121 | defer out_file.close(); | 124 | defer out_file.close(); |
| 122 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 125 | var writer = out_file.writer(&write_buf); |
| 123 | const writer = out_comp.writer(); | ||
| 124 | 126 | ||
| 125 | const endian = builtin.cpu.arch.endian(); | 127 | const endian = builtin.cpu.arch.endian(); |
| 126 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 128 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 127 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 129 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 128 | 130 | ||
| 129 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 131 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 130 | for (stage2.items) |i| try writer.writeInt(u8, i, endian); | 132 | for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); |
| 131 | 133 | ||
| 132 | try out_comp.flush(); | 134 | try writer.interface.flush(); |
| 133 | } | 135 | } |
diff --git a/codegen/lower.zig b/codegen/lower.zig index a053fe3..c11cb0a 100644 --- a/codegen/lower.zig +++ b/codegen/lower.zig | |||
| @@ -6,27 +6,25 @@ pub fn main() !void { | |||
| 6 | defer arena.deinit(); | 6 | defer arena.deinit(); |
| 7 | const allocator = arena.allocator(); | 7 | const allocator = arena.allocator(); |
| 8 | 8 | ||
| 9 | var line_buf: [4096]u8 = undefined; | ||
| 10 | var write_buf: [4096]u8 = undefined; | ||
| 9 | // Process UnicodeData.txt | 11 | // Process UnicodeData.txt |
| 10 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); | 12 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); |
| 11 | defer in_file.close(); | 13 | defer in_file.close(); |
| 12 | var in_buf = std.io.bufferedReader(in_file.reader()); | 14 | var in_reader = in_file.reader(&line_buf); |
| 13 | const in_reader = in_buf.reader(); | ||
| 14 | 15 | ||
| 15 | var args_iter = try std.process.argsWithAllocator(allocator); | 16 | var args_iter = try std.process.argsWithAllocator(allocator); |
| 16 | defer args_iter.deinit(); | 17 | defer args_iter.deinit(); |
| 17 | _ = args_iter.skip(); | 18 | _ = args_iter.skip(); |
| 18 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 19 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 19 | 20 | ||
| 20 | const compressor = std.compress.flate.deflate.compressor; | ||
| 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 22 | defer out_file.close(); | 22 | defer out_file.close(); |
| 23 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 23 | var writer = out_file.writer(&write_buf); |
| 24 | const writer = out_comp.writer(); | ||
| 25 | 24 | ||
| 26 | const endian = builtin.cpu.arch.endian(); | 25 | const endian = builtin.cpu.arch.endian(); |
| 27 | var line_buf: [4096]u8 = undefined; | ||
| 28 | 26 | ||
| 29 | lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 27 | lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 30 | if (line.len == 0) continue; | 28 | if (line.len == 0) continue; |
| 31 | 29 | ||
| 32 | var field_iter = std.mem.splitScalar(u8, line, ';'); | 30 | var field_iter = std.mem.splitScalar(u8, line, ';'); |
| @@ -42,16 +40,20 @@ pub fn main() !void { | |||
| 42 | 13 => { | 40 | 13 => { |
| 43 | // Simple lowercase mapping | 41 | // Simple lowercase mapping |
| 44 | if (field.len == 0) continue :lines; | 42 | if (field.len == 0) continue :lines; |
| 45 | try writer.writeInt(i24, cp, endian); | 43 | try writer.interface.writeInt(i24, cp, endian); |
| 46 | const mapping = try std.fmt.parseInt(i24, field, 16); | 44 | const mapping = try std.fmt.parseInt(i24, field, 16); |
| 47 | try writer.writeInt(i24, mapping - cp, endian); | 45 | try writer.interface.writeInt(i24, mapping - cp, endian); |
| 48 | }, | 46 | }, |
| 49 | 47 | ||
| 50 | else => {}, | 48 | else => {}, |
| 51 | } | 49 | } |
| 52 | } | 50 | } |
| 51 | } else |err| switch (err) { | ||
| 52 | error.EndOfStream => {}, | ||
| 53 | else => { | ||
| 54 | return err; | ||
| 55 | }, | ||
| 53 | } | 56 | } |
| 54 | 57 | try writer.interface.writeInt(u24, 0, endian); | |
| 55 | try writer.writeInt(u24, 0, endian); | 58 | try writer.interface.flush(); |
| 56 | try out_comp.flush(); | ||
| 57 | } | 59 | } |
diff --git a/codegen/normp.zig b/codegen/normp.zig index 60dabdc..71a1ae5 100644 --- a/codegen/normp.zig +++ b/codegen/normp.zig | |||
| @@ -21,7 +21,7 @@ const BlockMap = std.HashMap( | |||
| 21 | std.hash_map.default_max_load_percentage, | 21 | std.hash_map.default_max_load_percentage, |
| 22 | ); | 22 | ); |
| 23 | 23 | ||
| 24 | pub fn main() !void { | 24 | pub fn main() anyerror!void { |
| 25 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 25 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 26 | defer arena.deinit(); | 26 | defer arena.deinit(); |
| 27 | const allocator = arena.allocator(); | 27 | const allocator = arena.allocator(); |
| @@ -34,10 +34,9 @@ pub fn main() !void { | |||
| 34 | // Process DerivedNormalizationProps.txt | 34 | // Process DerivedNormalizationProps.txt |
| 35 | var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); | 35 | var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); |
| 36 | defer in_file.close(); | 36 | defer in_file.close(); |
| 37 | var in_buf = std.io.bufferedReader(in_file.reader()); | 37 | var in_reader = in_file.reader(&line_buf); |
| 38 | const in_reader = in_buf.reader(); | ||
| 39 | 38 | ||
| 40 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 39 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 41 | if (line.len == 0 or line[0] == '#') continue; | 40 | if (line.len == 0 or line[0] == '#') continue; |
| 42 | 41 | ||
| 43 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 42 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| @@ -78,15 +77,19 @@ pub fn main() !void { | |||
| 78 | else => {}, | 77 | else => {}, |
| 79 | } | 78 | } |
| 80 | } | 79 | } |
| 80 | } else |err| switch (err) { | ||
| 81 | error.EndOfStream => {}, | ||
| 82 | else => { | ||
| 83 | return err; | ||
| 84 | }, | ||
| 81 | } | 85 | } |
| 82 | |||
| 83 | var blocks_map = BlockMap.init(allocator); | 86 | var blocks_map = BlockMap.init(allocator); |
| 84 | defer blocks_map.deinit(); | 87 | defer blocks_map.deinit(); |
| 85 | 88 | ||
| 86 | var stage1 = std.ArrayList(u16).init(allocator); | 89 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 87 | defer stage1.deinit(); | 90 | defer stage1.deinit(); |
| 88 | 91 | ||
| 89 | var stage2 = std.ArrayList(u3).init(allocator); | 92 | var stage2 = std.array_list.Managed(u3).init(allocator); |
| 90 | defer stage2.deinit(); | 93 | defer stage2.deinit(); |
| 91 | 94 | ||
| 92 | var block: Block = [_]u3{0} ** block_size; | 95 | var block: Block = [_]u3{0} ** block_size; |
| @@ -117,18 +120,17 @@ pub fn main() !void { | |||
| 117 | _ = args_iter.skip(); | 120 | _ = args_iter.skip(); |
| 118 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 121 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 119 | 122 | ||
| 120 | const compressor = std.compress.flate.deflate.compressor; | 123 | var write_buf: [4096]u8 = undefined; |
| 121 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 124 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 122 | defer out_file.close(); | 125 | defer out_file.close(); |
| 123 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 126 | var writer = out_file.writer(&write_buf); |
| 124 | const writer = out_comp.writer(); | ||
| 125 | 127 | ||
| 126 | const endian = builtin.cpu.arch.endian(); | 128 | const endian = builtin.cpu.arch.endian(); |
| 127 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 129 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 128 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 130 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 129 | 131 | ||
| 130 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 132 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 131 | for (stage2.items) |i| try writer.writeInt(u8, i, endian); | 133 | for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); |
| 132 | 134 | ||
| 133 | try out_comp.flush(); | 135 | try writer.interface.flush(); |
| 134 | } | 136 | } |
diff --git a/codegen/numeric.zig b/codegen/numeric.zig index 038ac0a..cf918e8 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig | |||
| @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( | |||
| 22 | std.hash_map.default_max_load_percentage, | 22 | std.hash_map.default_max_load_percentage, |
| 23 | ); | 23 | ); |
| 24 | 24 | ||
| 25 | pub fn main() !void { | 25 | pub fn main() anyerror!void { |
| 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 27 | defer arena.deinit(); | 27 | defer arena.deinit(); |
| 28 | const allocator = arena.allocator(); | 28 | const allocator = arena.allocator(); |
| @@ -35,10 +35,9 @@ pub fn main() !void { | |||
| 35 | // Process DerivedNumericType.txt | 35 | // Process DerivedNumericType.txt |
| 36 | var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); | 36 | var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); |
| 37 | defer in_file.close(); | 37 | defer in_file.close(); |
| 38 | var in_buf = std.io.bufferedReader(in_file.reader()); | 38 | var in_reader = in_file.reader(&line_buf); |
| 39 | const in_reader = in_buf.reader(); | ||
| 40 | 39 | ||
| 41 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 40 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 42 | if (line.len == 0 or line[0] == '#') continue; | 41 | if (line.len == 0 or line[0] == '#') continue; |
| 43 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 42 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| 44 | 43 | ||
| @@ -79,15 +78,20 @@ pub fn main() !void { | |||
| 79 | else => {}, | 78 | else => {}, |
| 80 | } | 79 | } |
| 81 | } | 80 | } |
| 81 | } else |err| switch (err) { | ||
| 82 | error.EndOfStream => {}, | ||
| 83 | else => { | ||
| 84 | return err; | ||
| 85 | }, | ||
| 82 | } | 86 | } |
| 83 | 87 | ||
| 84 | var blocks_map = BlockMap.init(allocator); | 88 | var blocks_map = BlockMap.init(allocator); |
| 85 | defer blocks_map.deinit(); | 89 | defer blocks_map.deinit(); |
| 86 | 90 | ||
| 87 | var stage1 = std.ArrayList(u16).init(allocator); | 91 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 88 | defer stage1.deinit(); | 92 | defer stage1.deinit(); |
| 89 | 93 | ||
| 90 | var stage2 = std.ArrayList(u8).init(allocator); | 94 | var stage2 = std.array_list.Managed(u8).init(allocator); |
| 91 | defer stage2.deinit(); | 95 | defer stage2.deinit(); |
| 92 | 96 | ||
| 93 | var block: Block = [_]u8{0} ** block_size; | 97 | var block: Block = [_]u8{0} ** block_size; |
| @@ -118,18 +122,17 @@ pub fn main() !void { | |||
| 118 | _ = args_iter.skip(); | 122 | _ = args_iter.skip(); |
| 119 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 123 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 120 | 124 | ||
| 121 | const compressor = std.compress.flate.deflate.compressor; | 125 | var write_buf: [4096]u8 = undefined; |
| 122 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 126 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 123 | defer out_file.close(); | 127 | defer out_file.close(); |
| 124 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 128 | var writer = out_file.writer(&write_buf); |
| 125 | const writer = out_comp.writer(); | ||
| 126 | 129 | ||
| 127 | const endian = builtin.cpu.arch.endian(); | 130 | const endian = builtin.cpu.arch.endian(); |
| 128 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 131 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 129 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 132 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 130 | 133 | ||
| 131 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 134 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 132 | try writer.writeAll(stage2.items); | 135 | try writer.interface.writeAll(stage2.items); |
| 133 | 136 | ||
| 134 | try out_comp.flush(); | 137 | try writer.interface.flush(); |
| 135 | } | 138 | } |
diff --git a/codegen/props.zig b/codegen/props.zig index 24b22e0..6ff0a33 100644 --- a/codegen/props.zig +++ b/codegen/props.zig | |||
| @@ -22,7 +22,7 @@ const BlockMap = std.HashMap( | |||
| 22 | std.hash_map.default_max_load_percentage, | 22 | std.hash_map.default_max_load_percentage, |
| 23 | ); | 23 | ); |
| 24 | 24 | ||
| 25 | pub fn main() !void { | 25 | pub fn main() anyerror!void { |
| 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 27 | defer arena.deinit(); | 27 | defer arena.deinit(); |
| 28 | const allocator = arena.allocator(); | 28 | const allocator = arena.allocator(); |
| @@ -35,10 +35,9 @@ pub fn main() !void { | |||
| 35 | // Process PropList.txt | 35 | // Process PropList.txt |
| 36 | var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); | 36 | var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); |
| 37 | defer in_file.close(); | 37 | defer in_file.close(); |
| 38 | var in_buf = std.io.bufferedReader(in_file.reader()); | 38 | var in_reader = in_file.reader(&line_buf); |
| 39 | const in_reader = in_buf.reader(); | ||
| 40 | 39 | ||
| 41 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 40 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 42 | if (line.len == 0 or line[0] == '#') continue; | 41 | if (line.len == 0 or line[0] == '#') continue; |
| 43 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 42 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| 44 | 43 | ||
| @@ -79,15 +78,20 @@ pub fn main() !void { | |||
| 79 | else => {}, | 78 | else => {}, |
| 80 | } | 79 | } |
| 81 | } | 80 | } |
| 81 | } else |err| switch (err) { | ||
| 82 | error.EndOfStream => {}, | ||
| 83 | else => { | ||
| 84 | return err; | ||
| 85 | }, | ||
| 82 | } | 86 | } |
| 83 | 87 | ||
| 84 | var blocks_map = BlockMap.init(allocator); | 88 | var blocks_map = BlockMap.init(allocator); |
| 85 | defer blocks_map.deinit(); | 89 | defer blocks_map.deinit(); |
| 86 | 90 | ||
| 87 | var stage1 = std.ArrayList(u16).init(allocator); | 91 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 88 | defer stage1.deinit(); | 92 | defer stage1.deinit(); |
| 89 | 93 | ||
| 90 | var stage2 = std.ArrayList(u8).init(allocator); | 94 | var stage2 = std.array_list.Managed(u8).init(allocator); |
| 91 | defer stage2.deinit(); | 95 | defer stage2.deinit(); |
| 92 | 96 | ||
| 93 | var block: Block = [_]u8{0} ** block_size; | 97 | var block: Block = [_]u8{0} ** block_size; |
| @@ -118,18 +122,16 @@ pub fn main() !void { | |||
| 118 | _ = args_iter.skip(); | 122 | _ = args_iter.skip(); |
| 119 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 123 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 120 | 124 | ||
| 121 | const compressor = std.compress.flate.deflate.compressor; | 125 | var write_buf: [4096]u8 = undefined; |
| 122 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 126 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 123 | defer out_file.close(); | 127 | defer out_file.close(); |
| 124 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 128 | var writer = out_file.writer(&write_buf); |
| 125 | const writer = out_comp.writer(); | ||
| 126 | 129 | ||
| 127 | const endian = builtin.cpu.arch.endian(); | 130 | const endian = builtin.cpu.arch.endian(); |
| 128 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 131 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 129 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 132 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 130 | 133 | ||
| 131 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 134 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 132 | try writer.writeAll(stage2.items); | 135 | try writer.interface.writeAll(stage2.items); |
| 133 | 136 | try writer.interface.flush(); | |
| 134 | try out_comp.flush(); | ||
| 135 | } | 137 | } |
diff --git a/codegen/scripts.zig b/codegen/scripts.zig index 530205d..bdd3d9d 100644 --- a/codegen/scripts.zig +++ b/codegen/scripts.zig | |||
| @@ -195,7 +195,7 @@ const BlockMap = std.HashMap( | |||
| 195 | std.hash_map.default_max_load_percentage, | 195 | std.hash_map.default_max_load_percentage, |
| 196 | ); | 196 | ); |
| 197 | 197 | ||
| 198 | pub fn main() !void { | 198 | pub fn main() anyerror!void { |
| 199 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 199 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 200 | defer arena.deinit(); | 200 | defer arena.deinit(); |
| 201 | const allocator = arena.allocator(); | 201 | const allocator = arena.allocator(); |
| @@ -208,10 +208,9 @@ pub fn main() !void { | |||
| 208 | // Process DerivedGeneralCategory.txt | 208 | // Process DerivedGeneralCategory.txt |
| 209 | var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); | 209 | var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); |
| 210 | defer in_file.close(); | 210 | defer in_file.close(); |
| 211 | var in_buf = std.io.bufferedReader(in_file.reader()); | 211 | var in_reader = in_file.reader(&line_buf); |
| 212 | const in_reader = in_buf.reader(); | ||
| 213 | 212 | ||
| 214 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 213 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 215 | if (line.len == 0 or line[0] == '#') continue; | 214 | if (line.len == 0 or line[0] == '#') continue; |
| 216 | 215 | ||
| 217 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 216 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| @@ -245,18 +244,22 @@ pub fn main() !void { | |||
| 245 | else => {}, | 244 | else => {}, |
| 246 | } | 245 | } |
| 247 | } | 246 | } |
| 247 | } else |err| switch (err) { | ||
| 248 | error.EndOfStream => {}, | ||
| 249 | else => { | ||
| 250 | return err; | ||
| 251 | }, | ||
| 248 | } | 252 | } |
| 249 | |||
| 250 | var blocks_map = BlockMap.init(allocator); | 253 | var blocks_map = BlockMap.init(allocator); |
| 251 | defer blocks_map.deinit(); | 254 | defer blocks_map.deinit(); |
| 252 | 255 | ||
| 253 | var stage1 = std.ArrayList(u16).init(allocator); | 256 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 254 | defer stage1.deinit(); | 257 | defer stage1.deinit(); |
| 255 | 258 | ||
| 256 | var stage2 = std.ArrayList(u8).init(allocator); | 259 | var stage2 = std.array_list.Managed(u8).init(allocator); |
| 257 | defer stage2.deinit(); | 260 | defer stage2.deinit(); |
| 258 | 261 | ||
| 259 | var stage3 = std.ArrayList(u8).init(allocator); | 262 | var stage3 = std.array_list.Managed(u8).init(allocator); |
| 260 | defer stage3.deinit(); | 263 | defer stage3.deinit(); |
| 261 | 264 | ||
| 262 | var block: Block = [_]u8{0} ** block_size; | 265 | var block: Block = [_]u8{0} ** block_size; |
| @@ -295,21 +298,20 @@ pub fn main() !void { | |||
| 295 | _ = args_iter.skip(); | 298 | _ = args_iter.skip(); |
| 296 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 299 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 297 | 300 | ||
| 298 | const compressor = std.compress.flate.deflate.compressor; | 301 | var write_buf: [4096]u8 = undefined; |
| 299 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 302 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 300 | defer out_file.close(); | 303 | defer out_file.close(); |
| 301 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 304 | var writer = out_file.writer(&write_buf); |
| 302 | const writer = out_comp.writer(); | ||
| 303 | 305 | ||
| 304 | const endian = builtin.cpu.arch.endian(); | 306 | const endian = builtin.cpu.arch.endian(); |
| 305 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 307 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 306 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 308 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 307 | 309 | ||
| 308 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 310 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 309 | for (stage2.items) |i| try writer.writeInt(u8, i, endian); | 311 | for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); |
| 310 | 312 | ||
| 311 | try writer.writeInt(u8, @intCast(stage3.items.len), endian); | 313 | try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian); |
| 312 | for (stage3.items) |i| try writer.writeInt(u8, i, endian); | 314 | for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian); |
| 313 | 315 | ||
| 314 | try out_comp.flush(); | 316 | try writer.interface.flush(); |
| 315 | } | 317 | } |
diff --git a/codegen/upper.zig b/codegen/upper.zig index 5848911..7d96a0e 100644 --- a/codegen/upper.zig +++ b/codegen/upper.zig | |||
| @@ -1,32 +1,29 @@ | |||
| 1 | const std = @import("std"); | 1 | const std = @import("std"); |
| 2 | const builtin = @import("builtin"); | 2 | const builtin = @import("builtin"); |
| 3 | 3 | ||
| 4 | pub fn main() !void { | 4 | pub fn main() anyerror!void { |
| 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 6 | defer arena.deinit(); | 6 | defer arena.deinit(); |
| 7 | const allocator = arena.allocator(); | 7 | const allocator = arena.allocator(); |
| 8 | 8 | ||
| 9 | var line_buf: [4096]u8 = undefined; | ||
| 10 | var write_buf: [4096]u8 = undefined; | ||
| 9 | // Process UnicodeData.txt | 11 | // Process UnicodeData.txt |
| 10 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); | 12 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); |
| 11 | defer in_file.close(); | 13 | defer in_file.close(); |
| 12 | var in_buf = std.io.bufferedReader(in_file.reader()); | 14 | var in_reader = in_file.reader(&line_buf); |
| 13 | const in_reader = in_buf.reader(); | ||
| 14 | 15 | ||
| 15 | var args_iter = try std.process.argsWithAllocator(allocator); | 16 | var args_iter = try std.process.argsWithAllocator(allocator); |
| 16 | defer args_iter.deinit(); | 17 | defer args_iter.deinit(); |
| 17 | _ = args_iter.skip(); | 18 | _ = args_iter.skip(); |
| 18 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 19 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 19 | 20 | ||
| 20 | const compressor = std.compress.flate.deflate.compressor; | ||
| 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 22 | defer out_file.close(); | 22 | defer out_file.close(); |
| 23 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 23 | var file_writer = out_file.writer(&write_buf); |
| 24 | const writer = out_comp.writer(); | ||
| 25 | |||
| 26 | const endian = builtin.cpu.arch.endian(); | 24 | const endian = builtin.cpu.arch.endian(); |
| 27 | var line_buf: [4096]u8 = undefined; | ||
| 28 | 25 | ||
| 29 | lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 26 | lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 30 | if (line.len == 0) continue; | 27 | if (line.len == 0) continue; |
| 31 | 28 | ||
| 32 | var field_iter = std.mem.splitScalar(u8, line, ';'); | 29 | var field_iter = std.mem.splitScalar(u8, line, ';'); |
| @@ -42,16 +39,24 @@ pub fn main() !void { | |||
| 42 | 12 => { | 39 | 12 => { |
| 43 | // Simple uppercase mapping | 40 | // Simple uppercase mapping |
| 44 | if (field.len == 0) continue :lines; | 41 | if (field.len == 0) continue :lines; |
| 45 | try writer.writeInt(i24, cp, endian); | 42 | try file_writer.interface.writeInt(i24, cp, endian); |
| 46 | const mapping = try std.fmt.parseInt(i24, field, 16); | 43 | const mapping = try std.fmt.parseInt(i24, field, 16); |
| 47 | try writer.writeInt(i24, mapping - cp, endian); | 44 | try file_writer.interface.writeInt(i24, mapping - cp, endian); |
| 48 | }, | 45 | }, |
| 49 | 46 | ||
| 50 | else => {}, | 47 | else => {}, |
| 51 | } | 48 | } |
| 52 | } | 49 | } |
| 50 | } else |err| switch (err) { | ||
| 51 | error.ReadFailed => { | ||
| 52 | return in_reader.err orelse err; | ||
| 53 | }, | ||
| 54 | error.EndOfStream => {}, | ||
| 55 | else => { | ||
| 56 | return err; | ||
| 57 | }, | ||
| 53 | } | 58 | } |
| 54 | 59 | ||
| 55 | try writer.writeInt(u24, 0, endian); | 60 | try file_writer.interface.writeInt(u24, 0, endian); |
| 56 | try out_comp.flush(); | 61 | try file_writer.interface.flush(); |
| 57 | } | 62 | } |
diff --git a/codegen/wbp.zig b/codegen/wbp.zig index 741103e..b1ed56a 100644 --- a/codegen/wbp.zig +++ b/codegen/wbp.zig | |||
| @@ -43,7 +43,7 @@ const BlockMap = std.HashMap( | |||
| 43 | std.hash_map.default_max_load_percentage, | 43 | std.hash_map.default_max_load_percentage, |
| 44 | ); | 44 | ); |
| 45 | 45 | ||
| 46 | pub fn main() !void { | 46 | pub fn main() anyerror!void { |
| 47 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | 47 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); |
| 48 | defer arena.deinit(); | 48 | defer arena.deinit(); |
| 49 | const allocator = arena.allocator(); | 49 | const allocator = arena.allocator(); |
| @@ -56,10 +56,9 @@ pub fn main() !void { | |||
| 56 | // Process HangulSyllableType.txt | 56 | // Process HangulSyllableType.txt |
| 57 | var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); | 57 | var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); |
| 58 | defer in_file.close(); | 58 | defer in_file.close(); |
| 59 | var in_buf = std.io.bufferedReader(in_file.reader()); | 59 | var in_reader = in_file.reader(&line_buf); |
| 60 | const in_reader = in_buf.reader(); | ||
| 61 | 60 | ||
| 62 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 61 | while (in_reader.interface.takeDelimiterExclusive('\n')) |line| { |
| 63 | if (line.len == 0 or line[0] == '#') continue; | 62 | if (line.len == 0 or line[0] == '#') continue; |
| 64 | 63 | ||
| 65 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 64 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| @@ -90,15 +89,19 @@ pub fn main() !void { | |||
| 90 | else => {}, | 89 | else => {}, |
| 91 | } | 90 | } |
| 92 | } | 91 | } |
| 92 | } else |err| switch (err) { | ||
| 93 | error.EndOfStream => {}, | ||
| 94 | else => { | ||
| 95 | return err; | ||
| 96 | }, | ||
| 93 | } | 97 | } |
| 94 | |||
| 95 | var blocks_map = BlockMap.init(allocator); | 98 | var blocks_map = BlockMap.init(allocator); |
| 96 | defer blocks_map.deinit(); | 99 | defer blocks_map.deinit(); |
| 97 | 100 | ||
| 98 | var stage1 = std.ArrayList(u16).init(allocator); | 101 | var stage1 = std.array_list.Managed(u16).init(allocator); |
| 99 | defer stage1.deinit(); | 102 | defer stage1.deinit(); |
| 100 | 103 | ||
| 101 | var stage2 = std.ArrayList(u5).init(allocator); | 104 | var stage2 = std.array_list.Managed(u5).init(allocator); |
| 102 | defer stage2.deinit(); | 105 | defer stage2.deinit(); |
| 103 | 106 | ||
| 104 | var block: Block = [_]u5{0} ** block_size; | 107 | var block: Block = [_]u5{0} ** block_size; |
| @@ -129,18 +132,17 @@ pub fn main() !void { | |||
| 129 | _ = args_iter.skip(); | 132 | _ = args_iter.skip(); |
| 130 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | 133 | const output_path = args_iter.next() orelse @panic("No output file arg!"); |
| 131 | 134 | ||
| 132 | const compressor = std.compress.flate.deflate.compressor; | 135 | var write_buf: [4096]u8 = undefined; |
| 133 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | 136 | var out_file = try std.fs.cwd().createFile(output_path, .{}); |
| 134 | defer out_file.close(); | 137 | defer out_file.close(); |
| 135 | var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); | 138 | var writer = out_file.writer(&write_buf); |
| 136 | const writer = out_comp.writer(); | ||
| 137 | 139 | ||
| 138 | const endian = builtin.cpu.arch.endian(); | 140 | const endian = builtin.cpu.arch.endian(); |
| 139 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | 141 | try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian); |
| 140 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | 142 | for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian); |
| 141 | 143 | ||
| 142 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | 144 | try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian); |
| 143 | for (stage2.items) |i| try writer.writeInt(u8, i, endian); | 145 | for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian); |
| 144 | 146 | ||
| 145 | try out_comp.flush(); | 147 | try writer.interface.flush(); |
| 146 | } | 148 | } |
diff --git a/src/CanonData.zig b/src/CanonData.zig index 5d2332a..cf9dc8a 100644 --- a/src/CanonData.zig +++ b/src/CanonData.zig | |||
| @@ -7,11 +7,9 @@ cps: []u21 = undefined, | |||
| 7 | const CanonData = @This(); | 7 | const CanonData = @This(); |
| 8 | 8 | ||
| 9 | pub fn init(allocator: mem.Allocator) !CanonData { | 9 | pub fn init(allocator: mem.Allocator) !CanonData { |
| 10 | const decompressor = compress.flate.inflate.decompressor; | ||
| 11 | const in_bytes = @embedFile("canon"); | 10 | const in_bytes = @embedFile("canon"); |
| 12 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 11 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 13 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 12 | var reader = in_fbs.reader(); |
| 14 | var reader = in_decomp.reader(); | ||
| 15 | 13 | ||
| 16 | const endian = builtin.cpu.arch.endian(); | 14 | const endian = builtin.cpu.arch.endian(); |
| 17 | var cdata = CanonData{ | 15 | var cdata = CanonData{ |
diff --git a/src/CaseFolding.zig b/src/CaseFolding.zig index ff41b3e..df86b92 100644 --- a/src/CaseFolding.zig +++ b/src/CaseFolding.zig | |||
| @@ -48,11 +48,9 @@ fn setupImpl(casefold: *CaseFolding, allocator: Allocator) Allocator.Error!void | |||
| 48 | } | 48 | } |
| 49 | 49 | ||
| 50 | inline fn setupImplInner(casefold: *CaseFolding, allocator: Allocator) !void { | 50 | inline fn setupImplInner(casefold: *CaseFolding, allocator: Allocator) !void { |
| 51 | const decompressor = compress.flate.inflate.decompressor; | ||
| 52 | const in_bytes = @embedFile("fold"); | 51 | const in_bytes = @embedFile("fold"); |
| 53 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 52 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 54 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 53 | var reader = in_fbs.reader(); |
| 55 | var reader = in_decomp.reader(); | ||
| 56 | 54 | ||
| 57 | const endian = builtin.cpu.arch.endian(); | 55 | const endian = builtin.cpu.arch.endian(); |
| 58 | 56 | ||
| @@ -123,7 +121,7 @@ pub fn caseFoldAlloc( | |||
| 123 | allocator: Allocator, | 121 | allocator: Allocator, |
| 124 | cps: []const u21, | 122 | cps: []const u21, |
| 125 | ) Allocator.Error![]const u21 { | 123 | ) Allocator.Error![]const u21 { |
| 126 | var cfcps = std.ArrayList(u21).init(allocator); | 124 | var cfcps = std.array_list.Managed(u21).init(allocator); |
| 127 | defer cfcps.deinit(); | 125 | defer cfcps.deinit(); |
| 128 | var buf: [3]u21 = undefined; | 126 | var buf: [3]u21 = undefined; |
| 129 | 127 | ||
diff --git a/src/CombiningData.zig b/src/CombiningData.zig index fd64a3b..f58e0de 100644 --- a/src/CombiningData.zig +++ b/src/CombiningData.zig | |||
| @@ -6,11 +6,9 @@ s2: []u8 = undefined, | |||
| 6 | const CombiningData = @This(); | 6 | const CombiningData = @This(); |
| 7 | 7 | ||
| 8 | pub fn init(allocator: mem.Allocator) !CombiningData { | 8 | pub fn init(allocator: mem.Allocator) !CombiningData { |
| 9 | const decompressor = compress.flate.inflate.decompressor; | ||
| 10 | const in_bytes = @embedFile("ccc"); | 9 | const in_bytes = @embedFile("ccc"); |
| 11 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 10 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 12 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 11 | var reader = in_fbs.reader(); |
| 13 | var reader = in_decomp.reader(); | ||
| 14 | 12 | ||
| 15 | const endian = builtin.cpu.arch.endian(); | 13 | const endian = builtin.cpu.arch.endian(); |
| 16 | 14 | ||
| @@ -46,5 +44,4 @@ pub fn isStarter(cbdata: CombiningData, cp: u21) bool { | |||
| 46 | 44 | ||
| 47 | const std = @import("std"); | 45 | const std = @import("std"); |
| 48 | const builtin = @import("builtin"); | 46 | const builtin = @import("builtin"); |
| 49 | const compress = std.compress; | ||
| 50 | const mem = std.mem; | 47 | const mem = std.mem; |
diff --git a/src/CompatData.zig b/src/CompatData.zig index 794abca..40ecd12 100644 --- a/src/CompatData.zig +++ b/src/CompatData.zig | |||
| @@ -6,11 +6,9 @@ cps: []u21 = undefined, | |||
| 6 | const CompatData = @This(); | 6 | const CompatData = @This(); |
| 7 | 7 | ||
| 8 | pub fn init(allocator: mem.Allocator) !CompatData { | 8 | pub fn init(allocator: mem.Allocator) !CompatData { |
| 9 | const decompressor = compress.flate.inflate.decompressor; | ||
| 10 | const in_bytes = @embedFile("compat"); | 9 | const in_bytes = @embedFile("compat"); |
| 11 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 10 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 12 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 11 | var reader = in_fbs.reader(); |
| 13 | var reader = in_decomp.reader(); | ||
| 14 | 12 | ||
| 15 | const endian = builtin.cpu.arch.endian(); | 13 | const endian = builtin.cpu.arch.endian(); |
| 16 | var cpdata = CompatData{ | 14 | var cpdata = CompatData{ |
| @@ -55,6 +53,5 @@ pub fn toNfkd(cpdata: *const CompatData, cp: u21) []u21 { | |||
| 55 | 53 | ||
| 56 | const std = @import("std"); | 54 | const std = @import("std"); |
| 57 | const builtin = @import("builtin"); | 55 | const builtin = @import("builtin"); |
| 58 | const compress = std.compress; | ||
| 59 | const mem = std.mem; | 56 | const mem = std.mem; |
| 60 | const magic = @import("magic"); | 57 | const magic = @import("magic"); |
diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig index 3da2d24..82b2649 100644 --- a/src/DisplayWidth.zig +++ b/src/DisplayWidth.zig | |||
| @@ -39,11 +39,9 @@ pub fn setupWithGraphemes(dw: *DisplayWidth, allocator: Allocator, graphemes: Gr | |||
| 39 | 39 | ||
| 40 | // Sets up the DisplayWidthData, leaving the GraphemeData undefined. | 40 | // Sets up the DisplayWidthData, leaving the GraphemeData undefined. |
| 41 | pub fn setup(dw: *DisplayWidth, allocator: Allocator) Allocator.Error!void { | 41 | pub fn setup(dw: *DisplayWidth, allocator: Allocator) Allocator.Error!void { |
| 42 | const decompressor = compress.flate.inflate.decompressor; | ||
| 43 | const in_bytes = @embedFile("dwp"); | 42 | const in_bytes = @embedFile("dwp"); |
| 44 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 43 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 45 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 44 | var reader = in_fbs.reader(); |
| 46 | var reader = in_decomp.reader(); | ||
| 47 | 45 | ||
| 48 | const endian = builtin.cpu.arch.endian(); | 46 | const endian = builtin.cpu.arch.endian(); |
| 49 | 47 | ||
| @@ -400,7 +398,7 @@ pub fn wrap( | |||
| 400 | columns: usize, | 398 | columns: usize, |
| 401 | threshold: usize, | 399 | threshold: usize, |
| 402 | ) ![]u8 { | 400 | ) ![]u8 { |
| 403 | var result = ArrayList(u8).init(allocator); | 401 | var result = std.array_list.Managed(u8).init(allocator); |
| 404 | defer result.deinit(); | 402 | defer result.deinit(); |
| 405 | 403 | ||
| 406 | var line_iter = mem.tokenizeAny(u8, str, "\r\n"); | 404 | var line_iter = mem.tokenizeAny(u8, str, "\r\n"); |
| @@ -460,8 +458,6 @@ test "allocation test" { | |||
| 460 | const std = @import("std"); | 458 | const std = @import("std"); |
| 461 | const builtin = @import("builtin"); | 459 | const builtin = @import("builtin"); |
| 462 | const options = @import("options"); | 460 | const options = @import("options"); |
| 463 | const ArrayList = std.ArrayList; | ||
| 464 | const compress = std.compress; | ||
| 465 | const mem = std.mem; | 461 | const mem = std.mem; |
| 466 | const Allocator = mem.Allocator; | 462 | const Allocator = mem.Allocator; |
| 467 | const simd = std.simd; | 463 | const simd = std.simd; |
diff --git a/src/GeneralCategories.zig b/src/GeneralCategories.zig index 8c1b6a3..eee7e56 100644 --- a/src/GeneralCategories.zig +++ b/src/GeneralCategories.zig | |||
| @@ -47,11 +47,9 @@ pub fn init(allocator: Allocator) Allocator.Error!GeneralCategories { | |||
| 47 | } | 47 | } |
| 48 | 48 | ||
| 49 | pub fn setup(gencat: *GeneralCategories, allocator: Allocator) Allocator.Error!void { | 49 | pub fn setup(gencat: *GeneralCategories, allocator: Allocator) Allocator.Error!void { |
| 50 | const decompressor = compress.flate.inflate.decompressor; | ||
| 51 | const in_bytes = @embedFile("gencat"); | 50 | const in_bytes = @embedFile("gencat"); |
| 52 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 51 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 53 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 52 | var reader = in_fbs.reader(); |
| 54 | var reader = in_decomp.reader(); | ||
| 55 | 53 | ||
| 56 | const endian = builtin.cpu.arch.endian(); | 54 | const endian = builtin.cpu.arch.endian(); |
| 57 | 55 | ||
diff --git a/src/Graphemes.zig b/src/Graphemes.zig index f1c56ed..81d874c 100644 --- a/src/Graphemes.zig +++ b/src/Graphemes.zig | |||
| @@ -16,11 +16,9 @@ pub fn init(allocator: Allocator) Allocator.Error!Graphemes { | |||
| 16 | } | 16 | } |
| 17 | 17 | ||
| 18 | pub fn setup(graphemes: *Graphemes, allocator: Allocator) Allocator.Error!void { | 18 | pub fn setup(graphemes: *Graphemes, allocator: Allocator) Allocator.Error!void { |
| 19 | const decompressor = compress.flate.inflate.decompressor; | ||
| 20 | const in_bytes = @embedFile("gbp"); | 19 | const in_bytes = @embedFile("gbp"); |
| 21 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 20 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 22 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 21 | var reader = in_fbs.reader(); |
| 23 | var reader = in_decomp.reader(); | ||
| 24 | 22 | ||
| 25 | const endian = builtin.cpu.arch.endian(); | 23 | const endian = builtin.cpu.arch.endian(); |
| 26 | 24 | ||
diff --git a/src/HangulData.zig b/src/HangulData.zig index 8c5f3ad..cae8b97 100644 --- a/src/HangulData.zig +++ b/src/HangulData.zig | |||
| @@ -15,11 +15,9 @@ s2: []u3 = undefined, | |||
| 15 | const Hangul = @This(); | 15 | const Hangul = @This(); |
| 16 | 16 | ||
| 17 | pub fn init(allocator: mem.Allocator) !Hangul { | 17 | pub fn init(allocator: mem.Allocator) !Hangul { |
| 18 | const decompressor = compress.flate.inflate.decompressor; | ||
| 19 | const in_bytes = @embedFile("hangul"); | 18 | const in_bytes = @embedFile("hangul"); |
| 20 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 19 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 21 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 20 | var reader = in_fbs.reader(); |
| 22 | var reader = in_decomp.reader(); | ||
| 23 | 21 | ||
| 24 | const endian = builtin.cpu.arch.endian(); | 22 | const endian = builtin.cpu.arch.endian(); |
| 25 | var hangul = Hangul{}; | 23 | var hangul = Hangul{}; |
| @@ -49,6 +47,5 @@ pub fn syllable(hangul: *const Hangul, cp: u21) Syllable { | |||
| 49 | 47 | ||
| 50 | const std = @import("std"); | 48 | const std = @import("std"); |
| 51 | const builtin = @import("builtin"); | 49 | const builtin = @import("builtin"); |
| 52 | const compress = std.compress; | ||
| 53 | const mem = std.mem; | 50 | const mem = std.mem; |
| 54 | const testing = std.testing; | 51 | const testing = std.testing; |
diff --git a/src/LetterCasing.zig b/src/LetterCasing.zig index 11a3e96..33096fc 100644 --- a/src/LetterCasing.zig +++ b/src/LetterCasing.zig | |||
| @@ -22,7 +22,6 @@ pub fn setup(case: *LetterCasing, allocator: Allocator) Allocator.Error!void { | |||
| 22 | } | 22 | } |
| 23 | 23 | ||
| 24 | inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { | 24 | inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { |
| 25 | const decompressor = compress.flate.inflate.decompressor; | ||
| 26 | const endian = builtin.cpu.arch.endian(); | 25 | const endian = builtin.cpu.arch.endian(); |
| 27 | 26 | ||
| 28 | self.case_map = try allocator.alloc([2]u21, 0x110000); | 27 | self.case_map = try allocator.alloc([2]u21, 0x110000); |
| @@ -36,8 +35,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { | |||
| 36 | // Uppercase | 35 | // Uppercase |
| 37 | const upper_bytes = @embedFile("upper"); | 36 | const upper_bytes = @embedFile("upper"); |
| 38 | var upper_fbs = std.io.fixedBufferStream(upper_bytes); | 37 | var upper_fbs = std.io.fixedBufferStream(upper_bytes); |
| 39 | var upper_decomp = decompressor(.raw, upper_fbs.reader()); | 38 | var upper_reader = upper_fbs.reader(); |
| 40 | var upper_reader = upper_decomp.reader(); | ||
| 41 | 39 | ||
| 42 | while (true) { | 40 | while (true) { |
| 43 | const cp = try upper_reader.readInt(i24, endian); | 41 | const cp = try upper_reader.readInt(i24, endian); |
| @@ -49,8 +47,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { | |||
| 49 | // Lowercase | 47 | // Lowercase |
| 50 | const lower_bytes = @embedFile("lower"); | 48 | const lower_bytes = @embedFile("lower"); |
| 51 | var lower_fbs = std.io.fixedBufferStream(lower_bytes); | 49 | var lower_fbs = std.io.fixedBufferStream(lower_bytes); |
| 52 | var lower_decomp = decompressor(.raw, lower_fbs.reader()); | 50 | var lower_reader = lower_fbs.reader(); |
| 53 | var lower_reader = lower_decomp.reader(); | ||
| 54 | 51 | ||
| 55 | while (true) { | 52 | while (true) { |
| 56 | const cp = try lower_reader.readInt(i24, endian); | 53 | const cp = try lower_reader.readInt(i24, endian); |
| @@ -62,8 +59,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { | |||
| 62 | // Case properties | 59 | // Case properties |
| 63 | const cp_bytes = @embedFile("case_prop"); | 60 | const cp_bytes = @embedFile("case_prop"); |
| 64 | var cp_fbs = std.io.fixedBufferStream(cp_bytes); | 61 | var cp_fbs = std.io.fixedBufferStream(cp_bytes); |
| 65 | var cp_decomp = decompressor(.raw, cp_fbs.reader()); | 62 | var cp_reader = cp_fbs.reader(); |
| 66 | var cp_reader = cp_decomp.reader(); | ||
| 67 | 63 | ||
| 68 | const stage_1_len: u16 = try cp_reader.readInt(u16, endian); | 64 | const stage_1_len: u16 = try cp_reader.readInt(u16, endian); |
| 69 | self.prop_s1 = try allocator.alloc(u16, stage_1_len); | 65 | self.prop_s1 = try allocator.alloc(u16, stage_1_len); |
| @@ -122,7 +118,7 @@ pub fn toUpperStr( | |||
| 122 | allocator: mem.Allocator, | 118 | allocator: mem.Allocator, |
| 123 | str: []const u8, | 119 | str: []const u8, |
| 124 | ) ![]u8 { | 120 | ) ![]u8 { |
| 125 | var bytes = std.ArrayList(u8).init(allocator); | 121 | var bytes = std.array_list.Managed(u8).init(allocator); |
| 126 | defer bytes.deinit(); | 122 | defer bytes.deinit(); |
| 127 | 123 | ||
| 128 | var iter = CodePointIterator{ .bytes = str }; | 124 | var iter = CodePointIterator{ .bytes = str }; |
| @@ -180,7 +176,7 @@ pub fn toLowerStr( | |||
| 180 | allocator: mem.Allocator, | 176 | allocator: mem.Allocator, |
| 181 | str: []const u8, | 177 | str: []const u8, |
| 182 | ) ![]u8 { | 178 | ) ![]u8 { |
| 183 | var bytes = std.ArrayList(u8).init(allocator); | 179 | var bytes = std.array_list.Managed(u8).init(allocator); |
| 184 | defer bytes.deinit(); | 180 | defer bytes.deinit(); |
| 185 | 181 | ||
| 186 | var iter = CodePointIterator{ .bytes = str }; | 182 | var iter = CodePointIterator{ .bytes = str }; |
diff --git a/src/NormPropsData.zig b/src/NormPropsData.zig index ca69569..7b53542 100644 --- a/src/NormPropsData.zig +++ b/src/NormPropsData.zig | |||
| @@ -6,11 +6,9 @@ s2: []u4 = undefined, | |||
| 6 | const NormProps = @This(); | 6 | const NormProps = @This(); |
| 7 | 7 | ||
| 8 | pub fn init(allocator: mem.Allocator) !NormProps { | 8 | pub fn init(allocator: mem.Allocator) !NormProps { |
| 9 | const decompressor = compress.flate.inflate.decompressor; | ||
| 10 | const in_bytes = @embedFile("normp"); | 9 | const in_bytes = @embedFile("normp"); |
| 11 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 10 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 12 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 11 | var reader = in_fbs.reader(); |
| 13 | var reader = in_decomp.reader(); | ||
| 14 | 12 | ||
| 15 | const endian = builtin.cpu.arch.endian(); | 13 | const endian = builtin.cpu.arch.endian(); |
| 16 | var norms = NormProps{}; | 14 | var norms = NormProps{}; |
| @@ -50,6 +48,5 @@ pub fn isFcx(norms: *const NormProps, cp: u21) bool { | |||
| 50 | 48 | ||
| 51 | const std = @import("std"); | 49 | const std = @import("std"); |
| 52 | const builtin = @import("builtin"); | 50 | const builtin = @import("builtin"); |
| 53 | const compress = std.compress; | ||
| 54 | const mem = std.mem; | 51 | const mem = std.mem; |
| 55 | const testing = std.testing; | 52 | const testing = std.testing; |
diff --git a/src/Normalize.zig b/src/Normalize.zig index 989ec29..4a1bae8 100644 --- a/src/Normalize.zig +++ b/src/Normalize.zig | |||
| @@ -305,7 +305,7 @@ pub fn nfkd(self: Normalize, allocator: Allocator, str: []const u8) Allocator.Er | |||
| 305 | } | 305 | } |
| 306 | 306 | ||
| 307 | pub fn nfxdCodePoints(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allocator.Error![]u21 { | 307 | pub fn nfxdCodePoints(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allocator.Error![]u21 { |
| 308 | var dcp_list = std.ArrayList(u21).init(allocator); | 308 | var dcp_list = std.array_list.Managed(u21).init(allocator); |
| 309 | defer dcp_list.deinit(); | 309 | defer dcp_list.deinit(); |
| 310 | 310 | ||
| 311 | var cp_iter = CodePointIterator{ .bytes = str }; | 311 | var cp_iter = CodePointIterator{ .bytes = str }; |
| @@ -332,7 +332,7 @@ fn nfxd(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo | |||
| 332 | const dcps = try self.nfxdCodePoints(allocator, str, form); | 332 | const dcps = try self.nfxdCodePoints(allocator, str, form); |
| 333 | defer allocator.free(dcps); | 333 | defer allocator.free(dcps); |
| 334 | 334 | ||
| 335 | var dstr_list = std.ArrayList(u8).init(allocator); | 335 | var dstr_list = std.array_list.Managed(u8).init(allocator); |
| 336 | defer dstr_list.deinit(); | 336 | defer dstr_list.deinit(); |
| 337 | var buf: [4]u8 = undefined; | 337 | var buf: [4]u8 = undefined; |
| 338 | 338 | ||
| @@ -393,7 +393,7 @@ pub fn nfdCodePoints( | |||
| 393 | allocator: Allocator, | 393 | allocator: Allocator, |
| 394 | cps: []const u21, | 394 | cps: []const u21, |
| 395 | ) Allocator.Error![]u21 { | 395 | ) Allocator.Error![]u21 { |
| 396 | var dcp_list = std.ArrayList(u21).init(allocator); | 396 | var dcp_list = std.array_list.Managed(u21).init(allocator); |
| 397 | defer dcp_list.deinit(); | 397 | defer dcp_list.deinit(); |
| 398 | 398 | ||
| 399 | var dc_buf: [18]u21 = undefined; | 399 | var dc_buf: [18]u21 = undefined; |
| @@ -418,7 +418,7 @@ pub fn nfkdCodePoints( | |||
| 418 | allocator: Allocator, | 418 | allocator: Allocator, |
| 419 | cps: []const u21, | 419 | cps: []const u21, |
| 420 | ) Allocator.Error![]u21 { | 420 | ) Allocator.Error![]u21 { |
| 421 | var dcp_list = std.ArrayList(u21).init(allocator); | 421 | var dcp_list = std.array_list.Managed(u21).init(allocator); |
| 422 | defer dcp_list.deinit(); | 422 | defer dcp_list.deinit(); |
| 423 | 423 | ||
| 424 | var dc_buf: [18]u21 = undefined; | 424 | var dc_buf: [18]u21 = undefined; |
| @@ -560,7 +560,7 @@ fn nfxc(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo | |||
| 560 | // If we have no deletions. the code point sequence | 560 | // If we have no deletions. the code point sequence |
| 561 | // has been fully composed. | 561 | // has been fully composed. |
| 562 | if (deleted == 0) { | 562 | if (deleted == 0) { |
| 563 | var cstr_list = std.ArrayList(u8).init(allocator); | 563 | var cstr_list = std.array_list.Managed(u8).init(allocator); |
| 564 | defer cstr_list.deinit(); | 564 | defer cstr_list.deinit(); |
| 565 | var buf: [4]u8 = undefined; | 565 | var buf: [4]u8 = undefined; |
| 566 | 566 | ||
diff --git a/src/Properties.zig b/src/Properties.zig index 73602a0..432d176 100644 --- a/src/Properties.zig +++ b/src/Properties.zig | |||
| @@ -25,14 +25,12 @@ pub fn setup(props: *Properties, allocator: Allocator) Allocator.Error!void { | |||
| 25 | } | 25 | } |
| 26 | 26 | ||
| 27 | inline fn setupInner(props: *Properties, allocator: Allocator) !void { | 27 | inline fn setupInner(props: *Properties, allocator: Allocator) !void { |
| 28 | const decompressor = compress.flate.inflate.decompressor; | ||
| 29 | const endian = builtin.cpu.arch.endian(); | 28 | const endian = builtin.cpu.arch.endian(); |
| 30 | 29 | ||
| 31 | // Process DerivedCoreProperties.txt | 30 | // Process DerivedCoreProperties.txt |
| 32 | const core_bytes = @embedFile("core_props"); | 31 | const core_bytes = @embedFile("core_props"); |
| 33 | var core_fbs = std.io.fixedBufferStream(core_bytes); | 32 | var core_fbs = std.io.fixedBufferStream(core_bytes); |
| 34 | var core_decomp = decompressor(.raw, core_fbs.reader()); | 33 | var core_reader = core_fbs.reader(); |
| 35 | var core_reader = core_decomp.reader(); | ||
| 36 | 34 | ||
| 37 | const core_stage_1_len: u16 = try core_reader.readInt(u16, endian); | 35 | const core_stage_1_len: u16 = try core_reader.readInt(u16, endian); |
| 38 | props.core_s1 = try allocator.alloc(u16, core_stage_1_len); | 36 | props.core_s1 = try allocator.alloc(u16, core_stage_1_len); |
| @@ -47,8 +45,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void { | |||
| 47 | // Process PropList.txt | 45 | // Process PropList.txt |
| 48 | const props_bytes = @embedFile("props"); | 46 | const props_bytes = @embedFile("props"); |
| 49 | var props_fbs = std.io.fixedBufferStream(props_bytes); | 47 | var props_fbs = std.io.fixedBufferStream(props_bytes); |
| 50 | var props_decomp = decompressor(.raw, props_fbs.reader()); | 48 | var props_reader = props_fbs.reader(); |
| 51 | var props_reader = props_decomp.reader(); | ||
| 52 | 49 | ||
| 53 | const stage_1_len: u16 = try props_reader.readInt(u16, endian); | 50 | const stage_1_len: u16 = try props_reader.readInt(u16, endian); |
| 54 | props.props_s1 = try allocator.alloc(u16, stage_1_len); | 51 | props.props_s1 = try allocator.alloc(u16, stage_1_len); |
| @@ -63,8 +60,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void { | |||
| 63 | // Process DerivedNumericType.txt | 60 | // Process DerivedNumericType.txt |
| 64 | const num_bytes = @embedFile("numeric"); | 61 | const num_bytes = @embedFile("numeric"); |
| 65 | var num_fbs = std.io.fixedBufferStream(num_bytes); | 62 | var num_fbs = std.io.fixedBufferStream(num_bytes); |
| 66 | var num_decomp = decompressor(.raw, num_fbs.reader()); | 63 | var num_reader = num_fbs.reader(); |
| 67 | var num_reader = num_decomp.reader(); | ||
| 68 | 64 | ||
| 69 | const num_stage_1_len: u16 = try num_reader.readInt(u16, endian); | 65 | const num_stage_1_len: u16 = try num_reader.readInt(u16, endian); |
| 70 | props.num_s1 = try allocator.alloc(u16, num_stage_1_len); | 66 | props.num_s1 = try allocator.alloc(u16, num_stage_1_len); |
diff --git a/src/Scripts.zig b/src/Scripts.zig index 3bc90bc..719b01f 100644 --- a/src/Scripts.zig +++ b/src/Scripts.zig | |||
| @@ -196,11 +196,9 @@ pub fn setup(scripts: *Scripts, allocator: Allocator) Allocator.Error!void { | |||
| 196 | } | 196 | } |
| 197 | 197 | ||
| 198 | inline fn setupInner(scripts: *Scripts, allocator: mem.Allocator) !void { | 198 | inline fn setupInner(scripts: *Scripts, allocator: mem.Allocator) !void { |
| 199 | const decompressor = compress.flate.inflate.decompressor; | ||
| 200 | const in_bytes = @embedFile("scripts"); | 199 | const in_bytes = @embedFile("scripts"); |
| 201 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 200 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 202 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 201 | var reader = in_fbs.reader(); |
| 203 | var reader = in_decomp.reader(); | ||
| 204 | 202 | ||
| 205 | const endian = builtin.cpu.arch.endian(); | 203 | const endian = builtin.cpu.arch.endian(); |
| 206 | 204 | ||
| @@ -250,7 +248,6 @@ test "Allocation failure" { | |||
| 250 | 248 | ||
| 251 | const std = @import("std"); | 249 | const std = @import("std"); |
| 252 | const builtin = @import("builtin"); | 250 | const builtin = @import("builtin"); |
| 253 | const compress = std.compress; | ||
| 254 | const mem = std.mem; | 251 | const mem = std.mem; |
| 255 | const Allocator = mem.Allocator; | 252 | const Allocator = mem.Allocator; |
| 256 | const testing = std.testing; | 253 | const testing = std.testing; |
diff --git a/src/Words.zig b/src/Words.zig index 617c34d..ce3203f 100644 --- a/src/Words.zig +++ b/src/Words.zig | |||
| @@ -605,11 +605,9 @@ const SneakIterator = struct { | |||
| 605 | }; | 605 | }; |
| 606 | 606 | ||
| 607 | inline fn setupImpl(wb: *Words, allocator: Allocator) !void { | 607 | inline fn setupImpl(wb: *Words, allocator: Allocator) !void { |
| 608 | const decompressor = compress.flate.inflate.decompressor; | ||
| 609 | const in_bytes = @embedFile("wbp"); | 608 | const in_bytes = @embedFile("wbp"); |
| 610 | var in_fbs = std.io.fixedBufferStream(in_bytes); | 609 | var in_fbs = std.io.fixedBufferStream(in_bytes); |
| 611 | var in_decomp = decompressor(.raw, in_fbs.reader()); | 610 | var reader = in_fbs.reader(); |
| 612 | var reader = in_decomp.reader(); | ||
| 613 | 611 | ||
| 614 | const endian = builtin.cpu.arch.endian(); | 612 | const endian = builtin.cpu.arch.endian(); |
| 615 | 613 | ||
diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index ae177a9..ff49b2a 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig | |||
| @@ -3,35 +3,34 @@ const dbg_print = false; | |||
| 3 | test "Unicode normalization tests" { | 3 | test "Unicode normalization tests" { |
| 4 | var arena = heap.ArenaAllocator.init(testing.allocator); | 4 | var arena = heap.ArenaAllocator.init(testing.allocator); |
| 5 | defer arena.deinit(); | 5 | defer arena.deinit(); |
| 6 | var allocator = arena.allocator(); | 6 | const allocator = arena.allocator(); |
| 7 | 7 | ||
| 8 | const n = try Normalize.init(allocator); | 8 | const n = try Normalize.init(allocator); |
| 9 | defer n.deinit(allocator); | 9 | defer n.deinit(allocator); |
| 10 | 10 | ||
| 11 | var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); | 11 | var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); |
| 12 | defer file.close(); | 12 | defer file.close(); |
| 13 | var buf_reader = io.bufferedReader(file.reader()); | ||
| 14 | var input_stream = buf_reader.reader(); | ||
| 15 | |||
| 16 | var buf: [4096]u8 = undefined; | 13 | var buf: [4096]u8 = undefined; |
| 14 | var reader = file.reader(&buf); | ||
| 15 | |||
| 17 | var cp_buf: [4]u8 = undefined; | 16 | var cp_buf: [4]u8 = undefined; |
| 18 | 17 | ||
| 19 | var line_iter: IterRead = .{ .read = &input_stream }; | 18 | var line_iter: IterRead = .{ .read = &reader.interface }; |
| 20 | 19 | ||
| 21 | while (try line_iter.next(&buf)) |line| { | 20 | while (line_iter.next()) |line| { |
| 22 | // Iterate over fields. | 21 | // Iterate over fields. |
| 23 | var fields = mem.splitScalar(u8, line, ';'); | 22 | var fields = mem.splitScalar(u8, line, ';'); |
| 24 | var field_index: usize = 0; | 23 | var field_index: usize = 0; |
| 25 | var input: []u8 = undefined; | 24 | var input: []u8 = undefined; |
| 26 | defer allocator.free(input); | 25 | if (dbg_print) std.debug.print("Line: {s}\n", .{line}); |
| 27 | |||
| 28 | while (fields.next()) |field| : (field_index += 1) { | 26 | while (fields.next()) |field| : (field_index += 1) { |
| 29 | if (field_index == 0) { | 27 | if (field_index == 0) { |
| 30 | var i_buf = std.ArrayList(u8).init(allocator); | 28 | var i_buf = std.array_list.Managed(u8).init(allocator); |
| 31 | defer i_buf.deinit(); | 29 | defer i_buf.deinit(); |
| 32 | 30 | ||
| 33 | var i_fields = mem.splitScalar(u8, field, ' '); | 31 | var i_fields = mem.splitScalar(u8, field, ' '); |
| 34 | while (i_fields.next()) |s| { | 32 | while (i_fields.next()) |s| { |
| 33 | if (dbg_print) std.debug.print("Debug: {s}\n", .{s}); | ||
| 35 | const icp = try fmt.parseInt(u21, s, 16); | 34 | const icp = try fmt.parseInt(u21, s, 16); |
| 36 | const len = try unicode.utf8Encode(icp, &cp_buf); | 35 | const len = try unicode.utf8Encode(icp, &cp_buf); |
| 37 | try i_buf.appendSlice(cp_buf[0..len]); | 36 | try i_buf.appendSlice(cp_buf[0..len]); |
| @@ -41,7 +40,7 @@ test "Unicode normalization tests" { | |||
| 41 | } else if (field_index == 1) { | 40 | } else if (field_index == 1) { |
| 42 | if (dbg_print) debug.print("\n*** {s} ***\n", .{line}); | 41 | if (dbg_print) debug.print("\n*** {s} ***\n", .{line}); |
| 43 | // NFC, time to test. | 42 | // NFC, time to test. |
| 44 | var w_buf = std.ArrayList(u8).init(allocator); | 43 | var w_buf = std.array_list.Managed(u8).init(allocator); |
| 45 | defer w_buf.deinit(); | 44 | defer w_buf.deinit(); |
| 46 | 45 | ||
| 47 | var w_fields = mem.splitScalar(u8, field, ' '); | 46 | var w_fields = mem.splitScalar(u8, field, ' '); |
| @@ -58,7 +57,7 @@ test "Unicode normalization tests" { | |||
| 58 | try testing.expectEqualStrings(want, got.slice); | 57 | try testing.expectEqualStrings(want, got.slice); |
| 59 | } else if (field_index == 2) { | 58 | } else if (field_index == 2) { |
| 60 | // NFD, time to test. | 59 | // NFD, time to test. |
| 61 | var w_buf = std.ArrayList(u8).init(allocator); | 60 | var w_buf = std.array_list.Managed(u8).init(allocator); |
| 62 | defer w_buf.deinit(); | 61 | defer w_buf.deinit(); |
| 63 | 62 | ||
| 64 | var w_fields = mem.splitScalar(u8, field, ' '); | 63 | var w_fields = mem.splitScalar(u8, field, ' '); |
| @@ -75,7 +74,7 @@ test "Unicode normalization tests" { | |||
| 75 | try testing.expectEqualStrings(want, got.slice); | 74 | try testing.expectEqualStrings(want, got.slice); |
| 76 | } else if (field_index == 3) { | 75 | } else if (field_index == 3) { |
| 77 | // NFKC, time to test. | 76 | // NFKC, time to test. |
| 78 | var w_buf = std.ArrayList(u8).init(allocator); | 77 | var w_buf = std.array_list.Managed(u8).init(allocator); |
| 79 | defer w_buf.deinit(); | 78 | defer w_buf.deinit(); |
| 80 | 79 | ||
| 81 | var w_fields = mem.splitScalar(u8, field, ' '); | 80 | var w_fields = mem.splitScalar(u8, field, ' '); |
| @@ -92,7 +91,7 @@ test "Unicode normalization tests" { | |||
| 92 | try testing.expectEqualStrings(want, got.slice); | 91 | try testing.expectEqualStrings(want, got.slice); |
| 93 | } else if (field_index == 4) { | 92 | } else if (field_index == 4) { |
| 94 | // NFKD, time to test. | 93 | // NFKD, time to test. |
| 95 | var w_buf = std.ArrayList(u8).init(allocator); | 94 | var w_buf = std.array_list.Managed(u8).init(allocator); |
| 96 | defer w_buf.deinit(); | 95 | defer w_buf.deinit(); |
| 97 | 96 | ||
| 98 | var w_fields = mem.splitScalar(u8, field, ' '); | 97 | var w_fields = mem.splitScalar(u8, field, ' '); |
| @@ -111,6 +110,11 @@ test "Unicode normalization tests" { | |||
| 111 | continue; | 110 | continue; |
| 112 | } | 111 | } |
| 113 | } | 112 | } |
| 113 | } else |err| switch (err) { | ||
| 114 | error.EndOfStream => {}, | ||
| 115 | else => { | ||
| 116 | return err; | ||
| 117 | }, | ||
| 114 | } | 118 | } |
| 115 | } | 119 | } |
| 116 | 120 | ||
| @@ -118,26 +122,25 @@ test "Segmentation GraphemeIterator" { | |||
| 118 | const allocator = std.testing.allocator; | 122 | const allocator = std.testing.allocator; |
| 119 | var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{}); | 123 | var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{}); |
| 120 | defer file.close(); | 124 | defer file.close(); |
| 121 | var buf_reader = std.io.bufferedReader(file.reader()); | 125 | var buf: [4096]u8 = undefined; |
| 122 | var input_stream = buf_reader.reader(); | 126 | var reader = file.reader(&buf); |
| 123 | 127 | ||
| 124 | const graph = try Graphemes.init(allocator); | 128 | const graph = try Graphemes.init(allocator); |
| 125 | defer graph.deinit(allocator); | 129 | defer graph.deinit(allocator); |
| 126 | 130 | ||
| 127 | var buf: [4096]u8 = undefined; | 131 | var line_iter: IterRead = .{ .read = &reader.interface }; |
| 128 | var line_iter: IterRead = .{ .read = &input_stream }; | ||
| 129 | 132 | ||
| 130 | while (try line_iter.next(&buf)) |raw| { | 133 | while (line_iter.next()) |raw| { |
| 131 | // Clean up. | 134 | // Clean up. |
| 132 | var line = std.mem.trimLeft(u8, raw, "÷ "); | 135 | var line = std.mem.trimLeft(u8, raw, "÷ "); |
| 133 | if (std.mem.indexOf(u8, line, " ÷\t")) |final| { | 136 | if (std.mem.indexOf(u8, line, " ÷\t")) |final| { |
| 134 | line = line[0..final]; | 137 | line = line[0..final]; |
| 135 | } | 138 | } |
| 136 | // Iterate over fields. | 139 | // Iterate over fields. |
| 137 | var want = std.ArrayList(Grapheme).init(allocator); | 140 | var want = std.array_list.Managed(Grapheme).init(allocator); |
| 138 | defer want.deinit(); | 141 | defer want.deinit(); |
| 139 | 142 | ||
| 140 | var all_bytes = std.ArrayList(u8).init(allocator); | 143 | var all_bytes = std.array_list.Managed(u8).init(allocator); |
| 141 | defer all_bytes.deinit(); | 144 | defer all_bytes.deinit(); |
| 142 | 145 | ||
| 143 | var graphemes = std.mem.splitSequence(u8, line, " ÷ "); | 146 | var graphemes = std.mem.splitSequence(u8, line, " ÷ "); |
| @@ -250,6 +253,11 @@ test "Segmentation GraphemeIterator" { | |||
| 250 | } | 253 | } |
| 251 | } | 254 | } |
| 252 | } | 255 | } |
| 256 | } else |err| switch (err) { | ||
| 257 | error.EndOfStream => {}, | ||
| 258 | else => { | ||
| 259 | return err; | ||
| 260 | }, | ||
| 253 | } | 261 | } |
| 254 | } | 262 | } |
| 255 | 263 | ||
| @@ -257,26 +265,25 @@ test "Segmentation Word Iterator" { | |||
| 257 | const allocator = std.testing.allocator; | 265 | const allocator = std.testing.allocator; |
| 258 | var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{}); | 266 | var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{}); |
| 259 | defer file.close(); | 267 | defer file.close(); |
| 260 | var buf_reader = std.io.bufferedReader(file.reader()); | 268 | var buf: [4096]u8 = undefined; |
| 261 | var input_stream = buf_reader.reader(); | 269 | var reader = file.reader(&buf); |
| 262 | 270 | ||
| 263 | const wb = try Words.init(allocator); | 271 | const wb = try Words.init(allocator); |
| 264 | defer wb.deinit(allocator); | 272 | defer wb.deinit(allocator); |
| 265 | 273 | ||
| 266 | var buf: [4096]u8 = undefined; | 274 | var line_iter: IterRead = .{ .read = &reader.interface }; |
| 267 | var line_iter: IterRead = .{ .read = &input_stream }; | ||
| 268 | 275 | ||
| 269 | while (try line_iter.next(&buf)) |raw| { | 276 | while (line_iter.next()) |raw| { |
| 270 | // Clean up. | 277 | // Clean up. |
| 271 | var line = std.mem.trimLeft(u8, raw, "÷ "); | 278 | var line = std.mem.trimLeft(u8, raw, "÷ "); |
| 272 | if (std.mem.indexOf(u8, line, " ÷\t")) |final| { | 279 | if (std.mem.indexOf(u8, line, " ÷\t")) |final| { |
| 273 | line = line[0..final]; | 280 | line = line[0..final]; |
| 274 | } | 281 | } |
| 275 | // Iterate over fields. | 282 | // Iterate over fields. |
| 276 | var want = std.ArrayList(Word).init(allocator); | 283 | var want = std.array_list.Managed(Word).init(allocator); |
| 277 | defer want.deinit(); | 284 | defer want.deinit(); |
| 278 | 285 | ||
| 279 | var all_bytes = std.ArrayList(u8).init(allocator); | 286 | var all_bytes = std.array_list.Managed(u8).init(allocator); |
| 280 | defer all_bytes.deinit(); | 287 | defer all_bytes.deinit(); |
| 281 | 288 | ||
| 282 | var words = std.mem.splitSequence(u8, line, " ÷ "); | 289 | var words = std.mem.splitSequence(u8, line, " ÷ "); |
| @@ -439,26 +446,26 @@ test "Segmentation Word Iterator" { | |||
| 439 | if (idx == 0) break; | 446 | if (idx == 0) break; |
| 440 | } | 447 | } |
| 441 | } | 448 | } |
| 449 | } else |err| switch (err) { | ||
| 450 | error.EndOfStream => {}, | ||
| 451 | else => { | ||
| 452 | return err; | ||
| 453 | }, | ||
| 442 | } | 454 | } |
| 443 | } | 455 | } |
| 444 | 456 | ||
| 445 | const IterRead = struct { | 457 | const IterRead = struct { |
| 446 | read: *Reader, | 458 | read: *io.Reader, |
| 447 | line: usize = 0, | 459 | line: usize = 0, |
| 448 | 460 | ||
| 449 | pub fn next(iter: *IterRead, buf: []u8) !?[]const u8 { | 461 | pub fn next(iter: *IterRead) anyerror![]const u8 { |
| 450 | defer iter.line += 1; | 462 | iter.line += 1; |
| 451 | const maybe_line = try iter.read.readUntilDelimiterOrEof(buf, '#'); | 463 | const this_line = try iter.read.takeDelimiterExclusive('\n'); |
| 452 | if (maybe_line) |this_line| { | 464 | if (this_line.len == 0 or this_line[0] == '@' or this_line[0] == '#') { |
| 453 | try iter.read.skipUntilDelimiterOrEof('\n'); | 465 | // comment, next line |
| 454 | if (this_line.len == 0 or this_line[0] == '@') { | 466 | return iter.next(); |
| 455 | // comment, next line | ||
| 456 | return iter.next(buf); | ||
| 457 | } else { | ||
| 458 | return this_line; | ||
| 459 | } | ||
| 460 | } else { | 467 | } else { |
| 461 | return null; | 468 | return this_line; |
| 462 | } | 469 | } |
| 463 | } | 470 | } |
| 464 | }; | 471 | }; |
| @@ -467,7 +474,6 @@ const std = @import("std"); | |||
| 467 | const fmt = std.fmt; | 474 | const fmt = std.fmt; |
| 468 | const fs = std.fs; | 475 | const fs = std.fs; |
| 469 | const io = std.io; | 476 | const io = std.io; |
| 470 | const Reader = io.BufferedReader(4096, fs.File.Reader).Reader; | ||
| 471 | const heap = std.heap; | 477 | const heap = std.heap; |
| 472 | const mem = std.mem; | 478 | const mem = std.mem; |
| 473 | const debug = std.debug; | 479 | const debug = std.debug; |