summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGravatar Michael Chaten2025-09-13 08:38:24 -0700
committerGravatar Michael Chaten2025-09-14 01:48:41 -0700
commita3b5e884b12fdaa341010ef41bb9382fa0cd89f8 (patch)
tree8bdd22c1c54da0b56ef6b9fe0dc4c2ca667b71a3
parentUpdate build to 0.15.0 (diff)
downloadzg-a3b5e884b12fdaa341010ef41bb9382fa0cd89f8.tar.gz
zg-a3b5e884b12fdaa341010ef41bb9382fa0cd89f8.tar.xz
zg-a3b5e884b12fdaa341010ef41bb9382fa0cd89f8.zip
Update codebase to Zig 0.15.1.
Removes compression support
-rw-r--r--build.zig.zon4
-rw-r--r--codegen/canon.zig24
-rw-r--r--codegen/case_prop.zig31
-rw-r--r--codegen/ccc.zig31
-rw-r--r--codegen/compat.zig27
-rw-r--r--codegen/core_props.zig32
-rw-r--r--codegen/dwp.zig42
-rw-r--r--codegen/fold.zig59
-rw-r--r--codegen/gbp.zig57
-rw-r--r--codegen/gencat.zig35
-rw-r--r--codegen/hangul.zig32
-rw-r--r--codegen/lower.zig26
-rw-r--r--codegen/normp.zig32
-rw-r--r--codegen/numeric.zig31
-rw-r--r--codegen/props.zig32
-rw-r--r--codegen/scripts.zig38
-rw-r--r--codegen/upper.zig31
-rw-r--r--codegen/wbp.zig32
-rw-r--r--src/CanonData.zig4
-rw-r--r--src/CaseFolding.zig6
-rw-r--r--src/CombiningData.zig5
-rw-r--r--src/CompatData.zig5
-rw-r--r--src/DisplayWidth.zig8
-rw-r--r--src/GeneralCategories.zig4
-rw-r--r--src/Graphemes.zig4
-rw-r--r--src/HangulData.zig5
-rw-r--r--src/LetterCasing.zig14
-rw-r--r--src/NormPropsData.zig5
-rw-r--r--src/Normalize.zig10
-rw-r--r--src/Properties.zig10
-rw-r--r--src/Scripts.zig5
-rw-r--r--src/Words.zig4
-rw-r--r--src/unicode_tests.zig88
33 files changed, 399 insertions, 374 deletions
diff --git a/build.zig.zon b/build.zig.zon
index 3e1df95..e585449 100644
--- a/build.zig.zon
+++ b/build.zig.zon
@@ -1,7 +1,7 @@
1.{ 1.{
2 .name = .zg, 2 .name = .zg,
3 .version = "0.14.1", 3 .version = "0.15.1",
4 .minimum_zig_version = "0.14.0", 4 .minimum_zig_version = "0.15.1",
5 .fingerprint = 0x47df7778dc946aa0, 5 .fingerprint = 0x47df7778dc946aa0,
6 6
7 .paths = .{ 7 .paths = .{
diff --git a/codegen/canon.zig b/codegen/canon.zig
index 28b7f28..89a9070 100644
--- a/codegen/canon.zig
+++ b/codegen/canon.zig
@@ -1,32 +1,30 @@
1const std = @import("std"); 1const std = @import("std");
2const builtin = @import("builtin"); 2const builtin = @import("builtin");
3 3
4pub fn main() !void { 4pub fn main() anyerror!void {
5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
6 defer arena.deinit(); 6 defer arena.deinit();
7 const allocator = arena.allocator(); 7 const allocator = arena.allocator();
8 8
9 var line_buf: [4096]u8 = undefined;
10 var write_buf: [4096]u8 = undefined;
9 // Process UnicodeData.txt 11 // Process UnicodeData.txt
10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); 12 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{});
11 defer in_file.close(); 13 defer in_file.close();
12 var in_buf = std.io.bufferedReader(in_file.reader()); 14 var in_reader = in_file.reader(&line_buf);
13 const in_reader = in_buf.reader();
14 15
15 var args_iter = try std.process.argsWithAllocator(allocator); 16 var args_iter = try std.process.argsWithAllocator(allocator);
16 defer args_iter.deinit(); 17 defer args_iter.deinit();
17 _ = args_iter.skip(); 18 _ = args_iter.skip();
18 const output_path = args_iter.next() orelse @panic("No output file arg!"); 19 const output_path = args_iter.next() orelse @panic("No output file arg!");
19 20
20 const compressor = std.compress.flate.deflate.compressor;
21 var out_file = try std.fs.cwd().createFile(output_path, .{}); 21 var out_file = try std.fs.cwd().createFile(output_path, .{});
22 defer out_file.close(); 22 defer out_file.close();
23 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 23 var file_writer = out_file.writer(&write_buf);
24 const writer = out_comp.writer(); 24 var writer = &file_writer.interface;
25
26 const endian = builtin.cpu.arch.endian(); 25 const endian = builtin.cpu.arch.endian();
27 var line_buf: [4096]u8 = undefined;
28 26
29 lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 27 lines: while (in_reader.interface.takeDelimiterInclusive('\n')) |line| {
30 if (line.len == 0) continue; 28 if (line.len == 0) continue;
31 29
32 var field_iter = std.mem.splitScalar(u8, line, ';'); 30 var field_iter = std.mem.splitScalar(u8, line, ';');
@@ -60,8 +58,12 @@ pub fn main() !void {
60 58
61 try writer.writeInt(u8, @intCast(len), endian); 59 try writer.writeInt(u8, @intCast(len), endian);
62 for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); 60 for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian);
61 } else |err| switch (err) {
62 error.EndOfStream => {},
63 else => {
64 return err;
65 },
63 } 66 }
64
65 try writer.writeInt(u16, 0, endian); 67 try writer.writeInt(u16, 0, endian);
66 try out_comp.flush(); 68 try writer.flush();
67} 69}
diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig
index 6c912a8..66eb62c 100644
--- a/codegen/case_prop.zig
+++ b/codegen/case_prop.zig
@@ -22,7 +22,7 @@ const BlockMap = std.HashMap(
22 std.hash_map.default_max_load_percentage, 22 std.hash_map.default_max_load_percentage,
23); 23);
24 24
25pub fn main() !void { 25pub fn main() anyerror!void {
26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
27 defer arena.deinit(); 27 defer arena.deinit();
28 const allocator = arena.allocator(); 28 const allocator = arena.allocator();
@@ -35,10 +35,9 @@ pub fn main() !void {
35 // Process DerivedCoreProperties.txt 35 // Process DerivedCoreProperties.txt
36 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); 36 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{});
37 defer in_file.close(); 37 defer in_file.close();
38 var in_buf = std.io.bufferedReader(in_file.reader()); 38 var in_reader = in_file.reader(&line_buf);
39 const in_reader = in_buf.reader();
40 39
41 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 40 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
42 if (line.len == 0 or line[0] == '#') continue; 41 if (line.len == 0 or line[0] == '#') continue;
43 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 42 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
44 43
@@ -79,15 +78,20 @@ pub fn main() !void {
79 else => {}, 78 else => {},
80 } 79 }
81 } 80 }
81 } else |err| switch (err) {
82 error.EndOfStream => {},
83 else => {
84 return err;
85 },
82 } 86 }
83 87
84 var blocks_map = BlockMap.init(allocator); 88 var blocks_map = BlockMap.init(allocator);
85 defer blocks_map.deinit(); 89 defer blocks_map.deinit();
86 90
87 var stage1 = std.ArrayList(u16).init(allocator); 91 var stage1 = std.array_list.Managed(u16).init(allocator);
88 defer stage1.deinit(); 92 defer stage1.deinit();
89 93
90 var stage2 = std.ArrayList(u8).init(allocator); 94 var stage2 = std.array_list.Managed(u8).init(allocator);
91 defer stage2.deinit(); 95 defer stage2.deinit();
92 96
93 var block: Block = [_]u8{0} ** block_size; 97 var block: Block = [_]u8{0} ** block_size;
@@ -118,18 +122,17 @@ pub fn main() !void {
118 _ = args_iter.skip(); 122 _ = args_iter.skip();
119 const output_path = args_iter.next() orelse @panic("No output file arg!"); 123 const output_path = args_iter.next() orelse @panic("No output file arg!");
120 124
121 const compressor = std.compress.flate.deflate.compressor; 125 var write_buf: [4096]u8 = undefined;
122 var out_file = try std.fs.cwd().createFile(output_path, .{}); 126 var out_file = try std.fs.cwd().createFile(output_path, .{});
123 defer out_file.close(); 127 defer out_file.close();
124 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 128 var writer = out_file.writer(&write_buf);
125 const writer = out_comp.writer();
126 129
127 const endian = builtin.cpu.arch.endian(); 130 const endian = builtin.cpu.arch.endian();
128 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 131 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
129 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 132 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
130 133
131 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 134 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
132 try writer.writeAll(stage2.items); 135 try writer.interface.writeAll(stage2.items);
133 136
134 try out_comp.flush(); 137 try writer.interface.flush();
135} 138}
diff --git a/codegen/ccc.zig b/codegen/ccc.zig
index a01c8d2..b9b2bc3 100644
--- a/codegen/ccc.zig
+++ b/codegen/ccc.zig
@@ -21,7 +21,7 @@ const BlockMap = std.HashMap(
21 std.hash_map.default_max_load_percentage, 21 std.hash_map.default_max_load_percentage,
22); 22);
23 23
24pub fn main() !void { 24pub fn main() anyerror!void {
25 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 25 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
26 defer arena.deinit(); 26 defer arena.deinit();
27 const allocator = arena.allocator(); 27 const allocator = arena.allocator();
@@ -34,10 +34,9 @@ pub fn main() !void {
34 // Process DerivedCombiningClass.txt 34 // Process DerivedCombiningClass.txt
35 var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); 35 var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{});
36 defer cc_file.close(); 36 defer cc_file.close();
37 var cc_buf = std.io.bufferedReader(cc_file.reader()); 37 var cc_reader = cc_file.reader(&line_buf);
38 const cc_reader = cc_buf.reader();
39 38
40 while (try cc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 39 while (cc_reader.interface.takeDelimiterExclusive('\n')) |line| {
41 if (line.len == 0 or line[0] == '#') continue; 40 if (line.len == 0 or line[0] == '#') continue;
42 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 41 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
43 42
@@ -68,15 +67,20 @@ pub fn main() !void {
68 else => {}, 67 else => {},
69 } 68 }
70 } 69 }
70 } else |err| switch (err) {
71 error.EndOfStream => {},
72 else => {
73 return err;
74 },
71 } 75 }
72 76
73 var blocks_map = BlockMap.init(allocator); 77 var blocks_map = BlockMap.init(allocator);
74 defer blocks_map.deinit(); 78 defer blocks_map.deinit();
75 79
76 var stage1 = std.ArrayList(u16).init(allocator); 80 var stage1 = std.array_list.Managed(u16).init(allocator);
77 defer stage1.deinit(); 81 defer stage1.deinit();
78 82
79 var stage2 = std.ArrayList(u8).init(allocator); 83 var stage2 = std.array_list.Managed(u8).init(allocator);
80 defer stage2.deinit(); 84 defer stage2.deinit();
81 85
82 var block: Block = [_]u8{0} ** block_size; 86 var block: Block = [_]u8{0} ** block_size;
@@ -107,18 +111,17 @@ pub fn main() !void {
107 _ = args_iter.skip(); 111 _ = args_iter.skip();
108 const output_path = args_iter.next() orelse @panic("No output file arg!"); 112 const output_path = args_iter.next() orelse @panic("No output file arg!");
109 113
110 const compressor = std.compress.flate.deflate.compressor; 114 var write_buf: [4096]u8 = undefined;
111 var out_file = try std.fs.cwd().createFile(output_path, .{}); 115 var out_file = try std.fs.cwd().createFile(output_path, .{});
112 defer out_file.close(); 116 defer out_file.close();
113 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 117 var writer = out_file.writer(&write_buf);
114 const writer = out_comp.writer();
115 118
116 const endian = builtin.cpu.arch.endian(); 119 const endian = builtin.cpu.arch.endian();
117 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 120 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
118 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 121 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
119 122
120 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 123 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
121 try writer.writeAll(stage2.items); 124 try writer.interface.writeAll(stage2.items);
122 125
123 try out_comp.flush(); 126 try writer.interface.flush();
124} 127}
diff --git a/codegen/compat.zig b/codegen/compat.zig
index 07616fc..835a636 100644
--- a/codegen/compat.zig
+++ b/codegen/compat.zig
@@ -1,7 +1,7 @@
1const std = @import("std"); 1const std = @import("std");
2const builtin = @import("builtin"); 2const builtin = @import("builtin");
3 3
4pub fn main() !void { 4pub fn main() anyerror!void {
5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
6 defer arena.deinit(); 6 defer arena.deinit();
7 const allocator = arena.allocator(); 7 const allocator = arena.allocator();
@@ -9,24 +9,22 @@ pub fn main() !void {
9 // Process UnicodeData.txt 9 // Process UnicodeData.txt
10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); 10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{});
11 defer in_file.close(); 11 defer in_file.close();
12 var in_buf = std.io.bufferedReader(in_file.reader()); 12 var line_buf: [4096]u8 = undefined;
13 const in_reader = in_buf.reader(); 13 var write_buf: [4096]u8 = undefined;
14 var in_reader = in_file.reader(&line_buf);
14 15
15 var args_iter = try std.process.argsWithAllocator(allocator); 16 var args_iter = try std.process.argsWithAllocator(allocator);
16 defer args_iter.deinit(); 17 defer args_iter.deinit();
17 _ = args_iter.skip(); 18 _ = args_iter.skip();
18 const output_path = args_iter.next() orelse @panic("No output file arg!"); 19 const output_path = args_iter.next() orelse @panic("No output file arg!");
19 20
20 const compressor = std.compress.flate.deflate.compressor;
21 var out_file = try std.fs.cwd().createFile(output_path, .{}); 21 var out_file = try std.fs.cwd().createFile(output_path, .{});
22 defer out_file.close(); 22 defer out_file.close();
23 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 23 var writer = out_file.writer(&write_buf);
24 const writer = out_comp.writer();
25 24
26 const endian = builtin.cpu.arch.endian(); 25 const endian = builtin.cpu.arch.endian();
27 var line_buf: [4096]u8 = undefined;
28 26
29 lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 27 lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
30 if (line.len == 0) continue; 28 if (line.len == 0) continue;
31 29
32 var field_iter = std.mem.splitScalar(u8, line, ';'); 30 var field_iter = std.mem.splitScalar(u8, line, ';');
@@ -55,10 +53,15 @@ pub fn main() !void {
55 } 53 }
56 } 54 }
57 55
58 try writer.writeInt(u8, @intCast(len), endian); 56 try writer.interface.writeInt(u8, @intCast(len), endian);
59 for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); 57 for (cps[0..len]) |cp| try writer.interface.writeInt(u24, cp, endian);
58 } else |err| switch (err) {
59 error.EndOfStream => {},
60 else => {
61 return err;
62 },
60 } 63 }
61 64
62 try writer.writeInt(u16, 0, endian); 65 try writer.interface.writeInt(u16, 0, endian);
63 try out_comp.flush(); 66 try writer.interface.flush();
64} 67}
diff --git a/codegen/core_props.zig b/codegen/core_props.zig
index f60c7a9..19063f6 100644
--- a/codegen/core_props.zig
+++ b/codegen/core_props.zig
@@ -22,7 +22,7 @@ const BlockMap = std.HashMap(
22 std.hash_map.default_max_load_percentage, 22 std.hash_map.default_max_load_percentage,
23); 23);
24 24
25pub fn main() !void { 25pub fn main() anyerror!void {
26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
27 defer arena.deinit(); 27 defer arena.deinit();
28 const allocator = arena.allocator(); 28 const allocator = arena.allocator();
@@ -35,10 +35,9 @@ pub fn main() !void {
35 // Process DerivedCoreProperties.txt 35 // Process DerivedCoreProperties.txt
36 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); 36 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{});
37 defer in_file.close(); 37 defer in_file.close();
38 var in_buf = std.io.bufferedReader(in_file.reader()); 38 var in_reader = in_file.reader(&line_buf);
39 const in_reader = in_buf.reader();
40 39
41 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 40 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
42 if (line.len == 0 or line[0] == '#') continue; 41 if (line.len == 0 or line[0] == '#') continue;
43 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 42 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
44 43
@@ -82,15 +81,19 @@ pub fn main() !void {
82 else => {}, 81 else => {},
83 } 82 }
84 } 83 }
84 } else |err| switch (err) {
85 error.EndOfStream => {},
86 else => {
87 return err;
88 },
85 } 89 }
86
87 var blocks_map = BlockMap.init(allocator); 90 var blocks_map = BlockMap.init(allocator);
88 defer blocks_map.deinit(); 91 defer blocks_map.deinit();
89 92
90 var stage1 = std.ArrayList(u16).init(allocator); 93 var stage1 = std.array_list.Managed(u16).init(allocator);
91 defer stage1.deinit(); 94 defer stage1.deinit();
92 95
93 var stage2 = std.ArrayList(u8).init(allocator); 96 var stage2 = std.array_list.Managed(u8).init(allocator);
94 defer stage2.deinit(); 97 defer stage2.deinit();
95 98
96 var block: Block = [_]u8{0} ** block_size; 99 var block: Block = [_]u8{0} ** block_size;
@@ -121,18 +124,17 @@ pub fn main() !void {
121 _ = args_iter.skip(); 124 _ = args_iter.skip();
122 const output_path = args_iter.next() orelse @panic("No output file arg!"); 125 const output_path = args_iter.next() orelse @panic("No output file arg!");
123 126
124 const compressor = std.compress.flate.deflate.compressor; 127 var out_buf: [4096]u8 = undefined;
125 var out_file = try std.fs.cwd().createFile(output_path, .{}); 128 var out_file = try std.fs.cwd().createFile(output_path, .{});
126 defer out_file.close(); 129 defer out_file.close();
127 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 130 var writer = out_file.writer(&out_buf);
128 const writer = out_comp.writer();
129 131
130 const endian = builtin.cpu.arch.endian(); 132 const endian = builtin.cpu.arch.endian();
131 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 133 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
132 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 134 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
133 135
134 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 136 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
135 try writer.writeAll(stage2.items); 137 try writer.interface.writeAll(stage2.items);
136 138
137 try out_comp.flush(); 139 try writer.interface.flush();
138} 140}
diff --git a/codegen/dwp.zig b/codegen/dwp.zig
index 5e5bf6a..fad08d1 100644
--- a/codegen/dwp.zig
+++ b/codegen/dwp.zig
@@ -23,7 +23,7 @@ const BlockMap = std.HashMap(
23 std.hash_map.default_max_load_percentage, 23 std.hash_map.default_max_load_percentage,
24); 24);
25 25
26pub fn main() !void { 26pub fn main() anyerror!void {
27 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 27 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
28 defer arena.deinit(); 28 defer arena.deinit();
29 const allocator = arena.allocator(); 29 const allocator = arena.allocator();
@@ -36,10 +36,9 @@ pub fn main() !void {
36 // Process DerivedEastAsianWidth.txt 36 // Process DerivedEastAsianWidth.txt
37 var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); 37 var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{});
38 defer deaw_file.close(); 38 defer deaw_file.close();
39 var deaw_buf = std.io.bufferedReader(deaw_file.reader()); 39 var deaw_reader = deaw_file.reader(&line_buf);
40 const deaw_reader = deaw_buf.reader();
41 40
42 while (try deaw_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 41 while (deaw_reader.interface.takeDelimiterExclusive('\n')) |line| {
43 if (line.len == 0) continue; 42 if (line.len == 0) continue;
44 43
45 // @missing ranges 44 // @missing ranges
@@ -88,15 +87,18 @@ pub fn main() !void {
88 else => {}, 87 else => {},
89 } 88 }
90 } 89 }
90 } else |err| switch (err) {
91 error.EndOfStream => {},
92 else => {
93 return err;
94 },
91 } 95 }
92
93 // Process DerivedGeneralCategory.txt 96 // Process DerivedGeneralCategory.txt
94 var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); 97 var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{});
95 defer dgc_file.close(); 98 defer dgc_file.close();
96 var dgc_buf = std.io.bufferedReader(dgc_file.reader()); 99 var dgc_reader = dgc_file.reader(&line_buf);
97 const dgc_reader = dgc_buf.reader();
98 100
99 while (try dgc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 101 while (dgc_reader.interface.takeDelimiterExclusive('\n')) |line| {
100 if (line.len == 0 or line[0] == '#') continue; 102 if (line.len == 0 or line[0] == '#') continue;
101 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 103 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
102 104
@@ -139,15 +141,20 @@ pub fn main() !void {
139 else => {}, 141 else => {},
140 } 142 }
141 } 143 }
144 } else |err| switch (err) {
145 error.EndOfStream => {},
146 else => {
147 return err;
148 },
142 } 149 }
143 150
144 var blocks_map = BlockMap.init(allocator); 151 var blocks_map = BlockMap.init(allocator);
145 defer blocks_map.deinit(); 152 defer blocks_map.deinit();
146 153
147 var stage1 = std.ArrayList(u16).init(allocator); 154 var stage1 = std.array_list.Managed(u16).init(allocator);
148 defer stage1.deinit(); 155 defer stage1.deinit();
149 156
150 var stage2 = std.ArrayList(i4).init(allocator); 157 var stage2 = std.array_list.Managed(i4).init(allocator);
151 defer stage2.deinit(); 158 defer stage2.deinit();
152 159
153 var block: Block = [_]i4{0} ** block_size; 160 var block: Block = [_]i4{0} ** block_size;
@@ -227,18 +234,17 @@ pub fn main() !void {
227 _ = args_iter.skip(); 234 _ = args_iter.skip();
228 const output_path = args_iter.next() orelse @panic("No output file arg!"); 235 const output_path = args_iter.next() orelse @panic("No output file arg!");
229 236
230 const compressor = std.compress.flate.deflate.compressor; 237 var write_buf: [4096]u8 = undefined;
231 var out_file = try std.fs.cwd().createFile(output_path, .{}); 238 var out_file = try std.fs.cwd().createFile(output_path, .{});
232 defer out_file.close(); 239 defer out_file.close();
233 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 240 var writer = out_file.writer(&write_buf);
234 const writer = out_comp.writer();
235 241
236 const endian = builtin.cpu.arch.endian(); 242 const endian = builtin.cpu.arch.endian();
237 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 243 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
238 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 244 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
239 245
240 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 246 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
241 for (stage2.items) |i| try writer.writeInt(i8, i, endian); 247 for (stage2.items) |i| try writer.interface.writeInt(i8, i, endian);
242 248
243 try out_comp.flush(); 249 try writer.interface.flush();
244} 250}
diff --git a/codegen/fold.zig b/codegen/fold.zig
index cb73cca..cc381a8 100644
--- a/codegen/fold.zig
+++ b/codegen/fold.zig
@@ -2,23 +2,21 @@ const std = @import("std");
2const builtin = @import("builtin"); 2const builtin = @import("builtin");
3const mem = std.mem; 3const mem = std.mem;
4 4
5pub fn main() !void { 5pub fn main() anyerror!void {
6 var gpa = std.heap.GeneralPurposeAllocator(.{}){}; 6 var gpa = std.heap.GeneralPurposeAllocator(.{}){};
7 defer std.debug.assert(gpa.deinit() == .ok); 7 defer std.debug.assert(gpa.deinit() == .ok);
8 const allocator = gpa.allocator(); 8 const allocator = gpa.allocator();
9 9
10 var line_buf: [4096]u8 = undefined;
10 // Process DerivedCoreProperties.txt 11 // Process DerivedCoreProperties.txt
11 var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); 12 var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{});
12 defer props_file.close(); 13 defer props_file.close();
13 var props_buf = std.io.bufferedReader(props_file.reader()); 14 var props_reader = props_file.reader(&line_buf);
14 const props_reader = props_buf.reader();
15 15
16 var props_map = std.AutoHashMap(u21, void).init(allocator); 16 var props_map = std.AutoHashMap(u21, void).init(allocator);
17 defer props_map.deinit(); 17 defer props_map.deinit();
18 18
19 var line_buf: [4096]u8 = undefined; 19 props_lines: while (props_reader.interface.takeDelimiterExclusive('\n')) |line| {
20
21 props_lines: while (try props_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
22 if (line.len == 0 or line[0] == '#') continue; 20 if (line.len == 0 or line[0] == '#') continue;
23 21
24 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 22 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -49,18 +47,21 @@ pub fn main() !void {
49 else => {}, 47 else => {},
50 } 48 }
51 } 49 }
50 } else |err| switch (err) {
51 error.EndOfStream => {},
52 else => {
53 return err;
54 },
52 } 55 }
53
54 var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator); 56 var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator);
55 defer codepoint_mapping.deinit(); 57 defer codepoint_mapping.deinit();
56 58
57 // Process CaseFolding.txt 59 // Process CaseFolding.txt
58 var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{}); 60 var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{});
59 defer cp_file.close(); 61 defer cp_file.close();
60 var cp_buf = std.io.bufferedReader(cp_file.reader()); 62 var cp_reader = cp_file.reader(&line_buf);
61 const cp_reader = cp_buf.reader();
62 63
63 while (try cp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 64 while (cp_reader.interface.takeDelimiterExclusive('\n')) |line| {
64 if (line.len == 0 or line[0] == '#') continue; 65 if (line.len == 0 or line[0] == '#') continue;
65 66
66 var field_it = std.mem.splitScalar(u8, line, ';'); 67 var field_it = std.mem.splitScalar(u8, line, ';');
@@ -81,9 +82,14 @@ pub fn main() !void {
81 } 82 }
82 83
83 try codepoint_mapping.putNoClobber(codepoint, mapping_buf); 84 try codepoint_mapping.putNoClobber(codepoint, mapping_buf);
85 } else |err| switch (err) {
86 error.EndOfStream => {},
87 else => {
88 return err;
89 },
84 } 90 }
85 91
86 var changes_when_casefolded_exceptions = std.ArrayList(u21).init(allocator); 92 var changes_when_casefolded_exceptions = std.array_list.Managed(u21).init(allocator);
87 defer changes_when_casefolded_exceptions.deinit(); 93 defer changes_when_casefolded_exceptions.deinit();
88 94
89 { 95 {
@@ -221,32 +227,31 @@ pub fn main() !void {
221 _ = args_iter.skip(); 227 _ = args_iter.skip();
222 const output_path = args_iter.next() orelse @panic("No output file arg!"); 228 const output_path = args_iter.next() orelse @panic("No output file arg!");
223 229
224 const compressor = std.compress.flate.deflate.compressor; 230 var write_buf: [4096]u8 = undefined;
225 var out_file = try std.fs.cwd().createFile(output_path, .{}); 231 var out_file = try std.fs.cwd().createFile(output_path, .{});
226 defer out_file.close(); 232 defer out_file.close();
227 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 233 var writer = out_file.writer(&write_buf);
228 const writer = out_comp.writer();
229 234
230 const endian = builtin.cpu.arch.endian(); 235 const endian = builtin.cpu.arch.endian();
231 // Table metadata. 236 // Table metadata.
232 try writer.writeInt(u24, @intCast(codepoint_cutoff), endian); 237 try writer.interface.writeInt(u24, @intCast(codepoint_cutoff), endian);
233 try writer.writeInt(u24, @intCast(multiple_codepoint_start), endian); 238 try writer.interface.writeInt(u24, @intCast(multiple_codepoint_start), endian);
234 // Stage 1 239 // Stage 1
235 try writer.writeInt(u16, @intCast(meaningful_stage1.len), endian); 240 try writer.interface.writeInt(u16, @intCast(meaningful_stage1.len), endian);
236 try writer.writeAll(meaningful_stage1); 241 try writer.interface.writeAll(meaningful_stage1);
237 // Stage 2 242 // Stage 2
238 try writer.writeInt(u16, @intCast(stage2.len), endian); 243 try writer.interface.writeInt(u16, @intCast(stage2.len), endian);
239 try writer.writeAll(stage2); 244 try writer.interface.writeAll(stage2);
240 // Stage 3 245 // Stage 3
241 try writer.writeInt(u16, @intCast(stage3.len), endian); 246 try writer.interface.writeInt(u16, @intCast(stage3.len), endian);
242 for (stage3) |offset| try writer.writeInt(i24, offset, endian); 247 for (stage3) |offset| try writer.interface.writeInt(i24, offset, endian);
243 // Changes when case folded 248 // Changes when case folded
244 // Min and max 249 // Min and max
245 try writer.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); 250 try writer.interface.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian);
246 try writer.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); 251 try writer.interface.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian);
247 try writer.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); 252 try writer.interface.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian);
248 for (changes_when_casefolded_exceptions.items) |cp| try writer.writeInt(u24, cp, endian); 253 for (changes_when_casefolded_exceptions.items) |cp| try writer.interface.writeInt(u24, cp, endian);
249 254
250 try out_comp.flush(); 255 try writer.interface.flush();
251 } 256 }
252} 257}
diff --git a/codegen/gbp.zig b/codegen/gbp.zig
index 3fc4461..d654cf1 100644
--- a/codegen/gbp.zig
+++ b/codegen/gbp.zig
@@ -47,7 +47,7 @@ const BlockMap = std.HashMap(
47 std.hash_map.default_max_load_percentage, 47 std.hash_map.default_max_load_percentage,
48); 48);
49 49
50pub fn main() !void { 50pub fn main() anyerror!void {
51 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 51 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
52 defer arena.deinit(); 52 defer arena.deinit();
53 const allocator = arena.allocator(); 53 const allocator = arena.allocator();
@@ -66,10 +66,9 @@ pub fn main() !void {
66 // Process Indic 66 // Process Indic
67 var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); 67 var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{});
68 defer indic_file.close(); 68 defer indic_file.close();
69 var indic_buf = std.io.bufferedReader(indic_file.reader()); 69 var indic_reader = indic_file.reader(&line_buf);
70 const indic_reader = indic_buf.reader();
71 70
72 while (try indic_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 71 while (indic_reader.interface.takeDelimiterExclusive('\n')) |line| {
73 if (line.len == 0 or line[0] == '#') continue; 72 if (line.len == 0 or line[0] == '#') continue;
74 if (std.mem.indexOf(u8, line, "InCB") == null) continue; 73 if (std.mem.indexOf(u8, line, "InCB") == null) continue;
75 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 74 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -100,15 +99,18 @@ pub fn main() !void {
100 else => {}, 99 else => {},
101 } 100 }
102 } 101 }
102 } else |err| switch (err) {
103 error.EndOfStream => {},
104 else => {
105 return err;
106 },
103 } 107 }
104
105 // Process GBP 108 // Process GBP
106 var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{}); 109 var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{});
107 defer gbp_file.close(); 110 defer gbp_file.close();
108 var gbp_buf = std.io.bufferedReader(gbp_file.reader()); 111 var gbp_reader = gbp_file.reader(&line_buf);
109 const gbp_reader = gbp_buf.reader();
110 112
111 while (try gbp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 113 while (gbp_reader.interface.takeDelimiterExclusive('\n')) |line| {
112 if (line.len == 0 or line[0] == '#') continue; 114 if (line.len == 0 or line[0] == '#') continue;
113 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 115 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
114 116
@@ -138,15 +140,18 @@ pub fn main() !void {
138 else => {}, 140 else => {},
139 } 141 }
140 } 142 }
143 } else |err| switch (err) {
144 error.EndOfStream => {},
145 else => {
146 return err;
147 },
141 } 148 }
142
143 // Process Emoji 149 // Process Emoji
144 var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{}); 150 var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{});
145 defer emoji_file.close(); 151 defer emoji_file.close();
146 var emoji_buf = std.io.bufferedReader(emoji_file.reader()); 152 var emoji_reader = emoji_file.reader(&line_buf);
147 const emoji_reader = emoji_buf.reader();
148 153
149 while (try emoji_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 154 while (emoji_reader.interface.takeDelimiterExclusive('\n')) |line| {
150 if (line.len == 0 or line[0] == '#') continue; 155 if (line.len == 0 or line[0] == '#') continue;
151 if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; 156 if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue;
152 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 157 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -170,15 +175,20 @@ pub fn main() !void {
170 else => {}, 175 else => {},
171 } 176 }
172 } 177 }
178 } else |err| switch (err) {
179 error.EndOfStream => {},
180 else => {
181 return err;
182 },
173 } 183 }
174 184
175 var blocks_map = BlockMap.init(allocator); 185 var blocks_map = BlockMap.init(allocator);
176 defer blocks_map.deinit(); 186 defer blocks_map.deinit();
177 187
178 var stage1 = std.ArrayList(u16).init(allocator); 188 var stage1 = std.array_list.Managed(u16).init(allocator);
179 defer stage1.deinit(); 189 defer stage1.deinit();
180 190
181 var stage2 = std.ArrayList(u16).init(allocator); 191 var stage2 = std.array_list.Managed(u16).init(allocator);
182 defer stage2.deinit(); 192 defer stage2.deinit();
183 193
184 var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator); 194 var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator);
@@ -227,22 +237,21 @@ pub fn main() !void {
227 _ = args_iter.skip(); 237 _ = args_iter.skip();
228 const output_path = args_iter.next() orelse @panic("No output file arg!"); 238 const output_path = args_iter.next() orelse @panic("No output file arg!");
229 239
230 const compressor = std.compress.flate.deflate.compressor; 240 var write_buf: [4096]u8 = undefined;
231 var out_file = try std.fs.cwd().createFile(output_path, .{}); 241 var out_file = try std.fs.cwd().createFile(output_path, .{});
232 defer out_file.close(); 242 defer out_file.close();
233 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 243 var writer = out_file.writer(&write_buf);
234 const writer = out_comp.writer();
235 244
236 const endian = builtin.cpu.arch.endian(); 245 const endian = builtin.cpu.arch.endian();
237 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 246 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
238 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 247 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
239 248
240 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 249 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
241 for (stage2.items) |i| try writer.writeInt(u16, i, endian); 250 for (stage2.items) |i| try writer.interface.writeInt(u16, i, endian);
242 251
243 const props_bytes = stage3.keys(); 252 const props_bytes = stage3.keys();
244 try writer.writeInt(u16, @intCast(props_bytes.len), endian); 253 try writer.interface.writeInt(u16, @intCast(props_bytes.len), endian);
245 try writer.writeAll(props_bytes); 254 try writer.interface.writeAll(props_bytes);
246 255
247 try out_comp.flush(); 256 try writer.interface.flush();
248} 257}
diff --git a/codegen/gencat.zig b/codegen/gencat.zig
index fe06bd7..7dedb5d 100644
--- a/codegen/gencat.zig
+++ b/codegen/gencat.zig
@@ -67,10 +67,9 @@ pub fn main() !void {
67 // Process DerivedGeneralCategory.txt 67 // Process DerivedGeneralCategory.txt
68 var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); 68 var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{});
69 defer in_file.close(); 69 defer in_file.close();
70 var in_buf = std.io.bufferedReader(in_file.reader()); 70 var in_reader = in_file.reader(&line_buf);
71 const in_reader = in_buf.reader();
72 71
73 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 72 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
74 if (line.len == 0 or line[0] == '#') continue; 73 if (line.len == 0 or line[0] == '#') continue;
75 74
76 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 75 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -101,18 +100,23 @@ pub fn main() !void {
101 else => {}, 100 else => {},
102 } 101 }
103 } 102 }
103 } else |err| switch (err) {
104 error.EndOfStream => {},
105 else => {
106 return err;
107 },
104 } 108 }
105 109
106 var blocks_map = BlockMap.init(allocator); 110 var blocks_map = BlockMap.init(allocator);
107 defer blocks_map.deinit(); 111 defer blocks_map.deinit();
108 112
109 var stage1 = std.ArrayList(u16).init(allocator); 113 var stage1 = std.array_list.Managed(u16).init(allocator);
110 defer stage1.deinit(); 114 defer stage1.deinit();
111 115
112 var stage2 = std.ArrayList(u5).init(allocator); 116 var stage2 = std.array_list.Managed(u5).init(allocator);
113 defer stage2.deinit(); 117 defer stage2.deinit();
114 118
115 var stage3 = std.ArrayList(u5).init(allocator); 119 var stage3 = std.array_list.Managed(u5).init(allocator);
116 defer stage3.deinit(); 120 defer stage3.deinit();
117 121
118 var block: Block = [_]u5{0} ** block_size; 122 var block: Block = [_]u5{0} ** block_size;
@@ -151,21 +155,20 @@ pub fn main() !void {
151 _ = args_iter.skip(); 155 _ = args_iter.skip();
152 const output_path = args_iter.next() orelse @panic("No output file arg!"); 156 const output_path = args_iter.next() orelse @panic("No output file arg!");
153 157
154 const compressor = std.compress.flate.deflate.compressor; 158 var write_buf: [4096]u8 = undefined;
155 var out_file = try std.fs.cwd().createFile(output_path, .{}); 159 var out_file = try std.fs.cwd().createFile(output_path, .{});
156 defer out_file.close(); 160 defer out_file.close();
157 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 161 var writer = out_file.writer(&write_buf);
158 const writer = out_comp.writer();
159 162
160 const endian = builtin.cpu.arch.endian(); 163 const endian = builtin.cpu.arch.endian();
161 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 164 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
162 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 165 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
163 166
164 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 167 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
165 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 168 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
166 169
167 try writer.writeInt(u8, @intCast(stage3.items.len), endian); 170 try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian);
168 for (stage3.items) |i| try writer.writeInt(u8, i, endian); 171 for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian);
169 172
170 try out_comp.flush(); 173 try writer.interface.flush();
171} 174}
diff --git a/codegen/hangul.zig b/codegen/hangul.zig
index 2c42bb7..6768b3f 100644
--- a/codegen/hangul.zig
+++ b/codegen/hangul.zig
@@ -30,7 +30,7 @@ const BlockMap = std.HashMap(
30 std.hash_map.default_max_load_percentage, 30 std.hash_map.default_max_load_percentage,
31); 31);
32 32
33pub fn main() !void { 33pub fn main() anyerror!void {
34 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 34 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
35 defer arena.deinit(); 35 defer arena.deinit();
36 const allocator = arena.allocator(); 36 const allocator = arena.allocator();
@@ -43,10 +43,9 @@ pub fn main() !void {
43 // Process HangulSyllableType.txt 43 // Process HangulSyllableType.txt
44 var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); 44 var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{});
45 defer in_file.close(); 45 defer in_file.close();
46 var in_buf = std.io.bufferedReader(in_file.reader()); 46 var in_reader = in_file.reader(&line_buf);
47 const in_reader = in_buf.reader();
48 47
49 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 48 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
50 if (line.len == 0 or line[0] == '#') continue; 49 if (line.len == 0 or line[0] == '#') continue;
51 50
52 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 51 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -77,15 +76,19 @@ pub fn main() !void {
77 else => {}, 76 else => {},
78 } 77 }
79 } 78 }
79 } else |err| switch (err) {
80 error.EndOfStream => {},
81 else => {
82 return err;
83 },
80 } 84 }
81
82 var blocks_map = BlockMap.init(allocator); 85 var blocks_map = BlockMap.init(allocator);
83 defer blocks_map.deinit(); 86 defer blocks_map.deinit();
84 87
85 var stage1 = std.ArrayList(u16).init(allocator); 88 var stage1 = std.array_list.Managed(u16).init(allocator);
86 defer stage1.deinit(); 89 defer stage1.deinit();
87 90
88 var stage2 = std.ArrayList(u3).init(allocator); 91 var stage2 = std.array_list.Managed(u3).init(allocator);
89 defer stage2.deinit(); 92 defer stage2.deinit();
90 93
91 var block: Block = [_]u3{0} ** block_size; 94 var block: Block = [_]u3{0} ** block_size;
@@ -116,18 +119,17 @@ pub fn main() !void {
116 _ = args_iter.skip(); 119 _ = args_iter.skip();
117 const output_path = args_iter.next() orelse @panic("No output file arg!"); 120 const output_path = args_iter.next() orelse @panic("No output file arg!");
118 121
119 const compressor = std.compress.flate.deflate.compressor; 122 var write_buf: [4096]u8 = undefined;
120 var out_file = try std.fs.cwd().createFile(output_path, .{}); 123 var out_file = try std.fs.cwd().createFile(output_path, .{});
121 defer out_file.close(); 124 defer out_file.close();
122 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 125 var writer = out_file.writer(&write_buf);
123 const writer = out_comp.writer();
124 126
125 const endian = builtin.cpu.arch.endian(); 127 const endian = builtin.cpu.arch.endian();
126 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 128 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
127 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 129 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
128 130
129 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 131 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
130 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 132 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
131 133
132 try out_comp.flush(); 134 try writer.interface.flush();
133} 135}
diff --git a/codegen/lower.zig b/codegen/lower.zig
index a053fe3..c11cb0a 100644
--- a/codegen/lower.zig
+++ b/codegen/lower.zig
@@ -6,27 +6,25 @@ pub fn main() !void {
6 defer arena.deinit(); 6 defer arena.deinit();
7 const allocator = arena.allocator(); 7 const allocator = arena.allocator();
8 8
9 var line_buf: [4096]u8 = undefined;
10 var write_buf: [4096]u8 = undefined;
9 // Process UnicodeData.txt 11 // Process UnicodeData.txt
10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); 12 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{});
11 defer in_file.close(); 13 defer in_file.close();
12 var in_buf = std.io.bufferedReader(in_file.reader()); 14 var in_reader = in_file.reader(&line_buf);
13 const in_reader = in_buf.reader();
14 15
15 var args_iter = try std.process.argsWithAllocator(allocator); 16 var args_iter = try std.process.argsWithAllocator(allocator);
16 defer args_iter.deinit(); 17 defer args_iter.deinit();
17 _ = args_iter.skip(); 18 _ = args_iter.skip();
18 const output_path = args_iter.next() orelse @panic("No output file arg!"); 19 const output_path = args_iter.next() orelse @panic("No output file arg!");
19 20
20 const compressor = std.compress.flate.deflate.compressor;
21 var out_file = try std.fs.cwd().createFile(output_path, .{}); 21 var out_file = try std.fs.cwd().createFile(output_path, .{});
22 defer out_file.close(); 22 defer out_file.close();
23 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 23 var writer = out_file.writer(&write_buf);
24 const writer = out_comp.writer();
25 24
26 const endian = builtin.cpu.arch.endian(); 25 const endian = builtin.cpu.arch.endian();
27 var line_buf: [4096]u8 = undefined;
28 26
29 lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 27 lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
30 if (line.len == 0) continue; 28 if (line.len == 0) continue;
31 29
32 var field_iter = std.mem.splitScalar(u8, line, ';'); 30 var field_iter = std.mem.splitScalar(u8, line, ';');
@@ -42,16 +40,20 @@ pub fn main() !void {
42 13 => { 40 13 => {
43 // Simple lowercase mapping 41 // Simple lowercase mapping
44 if (field.len == 0) continue :lines; 42 if (field.len == 0) continue :lines;
45 try writer.writeInt(i24, cp, endian); 43 try writer.interface.writeInt(i24, cp, endian);
46 const mapping = try std.fmt.parseInt(i24, field, 16); 44 const mapping = try std.fmt.parseInt(i24, field, 16);
47 try writer.writeInt(i24, mapping - cp, endian); 45 try writer.interface.writeInt(i24, mapping - cp, endian);
48 }, 46 },
49 47
50 else => {}, 48 else => {},
51 } 49 }
52 } 50 }
51 } else |err| switch (err) {
52 error.EndOfStream => {},
53 else => {
54 return err;
55 },
53 } 56 }
54 57 try writer.interface.writeInt(u24, 0, endian);
55 try writer.writeInt(u24, 0, endian); 58 try writer.interface.flush();
56 try out_comp.flush();
57} 59}
diff --git a/codegen/normp.zig b/codegen/normp.zig
index 60dabdc..71a1ae5 100644
--- a/codegen/normp.zig
+++ b/codegen/normp.zig
@@ -21,7 +21,7 @@ const BlockMap = std.HashMap(
21 std.hash_map.default_max_load_percentage, 21 std.hash_map.default_max_load_percentage,
22); 22);
23 23
24pub fn main() !void { 24pub fn main() anyerror!void {
25 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 25 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
26 defer arena.deinit(); 26 defer arena.deinit();
27 const allocator = arena.allocator(); 27 const allocator = arena.allocator();
@@ -34,10 +34,9 @@ pub fn main() !void {
34 // Process DerivedNormalizationProps.txt 34 // Process DerivedNormalizationProps.txt
35 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); 35 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{});
36 defer in_file.close(); 36 defer in_file.close();
37 var in_buf = std.io.bufferedReader(in_file.reader()); 37 var in_reader = in_file.reader(&line_buf);
38 const in_reader = in_buf.reader();
39 38
40 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 39 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
41 if (line.len == 0 or line[0] == '#') continue; 40 if (line.len == 0 or line[0] == '#') continue;
42 41
43 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 42 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -78,15 +77,19 @@ pub fn main() !void {
78 else => {}, 77 else => {},
79 } 78 }
80 } 79 }
80 } else |err| switch (err) {
81 error.EndOfStream => {},
82 else => {
83 return err;
84 },
81 } 85 }
82
83 var blocks_map = BlockMap.init(allocator); 86 var blocks_map = BlockMap.init(allocator);
84 defer blocks_map.deinit(); 87 defer blocks_map.deinit();
85 88
86 var stage1 = std.ArrayList(u16).init(allocator); 89 var stage1 = std.array_list.Managed(u16).init(allocator);
87 defer stage1.deinit(); 90 defer stage1.deinit();
88 91
89 var stage2 = std.ArrayList(u3).init(allocator); 92 var stage2 = std.array_list.Managed(u3).init(allocator);
90 defer stage2.deinit(); 93 defer stage2.deinit();
91 94
92 var block: Block = [_]u3{0} ** block_size; 95 var block: Block = [_]u3{0} ** block_size;
@@ -117,18 +120,17 @@ pub fn main() !void {
117 _ = args_iter.skip(); 120 _ = args_iter.skip();
118 const output_path = args_iter.next() orelse @panic("No output file arg!"); 121 const output_path = args_iter.next() orelse @panic("No output file arg!");
119 122
120 const compressor = std.compress.flate.deflate.compressor; 123 var write_buf: [4096]u8 = undefined;
121 var out_file = try std.fs.cwd().createFile(output_path, .{}); 124 var out_file = try std.fs.cwd().createFile(output_path, .{});
122 defer out_file.close(); 125 defer out_file.close();
123 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 126 var writer = out_file.writer(&write_buf);
124 const writer = out_comp.writer();
125 127
126 const endian = builtin.cpu.arch.endian(); 128 const endian = builtin.cpu.arch.endian();
127 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 129 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
128 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 130 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
129 131
130 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 132 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
131 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 133 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
132 134
133 try out_comp.flush(); 135 try writer.interface.flush();
134} 136}
diff --git a/codegen/numeric.zig b/codegen/numeric.zig
index 038ac0a..cf918e8 100644
--- a/codegen/numeric.zig
+++ b/codegen/numeric.zig
@@ -22,7 +22,7 @@ const BlockMap = std.HashMap(
22 std.hash_map.default_max_load_percentage, 22 std.hash_map.default_max_load_percentage,
23); 23);
24 24
25pub fn main() !void { 25pub fn main() anyerror!void {
26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
27 defer arena.deinit(); 27 defer arena.deinit();
28 const allocator = arena.allocator(); 28 const allocator = arena.allocator();
@@ -35,10 +35,9 @@ pub fn main() !void {
35 // Process DerivedNumericType.txt 35 // Process DerivedNumericType.txt
36 var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); 36 var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{});
37 defer in_file.close(); 37 defer in_file.close();
38 var in_buf = std.io.bufferedReader(in_file.reader()); 38 var in_reader = in_file.reader(&line_buf);
39 const in_reader = in_buf.reader();
40 39
41 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 40 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
42 if (line.len == 0 or line[0] == '#') continue; 41 if (line.len == 0 or line[0] == '#') continue;
43 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 42 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
44 43
@@ -79,15 +78,20 @@ pub fn main() !void {
79 else => {}, 78 else => {},
80 } 79 }
81 } 80 }
81 } else |err| switch (err) {
82 error.EndOfStream => {},
83 else => {
84 return err;
85 },
82 } 86 }
83 87
84 var blocks_map = BlockMap.init(allocator); 88 var blocks_map = BlockMap.init(allocator);
85 defer blocks_map.deinit(); 89 defer blocks_map.deinit();
86 90
87 var stage1 = std.ArrayList(u16).init(allocator); 91 var stage1 = std.array_list.Managed(u16).init(allocator);
88 defer stage1.deinit(); 92 defer stage1.deinit();
89 93
90 var stage2 = std.ArrayList(u8).init(allocator); 94 var stage2 = std.array_list.Managed(u8).init(allocator);
91 defer stage2.deinit(); 95 defer stage2.deinit();
92 96
93 var block: Block = [_]u8{0} ** block_size; 97 var block: Block = [_]u8{0} ** block_size;
@@ -118,18 +122,17 @@ pub fn main() !void {
118 _ = args_iter.skip(); 122 _ = args_iter.skip();
119 const output_path = args_iter.next() orelse @panic("No output file arg!"); 123 const output_path = args_iter.next() orelse @panic("No output file arg!");
120 124
121 const compressor = std.compress.flate.deflate.compressor; 125 var write_buf: [4096]u8 = undefined;
122 var out_file = try std.fs.cwd().createFile(output_path, .{}); 126 var out_file = try std.fs.cwd().createFile(output_path, .{});
123 defer out_file.close(); 127 defer out_file.close();
124 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 128 var writer = out_file.writer(&write_buf);
125 const writer = out_comp.writer();
126 129
127 const endian = builtin.cpu.arch.endian(); 130 const endian = builtin.cpu.arch.endian();
128 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 131 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
129 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 132 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
130 133
131 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 134 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
132 try writer.writeAll(stage2.items); 135 try writer.interface.writeAll(stage2.items);
133 136
134 try out_comp.flush(); 137 try writer.interface.flush();
135} 138}
diff --git a/codegen/props.zig b/codegen/props.zig
index 24b22e0..6ff0a33 100644
--- a/codegen/props.zig
+++ b/codegen/props.zig
@@ -22,7 +22,7 @@ const BlockMap = std.HashMap(
22 std.hash_map.default_max_load_percentage, 22 std.hash_map.default_max_load_percentage,
23); 23);
24 24
25pub fn main() !void { 25pub fn main() anyerror!void {
26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
27 defer arena.deinit(); 27 defer arena.deinit();
28 const allocator = arena.allocator(); 28 const allocator = arena.allocator();
@@ -35,10 +35,9 @@ pub fn main() !void {
35 // Process PropList.txt 35 // Process PropList.txt
36 var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); 36 var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{});
37 defer in_file.close(); 37 defer in_file.close();
38 var in_buf = std.io.bufferedReader(in_file.reader()); 38 var in_reader = in_file.reader(&line_buf);
39 const in_reader = in_buf.reader();
40 39
41 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 40 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
42 if (line.len == 0 or line[0] == '#') continue; 41 if (line.len == 0 or line[0] == '#') continue;
43 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 42 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
44 43
@@ -79,15 +78,20 @@ pub fn main() !void {
79 else => {}, 78 else => {},
80 } 79 }
81 } 80 }
81 } else |err| switch (err) {
82 error.EndOfStream => {},
83 else => {
84 return err;
85 },
82 } 86 }
83 87
84 var blocks_map = BlockMap.init(allocator); 88 var blocks_map = BlockMap.init(allocator);
85 defer blocks_map.deinit(); 89 defer blocks_map.deinit();
86 90
87 var stage1 = std.ArrayList(u16).init(allocator); 91 var stage1 = std.array_list.Managed(u16).init(allocator);
88 defer stage1.deinit(); 92 defer stage1.deinit();
89 93
90 var stage2 = std.ArrayList(u8).init(allocator); 94 var stage2 = std.array_list.Managed(u8).init(allocator);
91 defer stage2.deinit(); 95 defer stage2.deinit();
92 96
93 var block: Block = [_]u8{0} ** block_size; 97 var block: Block = [_]u8{0} ** block_size;
@@ -118,18 +122,16 @@ pub fn main() !void {
118 _ = args_iter.skip(); 122 _ = args_iter.skip();
119 const output_path = args_iter.next() orelse @panic("No output file arg!"); 123 const output_path = args_iter.next() orelse @panic("No output file arg!");
120 124
121 const compressor = std.compress.flate.deflate.compressor; 125 var write_buf: [4096]u8 = undefined;
122 var out_file = try std.fs.cwd().createFile(output_path, .{}); 126 var out_file = try std.fs.cwd().createFile(output_path, .{});
123 defer out_file.close(); 127 defer out_file.close();
124 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 128 var writer = out_file.writer(&write_buf);
125 const writer = out_comp.writer();
126 129
127 const endian = builtin.cpu.arch.endian(); 130 const endian = builtin.cpu.arch.endian();
128 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 131 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
129 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 132 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
130 133
131 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 134 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
132 try writer.writeAll(stage2.items); 135 try writer.interface.writeAll(stage2.items);
133 136 try writer.interface.flush();
134 try out_comp.flush();
135} 137}
diff --git a/codegen/scripts.zig b/codegen/scripts.zig
index 530205d..bdd3d9d 100644
--- a/codegen/scripts.zig
+++ b/codegen/scripts.zig
@@ -195,7 +195,7 @@ const BlockMap = std.HashMap(
195 std.hash_map.default_max_load_percentage, 195 std.hash_map.default_max_load_percentage,
196); 196);
197 197
198pub fn main() !void { 198pub fn main() anyerror!void {
199 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 199 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
200 defer arena.deinit(); 200 defer arena.deinit();
201 const allocator = arena.allocator(); 201 const allocator = arena.allocator();
@@ -208,10 +208,9 @@ pub fn main() !void {
208 // Process DerivedGeneralCategory.txt 208 // Process DerivedGeneralCategory.txt
209 var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); 209 var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{});
210 defer in_file.close(); 210 defer in_file.close();
211 var in_buf = std.io.bufferedReader(in_file.reader()); 211 var in_reader = in_file.reader(&line_buf);
212 const in_reader = in_buf.reader();
213 212
214 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 213 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
215 if (line.len == 0 or line[0] == '#') continue; 214 if (line.len == 0 or line[0] == '#') continue;
216 215
217 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 216 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -245,18 +244,22 @@ pub fn main() !void {
245 else => {}, 244 else => {},
246 } 245 }
247 } 246 }
247 } else |err| switch (err) {
248 error.EndOfStream => {},
249 else => {
250 return err;
251 },
248 } 252 }
249
250 var blocks_map = BlockMap.init(allocator); 253 var blocks_map = BlockMap.init(allocator);
251 defer blocks_map.deinit(); 254 defer blocks_map.deinit();
252 255
253 var stage1 = std.ArrayList(u16).init(allocator); 256 var stage1 = std.array_list.Managed(u16).init(allocator);
254 defer stage1.deinit(); 257 defer stage1.deinit();
255 258
256 var stage2 = std.ArrayList(u8).init(allocator); 259 var stage2 = std.array_list.Managed(u8).init(allocator);
257 defer stage2.deinit(); 260 defer stage2.deinit();
258 261
259 var stage3 = std.ArrayList(u8).init(allocator); 262 var stage3 = std.array_list.Managed(u8).init(allocator);
260 defer stage3.deinit(); 263 defer stage3.deinit();
261 264
262 var block: Block = [_]u8{0} ** block_size; 265 var block: Block = [_]u8{0} ** block_size;
@@ -295,21 +298,20 @@ pub fn main() !void {
295 _ = args_iter.skip(); 298 _ = args_iter.skip();
296 const output_path = args_iter.next() orelse @panic("No output file arg!"); 299 const output_path = args_iter.next() orelse @panic("No output file arg!");
297 300
298 const compressor = std.compress.flate.deflate.compressor; 301 var write_buf: [4096]u8 = undefined;
299 var out_file = try std.fs.cwd().createFile(output_path, .{}); 302 var out_file = try std.fs.cwd().createFile(output_path, .{});
300 defer out_file.close(); 303 defer out_file.close();
301 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 304 var writer = out_file.writer(&write_buf);
302 const writer = out_comp.writer();
303 305
304 const endian = builtin.cpu.arch.endian(); 306 const endian = builtin.cpu.arch.endian();
305 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 307 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
306 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 308 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
307 309
308 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 310 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
309 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 311 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
310 312
311 try writer.writeInt(u8, @intCast(stage3.items.len), endian); 313 try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian);
312 for (stage3.items) |i| try writer.writeInt(u8, i, endian); 314 for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian);
313 315
314 try out_comp.flush(); 316 try writer.interface.flush();
315} 317}
diff --git a/codegen/upper.zig b/codegen/upper.zig
index 5848911..7d96a0e 100644
--- a/codegen/upper.zig
+++ b/codegen/upper.zig
@@ -1,32 +1,29 @@
1const std = @import("std"); 1const std = @import("std");
2const builtin = @import("builtin"); 2const builtin = @import("builtin");
3 3
4pub fn main() !void { 4pub fn main() anyerror!void {
5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
6 defer arena.deinit(); 6 defer arena.deinit();
7 const allocator = arena.allocator(); 7 const allocator = arena.allocator();
8 8
9 var line_buf: [4096]u8 = undefined;
10 var write_buf: [4096]u8 = undefined;
9 // Process UnicodeData.txt 11 // Process UnicodeData.txt
10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); 12 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{});
11 defer in_file.close(); 13 defer in_file.close();
12 var in_buf = std.io.bufferedReader(in_file.reader()); 14 var in_reader = in_file.reader(&line_buf);
13 const in_reader = in_buf.reader();
14 15
15 var args_iter = try std.process.argsWithAllocator(allocator); 16 var args_iter = try std.process.argsWithAllocator(allocator);
16 defer args_iter.deinit(); 17 defer args_iter.deinit();
17 _ = args_iter.skip(); 18 _ = args_iter.skip();
18 const output_path = args_iter.next() orelse @panic("No output file arg!"); 19 const output_path = args_iter.next() orelse @panic("No output file arg!");
19 20
20 const compressor = std.compress.flate.deflate.compressor;
21 var out_file = try std.fs.cwd().createFile(output_path, .{}); 21 var out_file = try std.fs.cwd().createFile(output_path, .{});
22 defer out_file.close(); 22 defer out_file.close();
23 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 23 var file_writer = out_file.writer(&write_buf);
24 const writer = out_comp.writer();
25
26 const endian = builtin.cpu.arch.endian(); 24 const endian = builtin.cpu.arch.endian();
27 var line_buf: [4096]u8 = undefined;
28 25
29 lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 26 lines: while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
30 if (line.len == 0) continue; 27 if (line.len == 0) continue;
31 28
32 var field_iter = std.mem.splitScalar(u8, line, ';'); 29 var field_iter = std.mem.splitScalar(u8, line, ';');
@@ -42,16 +39,24 @@ pub fn main() !void {
42 12 => { 39 12 => {
43 // Simple uppercase mapping 40 // Simple uppercase mapping
44 if (field.len == 0) continue :lines; 41 if (field.len == 0) continue :lines;
45 try writer.writeInt(i24, cp, endian); 42 try file_writer.interface.writeInt(i24, cp, endian);
46 const mapping = try std.fmt.parseInt(i24, field, 16); 43 const mapping = try std.fmt.parseInt(i24, field, 16);
47 try writer.writeInt(i24, mapping - cp, endian); 44 try file_writer.interface.writeInt(i24, mapping - cp, endian);
48 }, 45 },
49 46
50 else => {}, 47 else => {},
51 } 48 }
52 } 49 }
50 } else |err| switch (err) {
51 error.ReadFailed => {
52 return in_reader.err orelse err;
53 },
54 error.EndOfStream => {},
55 else => {
56 return err;
57 },
53 } 58 }
54 59
55 try writer.writeInt(u24, 0, endian); 60 try file_writer.interface.writeInt(u24, 0, endian);
56 try out_comp.flush(); 61 try file_writer.interface.flush();
57} 62}
diff --git a/codegen/wbp.zig b/codegen/wbp.zig
index 741103e..b1ed56a 100644
--- a/codegen/wbp.zig
+++ b/codegen/wbp.zig
@@ -43,7 +43,7 @@ const BlockMap = std.HashMap(
43 std.hash_map.default_max_load_percentage, 43 std.hash_map.default_max_load_percentage,
44); 44);
45 45
46pub fn main() !void { 46pub fn main() anyerror!void {
47 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 47 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
48 defer arena.deinit(); 48 defer arena.deinit();
49 const allocator = arena.allocator(); 49 const allocator = arena.allocator();
@@ -56,10 +56,9 @@ pub fn main() !void {
56 // Process HangulSyllableType.txt 56 // Process HangulSyllableType.txt
57 var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); 57 var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{});
58 defer in_file.close(); 58 defer in_file.close();
59 var in_buf = std.io.bufferedReader(in_file.reader()); 59 var in_reader = in_file.reader(&line_buf);
60 const in_reader = in_buf.reader();
61 60
62 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 61 while (in_reader.interface.takeDelimiterExclusive('\n')) |line| {
63 if (line.len == 0 or line[0] == '#') continue; 62 if (line.len == 0 or line[0] == '#') continue;
64 63
65 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 64 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -90,15 +89,19 @@ pub fn main() !void {
90 else => {}, 89 else => {},
91 } 90 }
92 } 91 }
92 } else |err| switch (err) {
93 error.EndOfStream => {},
94 else => {
95 return err;
96 },
93 } 97 }
94
95 var blocks_map = BlockMap.init(allocator); 98 var blocks_map = BlockMap.init(allocator);
96 defer blocks_map.deinit(); 99 defer blocks_map.deinit();
97 100
98 var stage1 = std.ArrayList(u16).init(allocator); 101 var stage1 = std.array_list.Managed(u16).init(allocator);
99 defer stage1.deinit(); 102 defer stage1.deinit();
100 103
101 var stage2 = std.ArrayList(u5).init(allocator); 104 var stage2 = std.array_list.Managed(u5).init(allocator);
102 defer stage2.deinit(); 105 defer stage2.deinit();
103 106
104 var block: Block = [_]u5{0} ** block_size; 107 var block: Block = [_]u5{0} ** block_size;
@@ -129,18 +132,17 @@ pub fn main() !void {
129 _ = args_iter.skip(); 132 _ = args_iter.skip();
130 const output_path = args_iter.next() orelse @panic("No output file arg!"); 133 const output_path = args_iter.next() orelse @panic("No output file arg!");
131 134
132 const compressor = std.compress.flate.deflate.compressor; 135 var write_buf: [4096]u8 = undefined;
133 var out_file = try std.fs.cwd().createFile(output_path, .{}); 136 var out_file = try std.fs.cwd().createFile(output_path, .{});
134 defer out_file.close(); 137 defer out_file.close();
135 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 138 var writer = out_file.writer(&write_buf);
136 const writer = out_comp.writer();
137 139
138 const endian = builtin.cpu.arch.endian(); 140 const endian = builtin.cpu.arch.endian();
139 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 141 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
140 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 142 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
141 143
142 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 144 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
143 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 145 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
144 146
145 try out_comp.flush(); 147 try writer.interface.flush();
146} 148}
diff --git a/src/CanonData.zig b/src/CanonData.zig
index 5d2332a..cf9dc8a 100644
--- a/src/CanonData.zig
+++ b/src/CanonData.zig
@@ -7,11 +7,9 @@ cps: []u21 = undefined,
7const CanonData = @This(); 7const CanonData = @This();
8 8
9pub fn init(allocator: mem.Allocator) !CanonData { 9pub fn init(allocator: mem.Allocator) !CanonData {
10 const decompressor = compress.flate.inflate.decompressor;
11 const in_bytes = @embedFile("canon"); 10 const in_bytes = @embedFile("canon");
12 var in_fbs = std.io.fixedBufferStream(in_bytes); 11 var in_fbs = std.io.fixedBufferStream(in_bytes);
13 var in_decomp = decompressor(.raw, in_fbs.reader()); 12 var reader = in_fbs.reader();
14 var reader = in_decomp.reader();
15 13
16 const endian = builtin.cpu.arch.endian(); 14 const endian = builtin.cpu.arch.endian();
17 var cdata = CanonData{ 15 var cdata = CanonData{
diff --git a/src/CaseFolding.zig b/src/CaseFolding.zig
index ff41b3e..df86b92 100644
--- a/src/CaseFolding.zig
+++ b/src/CaseFolding.zig
@@ -48,11 +48,9 @@ fn setupImpl(casefold: *CaseFolding, allocator: Allocator) Allocator.Error!void
48} 48}
49 49
50inline fn setupImplInner(casefold: *CaseFolding, allocator: Allocator) !void { 50inline fn setupImplInner(casefold: *CaseFolding, allocator: Allocator) !void {
51 const decompressor = compress.flate.inflate.decompressor;
52 const in_bytes = @embedFile("fold"); 51 const in_bytes = @embedFile("fold");
53 var in_fbs = std.io.fixedBufferStream(in_bytes); 52 var in_fbs = std.io.fixedBufferStream(in_bytes);
54 var in_decomp = decompressor(.raw, in_fbs.reader()); 53 var reader = in_fbs.reader();
55 var reader = in_decomp.reader();
56 54
57 const endian = builtin.cpu.arch.endian(); 55 const endian = builtin.cpu.arch.endian();
58 56
@@ -123,7 +121,7 @@ pub fn caseFoldAlloc(
123 allocator: Allocator, 121 allocator: Allocator,
124 cps: []const u21, 122 cps: []const u21,
125) Allocator.Error![]const u21 { 123) Allocator.Error![]const u21 {
126 var cfcps = std.ArrayList(u21).init(allocator); 124 var cfcps = std.array_list.Managed(u21).init(allocator);
127 defer cfcps.deinit(); 125 defer cfcps.deinit();
128 var buf: [3]u21 = undefined; 126 var buf: [3]u21 = undefined;
129 127
diff --git a/src/CombiningData.zig b/src/CombiningData.zig
index fd64a3b..f58e0de 100644
--- a/src/CombiningData.zig
+++ b/src/CombiningData.zig
@@ -6,11 +6,9 @@ s2: []u8 = undefined,
6const CombiningData = @This(); 6const CombiningData = @This();
7 7
8pub fn init(allocator: mem.Allocator) !CombiningData { 8pub fn init(allocator: mem.Allocator) !CombiningData {
9 const decompressor = compress.flate.inflate.decompressor;
10 const in_bytes = @embedFile("ccc"); 9 const in_bytes = @embedFile("ccc");
11 var in_fbs = std.io.fixedBufferStream(in_bytes); 10 var in_fbs = std.io.fixedBufferStream(in_bytes);
12 var in_decomp = decompressor(.raw, in_fbs.reader()); 11 var reader = in_fbs.reader();
13 var reader = in_decomp.reader();
14 12
15 const endian = builtin.cpu.arch.endian(); 13 const endian = builtin.cpu.arch.endian();
16 14
@@ -46,5 +44,4 @@ pub fn isStarter(cbdata: CombiningData, cp: u21) bool {
46 44
47const std = @import("std"); 45const std = @import("std");
48const builtin = @import("builtin"); 46const builtin = @import("builtin");
49const compress = std.compress;
50const mem = std.mem; 47const mem = std.mem;
diff --git a/src/CompatData.zig b/src/CompatData.zig
index 794abca..40ecd12 100644
--- a/src/CompatData.zig
+++ b/src/CompatData.zig
@@ -6,11 +6,9 @@ cps: []u21 = undefined,
6const CompatData = @This(); 6const CompatData = @This();
7 7
8pub fn init(allocator: mem.Allocator) !CompatData { 8pub fn init(allocator: mem.Allocator) !CompatData {
9 const decompressor = compress.flate.inflate.decompressor;
10 const in_bytes = @embedFile("compat"); 9 const in_bytes = @embedFile("compat");
11 var in_fbs = std.io.fixedBufferStream(in_bytes); 10 var in_fbs = std.io.fixedBufferStream(in_bytes);
12 var in_decomp = decompressor(.raw, in_fbs.reader()); 11 var reader = in_fbs.reader();
13 var reader = in_decomp.reader();
14 12
15 const endian = builtin.cpu.arch.endian(); 13 const endian = builtin.cpu.arch.endian();
16 var cpdata = CompatData{ 14 var cpdata = CompatData{
@@ -55,6 +53,5 @@ pub fn toNfkd(cpdata: *const CompatData, cp: u21) []u21 {
55 53
56const std = @import("std"); 54const std = @import("std");
57const builtin = @import("builtin"); 55const builtin = @import("builtin");
58const compress = std.compress;
59const mem = std.mem; 56const mem = std.mem;
60const magic = @import("magic"); 57const magic = @import("magic");
diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig
index 3da2d24..82b2649 100644
--- a/src/DisplayWidth.zig
+++ b/src/DisplayWidth.zig
@@ -39,11 +39,9 @@ pub fn setupWithGraphemes(dw: *DisplayWidth, allocator: Allocator, graphemes: Gr
39 39
40// Sets up the DisplayWidthData, leaving the GraphemeData undefined. 40// Sets up the DisplayWidthData, leaving the GraphemeData undefined.
41pub fn setup(dw: *DisplayWidth, allocator: Allocator) Allocator.Error!void { 41pub fn setup(dw: *DisplayWidth, allocator: Allocator) Allocator.Error!void {
42 const decompressor = compress.flate.inflate.decompressor;
43 const in_bytes = @embedFile("dwp"); 42 const in_bytes = @embedFile("dwp");
44 var in_fbs = std.io.fixedBufferStream(in_bytes); 43 var in_fbs = std.io.fixedBufferStream(in_bytes);
45 var in_decomp = decompressor(.raw, in_fbs.reader()); 44 var reader = in_fbs.reader();
46 var reader = in_decomp.reader();
47 45
48 const endian = builtin.cpu.arch.endian(); 46 const endian = builtin.cpu.arch.endian();
49 47
@@ -400,7 +398,7 @@ pub fn wrap(
400 columns: usize, 398 columns: usize,
401 threshold: usize, 399 threshold: usize,
402) ![]u8 { 400) ![]u8 {
403 var result = ArrayList(u8).init(allocator); 401 var result = std.array_list.Managed(u8).init(allocator);
404 defer result.deinit(); 402 defer result.deinit();
405 403
406 var line_iter = mem.tokenizeAny(u8, str, "\r\n"); 404 var line_iter = mem.tokenizeAny(u8, str, "\r\n");
@@ -460,8 +458,6 @@ test "allocation test" {
460const std = @import("std"); 458const std = @import("std");
461const builtin = @import("builtin"); 459const builtin = @import("builtin");
462const options = @import("options"); 460const options = @import("options");
463const ArrayList = std.ArrayList;
464const compress = std.compress;
465const mem = std.mem; 461const mem = std.mem;
466const Allocator = mem.Allocator; 462const Allocator = mem.Allocator;
467const simd = std.simd; 463const simd = std.simd;
diff --git a/src/GeneralCategories.zig b/src/GeneralCategories.zig
index 8c1b6a3..eee7e56 100644
--- a/src/GeneralCategories.zig
+++ b/src/GeneralCategories.zig
@@ -47,11 +47,9 @@ pub fn init(allocator: Allocator) Allocator.Error!GeneralCategories {
47} 47}
48 48
49pub fn setup(gencat: *GeneralCategories, allocator: Allocator) Allocator.Error!void { 49pub fn setup(gencat: *GeneralCategories, allocator: Allocator) Allocator.Error!void {
50 const decompressor = compress.flate.inflate.decompressor;
51 const in_bytes = @embedFile("gencat"); 50 const in_bytes = @embedFile("gencat");
52 var in_fbs = std.io.fixedBufferStream(in_bytes); 51 var in_fbs = std.io.fixedBufferStream(in_bytes);
53 var in_decomp = decompressor(.raw, in_fbs.reader()); 52 var reader = in_fbs.reader();
54 var reader = in_decomp.reader();
55 53
56 const endian = builtin.cpu.arch.endian(); 54 const endian = builtin.cpu.arch.endian();
57 55
diff --git a/src/Graphemes.zig b/src/Graphemes.zig
index f1c56ed..81d874c 100644
--- a/src/Graphemes.zig
+++ b/src/Graphemes.zig
@@ -16,11 +16,9 @@ pub fn init(allocator: Allocator) Allocator.Error!Graphemes {
16} 16}
17 17
18pub fn setup(graphemes: *Graphemes, allocator: Allocator) Allocator.Error!void { 18pub fn setup(graphemes: *Graphemes, allocator: Allocator) Allocator.Error!void {
19 const decompressor = compress.flate.inflate.decompressor;
20 const in_bytes = @embedFile("gbp"); 19 const in_bytes = @embedFile("gbp");
21 var in_fbs = std.io.fixedBufferStream(in_bytes); 20 var in_fbs = std.io.fixedBufferStream(in_bytes);
22 var in_decomp = decompressor(.raw, in_fbs.reader()); 21 var reader = in_fbs.reader();
23 var reader = in_decomp.reader();
24 22
25 const endian = builtin.cpu.arch.endian(); 23 const endian = builtin.cpu.arch.endian();
26 24
diff --git a/src/HangulData.zig b/src/HangulData.zig
index 8c5f3ad..cae8b97 100644
--- a/src/HangulData.zig
+++ b/src/HangulData.zig
@@ -15,11 +15,9 @@ s2: []u3 = undefined,
15const Hangul = @This(); 15const Hangul = @This();
16 16
17pub fn init(allocator: mem.Allocator) !Hangul { 17pub fn init(allocator: mem.Allocator) !Hangul {
18 const decompressor = compress.flate.inflate.decompressor;
19 const in_bytes = @embedFile("hangul"); 18 const in_bytes = @embedFile("hangul");
20 var in_fbs = std.io.fixedBufferStream(in_bytes); 19 var in_fbs = std.io.fixedBufferStream(in_bytes);
21 var in_decomp = decompressor(.raw, in_fbs.reader()); 20 var reader = in_fbs.reader();
22 var reader = in_decomp.reader();
23 21
24 const endian = builtin.cpu.arch.endian(); 22 const endian = builtin.cpu.arch.endian();
25 var hangul = Hangul{}; 23 var hangul = Hangul{};
@@ -49,6 +47,5 @@ pub fn syllable(hangul: *const Hangul, cp: u21) Syllable {
49 47
50const std = @import("std"); 48const std = @import("std");
51const builtin = @import("builtin"); 49const builtin = @import("builtin");
52const compress = std.compress;
53const mem = std.mem; 50const mem = std.mem;
54const testing = std.testing; 51const testing = std.testing;
diff --git a/src/LetterCasing.zig b/src/LetterCasing.zig
index 11a3e96..33096fc 100644
--- a/src/LetterCasing.zig
+++ b/src/LetterCasing.zig
@@ -22,7 +22,6 @@ pub fn setup(case: *LetterCasing, allocator: Allocator) Allocator.Error!void {
22} 22}
23 23
24inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void { 24inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void {
25 const decompressor = compress.flate.inflate.decompressor;
26 const endian = builtin.cpu.arch.endian(); 25 const endian = builtin.cpu.arch.endian();
27 26
28 self.case_map = try allocator.alloc([2]u21, 0x110000); 27 self.case_map = try allocator.alloc([2]u21, 0x110000);
@@ -36,8 +35,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void {
36 // Uppercase 35 // Uppercase
37 const upper_bytes = @embedFile("upper"); 36 const upper_bytes = @embedFile("upper");
38 var upper_fbs = std.io.fixedBufferStream(upper_bytes); 37 var upper_fbs = std.io.fixedBufferStream(upper_bytes);
39 var upper_decomp = decompressor(.raw, upper_fbs.reader()); 38 var upper_reader = upper_fbs.reader();
40 var upper_reader = upper_decomp.reader();
41 39
42 while (true) { 40 while (true) {
43 const cp = try upper_reader.readInt(i24, endian); 41 const cp = try upper_reader.readInt(i24, endian);
@@ -49,8 +47,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void {
49 // Lowercase 47 // Lowercase
50 const lower_bytes = @embedFile("lower"); 48 const lower_bytes = @embedFile("lower");
51 var lower_fbs = std.io.fixedBufferStream(lower_bytes); 49 var lower_fbs = std.io.fixedBufferStream(lower_bytes);
52 var lower_decomp = decompressor(.raw, lower_fbs.reader()); 50 var lower_reader = lower_fbs.reader();
53 var lower_reader = lower_decomp.reader();
54 51
55 while (true) { 52 while (true) {
56 const cp = try lower_reader.readInt(i24, endian); 53 const cp = try lower_reader.readInt(i24, endian);
@@ -62,8 +59,7 @@ inline fn setupInner(self: *LetterCasing, allocator: mem.Allocator) !void {
62 // Case properties 59 // Case properties
63 const cp_bytes = @embedFile("case_prop"); 60 const cp_bytes = @embedFile("case_prop");
64 var cp_fbs = std.io.fixedBufferStream(cp_bytes); 61 var cp_fbs = std.io.fixedBufferStream(cp_bytes);
65 var cp_decomp = decompressor(.raw, cp_fbs.reader()); 62 var cp_reader = cp_fbs.reader();
66 var cp_reader = cp_decomp.reader();
67 63
68 const stage_1_len: u16 = try cp_reader.readInt(u16, endian); 64 const stage_1_len: u16 = try cp_reader.readInt(u16, endian);
69 self.prop_s1 = try allocator.alloc(u16, stage_1_len); 65 self.prop_s1 = try allocator.alloc(u16, stage_1_len);
@@ -122,7 +118,7 @@ pub fn toUpperStr(
122 allocator: mem.Allocator, 118 allocator: mem.Allocator,
123 str: []const u8, 119 str: []const u8,
124) ![]u8 { 120) ![]u8 {
125 var bytes = std.ArrayList(u8).init(allocator); 121 var bytes = std.array_list.Managed(u8).init(allocator);
126 defer bytes.deinit(); 122 defer bytes.deinit();
127 123
128 var iter = CodePointIterator{ .bytes = str }; 124 var iter = CodePointIterator{ .bytes = str };
@@ -180,7 +176,7 @@ pub fn toLowerStr(
180 allocator: mem.Allocator, 176 allocator: mem.Allocator,
181 str: []const u8, 177 str: []const u8,
182) ![]u8 { 178) ![]u8 {
183 var bytes = std.ArrayList(u8).init(allocator); 179 var bytes = std.array_list.Managed(u8).init(allocator);
184 defer bytes.deinit(); 180 defer bytes.deinit();
185 181
186 var iter = CodePointIterator{ .bytes = str }; 182 var iter = CodePointIterator{ .bytes = str };
diff --git a/src/NormPropsData.zig b/src/NormPropsData.zig
index ca69569..7b53542 100644
--- a/src/NormPropsData.zig
+++ b/src/NormPropsData.zig
@@ -6,11 +6,9 @@ s2: []u4 = undefined,
6const NormProps = @This(); 6const NormProps = @This();
7 7
8pub fn init(allocator: mem.Allocator) !NormProps { 8pub fn init(allocator: mem.Allocator) !NormProps {
9 const decompressor = compress.flate.inflate.decompressor;
10 const in_bytes = @embedFile("normp"); 9 const in_bytes = @embedFile("normp");
11 var in_fbs = std.io.fixedBufferStream(in_bytes); 10 var in_fbs = std.io.fixedBufferStream(in_bytes);
12 var in_decomp = decompressor(.raw, in_fbs.reader()); 11 var reader = in_fbs.reader();
13 var reader = in_decomp.reader();
14 12
15 const endian = builtin.cpu.arch.endian(); 13 const endian = builtin.cpu.arch.endian();
16 var norms = NormProps{}; 14 var norms = NormProps{};
@@ -50,6 +48,5 @@ pub fn isFcx(norms: *const NormProps, cp: u21) bool {
50 48
51const std = @import("std"); 49const std = @import("std");
52const builtin = @import("builtin"); 50const builtin = @import("builtin");
53const compress = std.compress;
54const mem = std.mem; 51const mem = std.mem;
55const testing = std.testing; 52const testing = std.testing;
diff --git a/src/Normalize.zig b/src/Normalize.zig
index 989ec29..4a1bae8 100644
--- a/src/Normalize.zig
+++ b/src/Normalize.zig
@@ -305,7 +305,7 @@ pub fn nfkd(self: Normalize, allocator: Allocator, str: []const u8) Allocator.Er
305} 305}
306 306
307pub fn nfxdCodePoints(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allocator.Error![]u21 { 307pub fn nfxdCodePoints(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allocator.Error![]u21 {
308 var dcp_list = std.ArrayList(u21).init(allocator); 308 var dcp_list = std.array_list.Managed(u21).init(allocator);
309 defer dcp_list.deinit(); 309 defer dcp_list.deinit();
310 310
311 var cp_iter = CodePointIterator{ .bytes = str }; 311 var cp_iter = CodePointIterator{ .bytes = str };
@@ -332,7 +332,7 @@ fn nfxd(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo
332 const dcps = try self.nfxdCodePoints(allocator, str, form); 332 const dcps = try self.nfxdCodePoints(allocator, str, form);
333 defer allocator.free(dcps); 333 defer allocator.free(dcps);
334 334
335 var dstr_list = std.ArrayList(u8).init(allocator); 335 var dstr_list = std.array_list.Managed(u8).init(allocator);
336 defer dstr_list.deinit(); 336 defer dstr_list.deinit();
337 var buf: [4]u8 = undefined; 337 var buf: [4]u8 = undefined;
338 338
@@ -393,7 +393,7 @@ pub fn nfdCodePoints(
393 allocator: Allocator, 393 allocator: Allocator,
394 cps: []const u21, 394 cps: []const u21,
395) Allocator.Error![]u21 { 395) Allocator.Error![]u21 {
396 var dcp_list = std.ArrayList(u21).init(allocator); 396 var dcp_list = std.array_list.Managed(u21).init(allocator);
397 defer dcp_list.deinit(); 397 defer dcp_list.deinit();
398 398
399 var dc_buf: [18]u21 = undefined; 399 var dc_buf: [18]u21 = undefined;
@@ -418,7 +418,7 @@ pub fn nfkdCodePoints(
418 allocator: Allocator, 418 allocator: Allocator,
419 cps: []const u21, 419 cps: []const u21,
420) Allocator.Error![]u21 { 420) Allocator.Error![]u21 {
421 var dcp_list = std.ArrayList(u21).init(allocator); 421 var dcp_list = std.array_list.Managed(u21).init(allocator);
422 defer dcp_list.deinit(); 422 defer dcp_list.deinit();
423 423
424 var dc_buf: [18]u21 = undefined; 424 var dc_buf: [18]u21 = undefined;
@@ -560,7 +560,7 @@ fn nfxc(self: Normalize, allocator: Allocator, str: []const u8, form: Form) Allo
560 // If we have no deletions. the code point sequence 560 // If we have no deletions. the code point sequence
561 // has been fully composed. 561 // has been fully composed.
562 if (deleted == 0) { 562 if (deleted == 0) {
563 var cstr_list = std.ArrayList(u8).init(allocator); 563 var cstr_list = std.array_list.Managed(u8).init(allocator);
564 defer cstr_list.deinit(); 564 defer cstr_list.deinit();
565 var buf: [4]u8 = undefined; 565 var buf: [4]u8 = undefined;
566 566
diff --git a/src/Properties.zig b/src/Properties.zig
index 73602a0..432d176 100644
--- a/src/Properties.zig
+++ b/src/Properties.zig
@@ -25,14 +25,12 @@ pub fn setup(props: *Properties, allocator: Allocator) Allocator.Error!void {
25} 25}
26 26
27inline fn setupInner(props: *Properties, allocator: Allocator) !void { 27inline fn setupInner(props: *Properties, allocator: Allocator) !void {
28 const decompressor = compress.flate.inflate.decompressor;
29 const endian = builtin.cpu.arch.endian(); 28 const endian = builtin.cpu.arch.endian();
30 29
31 // Process DerivedCoreProperties.txt 30 // Process DerivedCoreProperties.txt
32 const core_bytes = @embedFile("core_props"); 31 const core_bytes = @embedFile("core_props");
33 var core_fbs = std.io.fixedBufferStream(core_bytes); 32 var core_fbs = std.io.fixedBufferStream(core_bytes);
34 var core_decomp = decompressor(.raw, core_fbs.reader()); 33 var core_reader = core_fbs.reader();
35 var core_reader = core_decomp.reader();
36 34
37 const core_stage_1_len: u16 = try core_reader.readInt(u16, endian); 35 const core_stage_1_len: u16 = try core_reader.readInt(u16, endian);
38 props.core_s1 = try allocator.alloc(u16, core_stage_1_len); 36 props.core_s1 = try allocator.alloc(u16, core_stage_1_len);
@@ -47,8 +45,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void {
47 // Process PropList.txt 45 // Process PropList.txt
48 const props_bytes = @embedFile("props"); 46 const props_bytes = @embedFile("props");
49 var props_fbs = std.io.fixedBufferStream(props_bytes); 47 var props_fbs = std.io.fixedBufferStream(props_bytes);
50 var props_decomp = decompressor(.raw, props_fbs.reader()); 48 var props_reader = props_fbs.reader();
51 var props_reader = props_decomp.reader();
52 49
53 const stage_1_len: u16 = try props_reader.readInt(u16, endian); 50 const stage_1_len: u16 = try props_reader.readInt(u16, endian);
54 props.props_s1 = try allocator.alloc(u16, stage_1_len); 51 props.props_s1 = try allocator.alloc(u16, stage_1_len);
@@ -63,8 +60,7 @@ inline fn setupInner(props: *Properties, allocator: Allocator) !void {
63 // Process DerivedNumericType.txt 60 // Process DerivedNumericType.txt
64 const num_bytes = @embedFile("numeric"); 61 const num_bytes = @embedFile("numeric");
65 var num_fbs = std.io.fixedBufferStream(num_bytes); 62 var num_fbs = std.io.fixedBufferStream(num_bytes);
66 var num_decomp = decompressor(.raw, num_fbs.reader()); 63 var num_reader = num_fbs.reader();
67 var num_reader = num_decomp.reader();
68 64
69 const num_stage_1_len: u16 = try num_reader.readInt(u16, endian); 65 const num_stage_1_len: u16 = try num_reader.readInt(u16, endian);
70 props.num_s1 = try allocator.alloc(u16, num_stage_1_len); 66 props.num_s1 = try allocator.alloc(u16, num_stage_1_len);
diff --git a/src/Scripts.zig b/src/Scripts.zig
index 3bc90bc..719b01f 100644
--- a/src/Scripts.zig
+++ b/src/Scripts.zig
@@ -196,11 +196,9 @@ pub fn setup(scripts: *Scripts, allocator: Allocator) Allocator.Error!void {
196} 196}
197 197
198inline fn setupInner(scripts: *Scripts, allocator: mem.Allocator) !void { 198inline fn setupInner(scripts: *Scripts, allocator: mem.Allocator) !void {
199 const decompressor = compress.flate.inflate.decompressor;
200 const in_bytes = @embedFile("scripts"); 199 const in_bytes = @embedFile("scripts");
201 var in_fbs = std.io.fixedBufferStream(in_bytes); 200 var in_fbs = std.io.fixedBufferStream(in_bytes);
202 var in_decomp = decompressor(.raw, in_fbs.reader()); 201 var reader = in_fbs.reader();
203 var reader = in_decomp.reader();
204 202
205 const endian = builtin.cpu.arch.endian(); 203 const endian = builtin.cpu.arch.endian();
206 204
@@ -250,7 +248,6 @@ test "Allocation failure" {
250 248
251const std = @import("std"); 249const std = @import("std");
252const builtin = @import("builtin"); 250const builtin = @import("builtin");
253const compress = std.compress;
254const mem = std.mem; 251const mem = std.mem;
255const Allocator = mem.Allocator; 252const Allocator = mem.Allocator;
256const testing = std.testing; 253const testing = std.testing;
diff --git a/src/Words.zig b/src/Words.zig
index 617c34d..ce3203f 100644
--- a/src/Words.zig
+++ b/src/Words.zig
@@ -605,11 +605,9 @@ const SneakIterator = struct {
605}; 605};
606 606
607inline fn setupImpl(wb: *Words, allocator: Allocator) !void { 607inline fn setupImpl(wb: *Words, allocator: Allocator) !void {
608 const decompressor = compress.flate.inflate.decompressor;
609 const in_bytes = @embedFile("wbp"); 608 const in_bytes = @embedFile("wbp");
610 var in_fbs = std.io.fixedBufferStream(in_bytes); 609 var in_fbs = std.io.fixedBufferStream(in_bytes);
611 var in_decomp = decompressor(.raw, in_fbs.reader()); 610 var reader = in_fbs.reader();
612 var reader = in_decomp.reader();
613 611
614 const endian = builtin.cpu.arch.endian(); 612 const endian = builtin.cpu.arch.endian();
615 613
diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig
index ae177a9..ff49b2a 100644
--- a/src/unicode_tests.zig
+++ b/src/unicode_tests.zig
@@ -3,35 +3,34 @@ const dbg_print = false;
3test "Unicode normalization tests" { 3test "Unicode normalization tests" {
4 var arena = heap.ArenaAllocator.init(testing.allocator); 4 var arena = heap.ArenaAllocator.init(testing.allocator);
5 defer arena.deinit(); 5 defer arena.deinit();
6 var allocator = arena.allocator(); 6 const allocator = arena.allocator();
7 7
8 const n = try Normalize.init(allocator); 8 const n = try Normalize.init(allocator);
9 defer n.deinit(allocator); 9 defer n.deinit(allocator);
10 10
11 var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); 11 var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{});
12 defer file.close(); 12 defer file.close();
13 var buf_reader = io.bufferedReader(file.reader());
14 var input_stream = buf_reader.reader();
15
16 var buf: [4096]u8 = undefined; 13 var buf: [4096]u8 = undefined;
14 var reader = file.reader(&buf);
15
17 var cp_buf: [4]u8 = undefined; 16 var cp_buf: [4]u8 = undefined;
18 17
19 var line_iter: IterRead = .{ .read = &input_stream }; 18 var line_iter: IterRead = .{ .read = &reader.interface };
20 19
21 while (try line_iter.next(&buf)) |line| { 20 while (line_iter.next()) |line| {
22 // Iterate over fields. 21 // Iterate over fields.
23 var fields = mem.splitScalar(u8, line, ';'); 22 var fields = mem.splitScalar(u8, line, ';');
24 var field_index: usize = 0; 23 var field_index: usize = 0;
25 var input: []u8 = undefined; 24 var input: []u8 = undefined;
26 defer allocator.free(input); 25 if (dbg_print) std.debug.print("Line: {s}\n", .{line});
27
28 while (fields.next()) |field| : (field_index += 1) { 26 while (fields.next()) |field| : (field_index += 1) {
29 if (field_index == 0) { 27 if (field_index == 0) {
30 var i_buf = std.ArrayList(u8).init(allocator); 28 var i_buf = std.array_list.Managed(u8).init(allocator);
31 defer i_buf.deinit(); 29 defer i_buf.deinit();
32 30
33 var i_fields = mem.splitScalar(u8, field, ' '); 31 var i_fields = mem.splitScalar(u8, field, ' ');
34 while (i_fields.next()) |s| { 32 while (i_fields.next()) |s| {
33 if (dbg_print) std.debug.print("Debug: {s}\n", .{s});
35 const icp = try fmt.parseInt(u21, s, 16); 34 const icp = try fmt.parseInt(u21, s, 16);
36 const len = try unicode.utf8Encode(icp, &cp_buf); 35 const len = try unicode.utf8Encode(icp, &cp_buf);
37 try i_buf.appendSlice(cp_buf[0..len]); 36 try i_buf.appendSlice(cp_buf[0..len]);
@@ -41,7 +40,7 @@ test "Unicode normalization tests" {
41 } else if (field_index == 1) { 40 } else if (field_index == 1) {
42 if (dbg_print) debug.print("\n*** {s} ***\n", .{line}); 41 if (dbg_print) debug.print("\n*** {s} ***\n", .{line});
43 // NFC, time to test. 42 // NFC, time to test.
44 var w_buf = std.ArrayList(u8).init(allocator); 43 var w_buf = std.array_list.Managed(u8).init(allocator);
45 defer w_buf.deinit(); 44 defer w_buf.deinit();
46 45
47 var w_fields = mem.splitScalar(u8, field, ' '); 46 var w_fields = mem.splitScalar(u8, field, ' ');
@@ -58,7 +57,7 @@ test "Unicode normalization tests" {
58 try testing.expectEqualStrings(want, got.slice); 57 try testing.expectEqualStrings(want, got.slice);
59 } else if (field_index == 2) { 58 } else if (field_index == 2) {
60 // NFD, time to test. 59 // NFD, time to test.
61 var w_buf = std.ArrayList(u8).init(allocator); 60 var w_buf = std.array_list.Managed(u8).init(allocator);
62 defer w_buf.deinit(); 61 defer w_buf.deinit();
63 62
64 var w_fields = mem.splitScalar(u8, field, ' '); 63 var w_fields = mem.splitScalar(u8, field, ' ');
@@ -75,7 +74,7 @@ test "Unicode normalization tests" {
75 try testing.expectEqualStrings(want, got.slice); 74 try testing.expectEqualStrings(want, got.slice);
76 } else if (field_index == 3) { 75 } else if (field_index == 3) {
77 // NFKC, time to test. 76 // NFKC, time to test.
78 var w_buf = std.ArrayList(u8).init(allocator); 77 var w_buf = std.array_list.Managed(u8).init(allocator);
79 defer w_buf.deinit(); 78 defer w_buf.deinit();
80 79
81 var w_fields = mem.splitScalar(u8, field, ' '); 80 var w_fields = mem.splitScalar(u8, field, ' ');
@@ -92,7 +91,7 @@ test "Unicode normalization tests" {
92 try testing.expectEqualStrings(want, got.slice); 91 try testing.expectEqualStrings(want, got.slice);
93 } else if (field_index == 4) { 92 } else if (field_index == 4) {
94 // NFKD, time to test. 93 // NFKD, time to test.
95 var w_buf = std.ArrayList(u8).init(allocator); 94 var w_buf = std.array_list.Managed(u8).init(allocator);
96 defer w_buf.deinit(); 95 defer w_buf.deinit();
97 96
98 var w_fields = mem.splitScalar(u8, field, ' '); 97 var w_fields = mem.splitScalar(u8, field, ' ');
@@ -111,6 +110,11 @@ test "Unicode normalization tests" {
111 continue; 110 continue;
112 } 111 }
113 } 112 }
113 } else |err| switch (err) {
114 error.EndOfStream => {},
115 else => {
116 return err;
117 },
114 } 118 }
115} 119}
116 120
@@ -118,26 +122,25 @@ test "Segmentation GraphemeIterator" {
118 const allocator = std.testing.allocator; 122 const allocator = std.testing.allocator;
119 var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{}); 123 var file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakTest.txt", .{});
120 defer file.close(); 124 defer file.close();
121 var buf_reader = std.io.bufferedReader(file.reader()); 125 var buf: [4096]u8 = undefined;
122 var input_stream = buf_reader.reader(); 126 var reader = file.reader(&buf);
123 127
124 const graph = try Graphemes.init(allocator); 128 const graph = try Graphemes.init(allocator);
125 defer graph.deinit(allocator); 129 defer graph.deinit(allocator);
126 130
127 var buf: [4096]u8 = undefined; 131 var line_iter: IterRead = .{ .read = &reader.interface };
128 var line_iter: IterRead = .{ .read = &input_stream };
129 132
130 while (try line_iter.next(&buf)) |raw| { 133 while (line_iter.next()) |raw| {
131 // Clean up. 134 // Clean up.
132 var line = std.mem.trimLeft(u8, raw, "÷ "); 135 var line = std.mem.trimLeft(u8, raw, "÷ ");
133 if (std.mem.indexOf(u8, line, " ÷\t")) |final| { 136 if (std.mem.indexOf(u8, line, " ÷\t")) |final| {
134 line = line[0..final]; 137 line = line[0..final];
135 } 138 }
136 // Iterate over fields. 139 // Iterate over fields.
137 var want = std.ArrayList(Grapheme).init(allocator); 140 var want = std.array_list.Managed(Grapheme).init(allocator);
138 defer want.deinit(); 141 defer want.deinit();
139 142
140 var all_bytes = std.ArrayList(u8).init(allocator); 143 var all_bytes = std.array_list.Managed(u8).init(allocator);
141 defer all_bytes.deinit(); 144 defer all_bytes.deinit();
142 145
143 var graphemes = std.mem.splitSequence(u8, line, " ÷ "); 146 var graphemes = std.mem.splitSequence(u8, line, " ÷ ");
@@ -250,6 +253,11 @@ test "Segmentation GraphemeIterator" {
250 } 253 }
251 } 254 }
252 } 255 }
256 } else |err| switch (err) {
257 error.EndOfStream => {},
258 else => {
259 return err;
260 },
253 } 261 }
254} 262}
255 263
@@ -257,26 +265,25 @@ test "Segmentation Word Iterator" {
257 const allocator = std.testing.allocator; 265 const allocator = std.testing.allocator;
258 var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{}); 266 var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{});
259 defer file.close(); 267 defer file.close();
260 var buf_reader = std.io.bufferedReader(file.reader()); 268 var buf: [4096]u8 = undefined;
261 var input_stream = buf_reader.reader(); 269 var reader = file.reader(&buf);
262 270
263 const wb = try Words.init(allocator); 271 const wb = try Words.init(allocator);
264 defer wb.deinit(allocator); 272 defer wb.deinit(allocator);
265 273
266 var buf: [4096]u8 = undefined; 274 var line_iter: IterRead = .{ .read = &reader.interface };
267 var line_iter: IterRead = .{ .read = &input_stream };
268 275
269 while (try line_iter.next(&buf)) |raw| { 276 while (line_iter.next()) |raw| {
270 // Clean up. 277 // Clean up.
271 var line = std.mem.trimLeft(u8, raw, "÷ "); 278 var line = std.mem.trimLeft(u8, raw, "÷ ");
272 if (std.mem.indexOf(u8, line, " ÷\t")) |final| { 279 if (std.mem.indexOf(u8, line, " ÷\t")) |final| {
273 line = line[0..final]; 280 line = line[0..final];
274 } 281 }
275 // Iterate over fields. 282 // Iterate over fields.
276 var want = std.ArrayList(Word).init(allocator); 283 var want = std.array_list.Managed(Word).init(allocator);
277 defer want.deinit(); 284 defer want.deinit();
278 285
279 var all_bytes = std.ArrayList(u8).init(allocator); 286 var all_bytes = std.array_list.Managed(u8).init(allocator);
280 defer all_bytes.deinit(); 287 defer all_bytes.deinit();
281 288
282 var words = std.mem.splitSequence(u8, line, " ÷ "); 289 var words = std.mem.splitSequence(u8, line, " ÷ ");
@@ -439,26 +446,26 @@ test "Segmentation Word Iterator" {
439 if (idx == 0) break; 446 if (idx == 0) break;
440 } 447 }
441 } 448 }
449 } else |err| switch (err) {
450 error.EndOfStream => {},
451 else => {
452 return err;
453 },
442 } 454 }
443} 455}
444 456
445const IterRead = struct { 457const IterRead = struct {
446 read: *Reader, 458 read: *io.Reader,
447 line: usize = 0, 459 line: usize = 0,
448 460
449 pub fn next(iter: *IterRead, buf: []u8) !?[]const u8 { 461 pub fn next(iter: *IterRead) anyerror![]const u8 {
450 defer iter.line += 1; 462 iter.line += 1;
451 const maybe_line = try iter.read.readUntilDelimiterOrEof(buf, '#'); 463 const this_line = try iter.read.takeDelimiterExclusive('\n');
452 if (maybe_line) |this_line| { 464 if (this_line.len == 0 or this_line[0] == '@' or this_line[0] == '#') {
453 try iter.read.skipUntilDelimiterOrEof('\n'); 465 // comment, next line
454 if (this_line.len == 0 or this_line[0] == '@') { 466 return iter.next();
455 // comment, next line
456 return iter.next(buf);
457 } else {
458 return this_line;
459 }
460 } else { 467 } else {
461 return null; 468 return this_line;
462 } 469 }
463 } 470 }
464}; 471};
@@ -467,7 +474,6 @@ const std = @import("std");
467const fmt = std.fmt; 474const fmt = std.fmt;
468const fs = std.fs; 475const fs = std.fs;
469const io = std.io; 476const io = std.io;
470const Reader = io.BufferedReader(4096, fs.File.Reader).Reader;
471const heap = std.heap; 477const heap = std.heap;
472const mem = std.mem; 478const mem = std.mem;
473const debug = std.debug; 479const debug = std.debug;