summaryrefslogtreecommitdiff
path: root/codegen
diff options
context:
space:
mode:
authorGravatar Sam Atman2025-12-23 09:34:19 -0500
committerGravatar Sam Atman2025-12-23 09:34:19 -0500
commit79b133e5d88fe6cfce337dd401fc09999db08852 (patch)
tree8b3f9062edde82724c73147abf42143a885640fc /codegen
parentMerge branch 'develop-next' (diff)
parentUse takeDelimiterInclusive to support Zig 0.15.2 (diff)
downloadzg-79b133e5d88fe6cfce337dd401fc09999db08852.tar.gz
zg-79b133e5d88fe6cfce337dd401fc09999db08852.tar.xz
zg-79b133e5d88fe6cfce337dd401fc09999db08852.zip
Merge branch 'fifteen-two'
Close #90 Close #87 Close #83 Thanks everyone.
Diffstat (limited to 'codegen')
-rw-r--r--codegen/canon.zig27
-rw-r--r--codegen/case_prop.zig37
-rw-r--r--codegen/ccc.zig37
-rw-r--r--codegen/compat.zig29
-rw-r--r--codegen/core_props.zig38
-rw-r--r--codegen/dwp.zig50
-rw-r--r--codegen/fold.zig64
-rw-r--r--codegen/gbp.zig71
-rw-r--r--codegen/gencat.zig41
-rw-r--r--codegen/hangul.zig38
-rw-r--r--codegen/lower.zig29
-rw-r--r--codegen/normp.zig38
-rw-r--r--codegen/numeric.zig37
-rw-r--r--codegen/props.zig38
-rw-r--r--codegen/scripts.zig46
-rw-r--r--codegen/upper.zig31
-rw-r--r--codegen/wbp.zig39
17 files changed, 342 insertions, 348 deletions
diff --git a/codegen/canon.zig b/codegen/canon.zig
index 28b7f28..d95a905 100644
--- a/codegen/canon.zig
+++ b/codegen/canon.zig
@@ -1,32 +1,27 @@
1const std = @import("std"); 1const std = @import("std");
2const builtin = @import("builtin"); 2const builtin = @import("builtin");
3 3
4pub fn main() !void { 4pub fn main() anyerror!void {
5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
6 defer arena.deinit(); 6 defer arena.deinit();
7 const allocator = arena.allocator(); 7 const allocator = arena.allocator();
8 8
9 var write_buf: [4096]u8 = undefined;
9 // Process UnicodeData.txt 10 // Process UnicodeData.txt
10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); 11 var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt"));
11 defer in_file.close();
12 var in_buf = std.io.bufferedReader(in_file.reader());
13 const in_reader = in_buf.reader();
14
15 var args_iter = try std.process.argsWithAllocator(allocator); 12 var args_iter = try std.process.argsWithAllocator(allocator);
16 defer args_iter.deinit(); 13 defer args_iter.deinit();
17 _ = args_iter.skip(); 14 _ = args_iter.skip();
18 const output_path = args_iter.next() orelse @panic("No output file arg!"); 15 const output_path = args_iter.next() orelse @panic("No output file arg!");
19 16
20 const compressor = std.compress.flate.deflate.compressor;
21 var out_file = try std.fs.cwd().createFile(output_path, .{}); 17 var out_file = try std.fs.cwd().createFile(output_path, .{});
22 defer out_file.close(); 18 defer out_file.close();
23 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 19 var file_writer = out_file.writer(&write_buf);
24 const writer = out_comp.writer(); 20 var writer = &file_writer.interface;
25
26 const endian = builtin.cpu.arch.endian(); 21 const endian = builtin.cpu.arch.endian();
27 var line_buf: [4096]u8 = undefined;
28 22
29 lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 23 lines: while (in_reader.takeDelimiterInclusive('\n')) |took| {
24 const line = std.mem.trimRight(u8, took, "\n");
30 if (line.len == 0) continue; 25 if (line.len == 0) continue;
31 26
32 var field_iter = std.mem.splitScalar(u8, line, ';'); 27 var field_iter = std.mem.splitScalar(u8, line, ';');
@@ -60,8 +55,12 @@ pub fn main() !void {
60 55
61 try writer.writeInt(u8, @intCast(len), endian); 56 try writer.writeInt(u8, @intCast(len), endian);
62 for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); 57 for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian);
58 } else |err| switch (err) {
59 error.EndOfStream => {},
60 else => {
61 return err;
62 },
63 } 63 }
64
65 try writer.writeInt(u16, 0, endian); 64 try writer.writeInt(u16, 0, endian);
66 try out_comp.flush(); 65 try writer.flush();
67} 66}
diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig
index 6c912a8..613f7f6 100644
--- a/codegen/case_prop.zig
+++ b/codegen/case_prop.zig
@@ -22,7 +22,7 @@ const BlockMap = std.HashMap(
22 std.hash_map.default_max_load_percentage, 22 std.hash_map.default_max_load_percentage,
23); 23);
24 24
25pub fn main() !void { 25pub fn main() anyerror!void {
26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
27 defer arena.deinit(); 27 defer arena.deinit();
28 const allocator = arena.allocator(); 28 const allocator = arena.allocator();
@@ -30,15 +30,10 @@ pub fn main() !void {
30 var flat_map = std.AutoHashMap(u21, u8).init(allocator); 30 var flat_map = std.AutoHashMap(u21, u8).init(allocator);
31 defer flat_map.deinit(); 31 defer flat_map.deinit();
32 32
33 var line_buf: [4096]u8 = undefined;
34
35 // Process DerivedCoreProperties.txt 33 // Process DerivedCoreProperties.txt
36 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); 34 var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt"));
37 defer in_file.close(); 35 while (in_reader.takeDelimiterInclusive('\n')) |took| {
38 var in_buf = std.io.bufferedReader(in_file.reader()); 36 const line = std.mem.trimRight(u8, took, "\n");
39 const in_reader = in_buf.reader();
40
41 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
42 if (line.len == 0 or line[0] == '#') continue; 37 if (line.len == 0 or line[0] == '#') continue;
43 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 38 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
44 39
@@ -79,15 +74,20 @@ pub fn main() !void {
79 else => {}, 74 else => {},
80 } 75 }
81 } 76 }
77 } else |err| switch (err) {
78 error.EndOfStream => {},
79 else => {
80 return err;
81 },
82 } 82 }
83 83
84 var blocks_map = BlockMap.init(allocator); 84 var blocks_map = BlockMap.init(allocator);
85 defer blocks_map.deinit(); 85 defer blocks_map.deinit();
86 86
87 var stage1 = std.ArrayList(u16).init(allocator); 87 var stage1 = std.array_list.Managed(u16).init(allocator);
88 defer stage1.deinit(); 88 defer stage1.deinit();
89 89
90 var stage2 = std.ArrayList(u8).init(allocator); 90 var stage2 = std.array_list.Managed(u8).init(allocator);
91 defer stage2.deinit(); 91 defer stage2.deinit();
92 92
93 var block: Block = [_]u8{0} ** block_size; 93 var block: Block = [_]u8{0} ** block_size;
@@ -118,18 +118,17 @@ pub fn main() !void {
118 _ = args_iter.skip(); 118 _ = args_iter.skip();
119 const output_path = args_iter.next() orelse @panic("No output file arg!"); 119 const output_path = args_iter.next() orelse @panic("No output file arg!");
120 120
121 const compressor = std.compress.flate.deflate.compressor; 121 var write_buf: [4096]u8 = undefined;
122 var out_file = try std.fs.cwd().createFile(output_path, .{}); 122 var out_file = try std.fs.cwd().createFile(output_path, .{});
123 defer out_file.close(); 123 defer out_file.close();
124 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 124 var writer = out_file.writer(&write_buf);
125 const writer = out_comp.writer();
126 125
127 const endian = builtin.cpu.arch.endian(); 126 const endian = builtin.cpu.arch.endian();
128 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 127 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
129 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 128 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
130 129
131 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 130 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
132 try writer.writeAll(stage2.items); 131 try writer.interface.writeAll(stage2.items);
133 132
134 try out_comp.flush(); 133 try writer.interface.flush();
135} 134}
diff --git a/codegen/ccc.zig b/codegen/ccc.zig
index a01c8d2..4e470ae 100644
--- a/codegen/ccc.zig
+++ b/codegen/ccc.zig
@@ -21,7 +21,7 @@ const BlockMap = std.HashMap(
21 std.hash_map.default_max_load_percentage, 21 std.hash_map.default_max_load_percentage,
22); 22);
23 23
24pub fn main() !void { 24pub fn main() anyerror!void {
25 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 25 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
26 defer arena.deinit(); 26 defer arena.deinit();
27 const allocator = arena.allocator(); 27 const allocator = arena.allocator();
@@ -29,15 +29,10 @@ pub fn main() !void {
29 var flat_map = std.AutoHashMap(u21, u8).init(allocator); 29 var flat_map = std.AutoHashMap(u21, u8).init(allocator);
30 defer flat_map.deinit(); 30 defer flat_map.deinit();
31 31
32 var line_buf: [4096]u8 = undefined;
33
34 // Process DerivedCombiningClass.txt 32 // Process DerivedCombiningClass.txt
35 var cc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedCombiningClass.txt", .{}); 33 var cc_reader = std.io.Reader.fixed(@embedFile("DerivedCombiningClass.txt"));
36 defer cc_file.close(); 34 while (cc_reader.takeDelimiterInclusive('\n')) |took| {
37 var cc_buf = std.io.bufferedReader(cc_file.reader()); 35 const line = std.mem.trimRight(u8, took, "\n");
38 const cc_reader = cc_buf.reader();
39
40 while (try cc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
41 if (line.len == 0 or line[0] == '#') continue; 36 if (line.len == 0 or line[0] == '#') continue;
42 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 37 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
43 38
@@ -68,15 +63,20 @@ pub fn main() !void {
68 else => {}, 63 else => {},
69 } 64 }
70 } 65 }
66 } else |err| switch (err) {
67 error.EndOfStream => {},
68 else => {
69 return err;
70 },
71 } 71 }
72 72
73 var blocks_map = BlockMap.init(allocator); 73 var blocks_map = BlockMap.init(allocator);
74 defer blocks_map.deinit(); 74 defer blocks_map.deinit();
75 75
76 var stage1 = std.ArrayList(u16).init(allocator); 76 var stage1 = std.array_list.Managed(u16).init(allocator);
77 defer stage1.deinit(); 77 defer stage1.deinit();
78 78
79 var stage2 = std.ArrayList(u8).init(allocator); 79 var stage2 = std.array_list.Managed(u8).init(allocator);
80 defer stage2.deinit(); 80 defer stage2.deinit();
81 81
82 var block: Block = [_]u8{0} ** block_size; 82 var block: Block = [_]u8{0} ** block_size;
@@ -107,18 +107,17 @@ pub fn main() !void {
107 _ = args_iter.skip(); 107 _ = args_iter.skip();
108 const output_path = args_iter.next() orelse @panic("No output file arg!"); 108 const output_path = args_iter.next() orelse @panic("No output file arg!");
109 109
110 const compressor = std.compress.flate.deflate.compressor; 110 var write_buf: [4096]u8 = undefined;
111 var out_file = try std.fs.cwd().createFile(output_path, .{}); 111 var out_file = try std.fs.cwd().createFile(output_path, .{});
112 defer out_file.close(); 112 defer out_file.close();
113 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 113 var writer = out_file.writer(&write_buf);
114 const writer = out_comp.writer();
115 114
116 const endian = builtin.cpu.arch.endian(); 115 const endian = builtin.cpu.arch.endian();
117 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 116 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
118 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 117 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
119 118
120 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 119 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
121 try writer.writeAll(stage2.items); 120 try writer.interface.writeAll(stage2.items);
122 121
123 try out_comp.flush(); 122 try writer.interface.flush();
124} 123}
diff --git a/codegen/compat.zig b/codegen/compat.zig
index 07616fc..debb83d 100644
--- a/codegen/compat.zig
+++ b/codegen/compat.zig
@@ -1,32 +1,28 @@
1const std = @import("std"); 1const std = @import("std");
2const builtin = @import("builtin"); 2const builtin = @import("builtin");
3 3
4pub fn main() !void { 4pub fn main() anyerror!void {
5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
6 defer arena.deinit(); 6 defer arena.deinit();
7 const allocator = arena.allocator(); 7 const allocator = arena.allocator();
8 8
9 // Process UnicodeData.txt 9 // Process UnicodeData.txt
10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); 10 var write_buf: [4096]u8 = undefined;
11 defer in_file.close();
12 var in_buf = std.io.bufferedReader(in_file.reader());
13 const in_reader = in_buf.reader();
14 11
12 var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt"));
15 var args_iter = try std.process.argsWithAllocator(allocator); 13 var args_iter = try std.process.argsWithAllocator(allocator);
16 defer args_iter.deinit(); 14 defer args_iter.deinit();
17 _ = args_iter.skip(); 15 _ = args_iter.skip();
18 const output_path = args_iter.next() orelse @panic("No output file arg!"); 16 const output_path = args_iter.next() orelse @panic("No output file arg!");
19 17
20 const compressor = std.compress.flate.deflate.compressor;
21 var out_file = try std.fs.cwd().createFile(output_path, .{}); 18 var out_file = try std.fs.cwd().createFile(output_path, .{});
22 defer out_file.close(); 19 defer out_file.close();
23 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 20 var writer = out_file.writer(&write_buf);
24 const writer = out_comp.writer();
25 21
26 const endian = builtin.cpu.arch.endian(); 22 const endian = builtin.cpu.arch.endian();
27 var line_buf: [4096]u8 = undefined;
28 23
29 lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 24 lines: while (in_reader.takeDelimiterInclusive('\n')) |took| {
25 const line = std.mem.trimRight(u8, took, "\n");
30 if (line.len == 0) continue; 26 if (line.len == 0) continue;
31 27
32 var field_iter = std.mem.splitScalar(u8, line, ';'); 28 var field_iter = std.mem.splitScalar(u8, line, ';');
@@ -55,10 +51,15 @@ pub fn main() !void {
55 } 51 }
56 } 52 }
57 53
58 try writer.writeInt(u8, @intCast(len), endian); 54 try writer.interface.writeInt(u8, @intCast(len), endian);
59 for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian); 55 for (cps[0..len]) |cp| try writer.interface.writeInt(u24, cp, endian);
56 } else |err| switch (err) {
57 error.EndOfStream => {},
58 else => {
59 return err;
60 },
60 } 61 }
61 62
62 try writer.writeInt(u16, 0, endian); 63 try writer.interface.writeInt(u16, 0, endian);
63 try out_comp.flush(); 64 try writer.interface.flush();
64} 65}
diff --git a/codegen/core_props.zig b/codegen/core_props.zig
index f60c7a9..6ffdf91 100644
--- a/codegen/core_props.zig
+++ b/codegen/core_props.zig
@@ -22,7 +22,7 @@ const BlockMap = std.HashMap(
22 std.hash_map.default_max_load_percentage, 22 std.hash_map.default_max_load_percentage,
23); 23);
24 24
25pub fn main() !void { 25pub fn main() anyerror!void {
26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
27 defer arena.deinit(); 27 defer arena.deinit();
28 const allocator = arena.allocator(); 28 const allocator = arena.allocator();
@@ -30,15 +30,10 @@ pub fn main() !void {
30 var flat_map = std.AutoHashMap(u21, u8).init(allocator); 30 var flat_map = std.AutoHashMap(u21, u8).init(allocator);
31 defer flat_map.deinit(); 31 defer flat_map.deinit();
32 32
33 var line_buf: [4096]u8 = undefined;
34
35 // Process DerivedCoreProperties.txt 33 // Process DerivedCoreProperties.txt
36 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); 34 var in_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt"));
37 defer in_file.close(); 35 while (in_reader.takeDelimiterInclusive('\n')) |took| {
38 var in_buf = std.io.bufferedReader(in_file.reader()); 36 const line = std.mem.trimRight(u8, took, "\n");
39 const in_reader = in_buf.reader();
40
41 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
42 if (line.len == 0 or line[0] == '#') continue; 37 if (line.len == 0 or line[0] == '#') continue;
43 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 38 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
44 39
@@ -82,15 +77,19 @@ pub fn main() !void {
82 else => {}, 77 else => {},
83 } 78 }
84 } 79 }
80 } else |err| switch (err) {
81 error.EndOfStream => {},
82 else => {
83 return err;
84 },
85 } 85 }
86
87 var blocks_map = BlockMap.init(allocator); 86 var blocks_map = BlockMap.init(allocator);
88 defer blocks_map.deinit(); 87 defer blocks_map.deinit();
89 88
90 var stage1 = std.ArrayList(u16).init(allocator); 89 var stage1 = std.array_list.Managed(u16).init(allocator);
91 defer stage1.deinit(); 90 defer stage1.deinit();
92 91
93 var stage2 = std.ArrayList(u8).init(allocator); 92 var stage2 = std.array_list.Managed(u8).init(allocator);
94 defer stage2.deinit(); 93 defer stage2.deinit();
95 94
96 var block: Block = [_]u8{0} ** block_size; 95 var block: Block = [_]u8{0} ** block_size;
@@ -121,18 +120,17 @@ pub fn main() !void {
121 _ = args_iter.skip(); 120 _ = args_iter.skip();
122 const output_path = args_iter.next() orelse @panic("No output file arg!"); 121 const output_path = args_iter.next() orelse @panic("No output file arg!");
123 122
124 const compressor = std.compress.flate.deflate.compressor; 123 var out_buf: [4096]u8 = undefined;
125 var out_file = try std.fs.cwd().createFile(output_path, .{}); 124 var out_file = try std.fs.cwd().createFile(output_path, .{});
126 defer out_file.close(); 125 defer out_file.close();
127 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 126 var writer = out_file.writer(&out_buf);
128 const writer = out_comp.writer();
129 127
130 const endian = builtin.cpu.arch.endian(); 128 const endian = builtin.cpu.arch.endian();
131 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 129 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
132 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 130 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
133 131
134 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 132 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
135 try writer.writeAll(stage2.items); 133 try writer.interface.writeAll(stage2.items);
136 134
137 try out_comp.flush(); 135 try writer.interface.flush();
138} 136}
diff --git a/codegen/dwp.zig b/codegen/dwp.zig
index 5e5bf6a..75ac68e 100644
--- a/codegen/dwp.zig
+++ b/codegen/dwp.zig
@@ -23,7 +23,7 @@ const BlockMap = std.HashMap(
23 std.hash_map.default_max_load_percentage, 23 std.hash_map.default_max_load_percentage,
24); 24);
25 25
26pub fn main() !void { 26pub fn main() anyerror!void {
27 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 27 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
28 defer arena.deinit(); 28 defer arena.deinit();
29 const allocator = arena.allocator(); 29 const allocator = arena.allocator();
@@ -31,15 +31,11 @@ pub fn main() !void {
31 var flat_map = std.AutoHashMap(u21, i4).init(allocator); 31 var flat_map = std.AutoHashMap(u21, i4).init(allocator);
32 defer flat_map.deinit(); 32 defer flat_map.deinit();
33 33
34 var line_buf: [4096]u8 = undefined;
35
36 // Process DerivedEastAsianWidth.txt 34 // Process DerivedEastAsianWidth.txt
37 var deaw_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedEastAsianWidth.txt", .{}); 35 var deaw_reader = std.io.Reader.fixed(@embedFile("DerivedEastAsianWidth.txt"));
38 defer deaw_file.close();
39 var deaw_buf = std.io.bufferedReader(deaw_file.reader());
40 const deaw_reader = deaw_buf.reader();
41 36
42 while (try deaw_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 37 while (deaw_reader.takeDelimiterInclusive('\n')) |took| {
38 const line = std.mem.trimRight(u8, took, "\n");
43 if (line.len == 0) continue; 39 if (line.len == 0) continue;
44 40
45 // @missing ranges 41 // @missing ranges
@@ -88,15 +84,17 @@ pub fn main() !void {
88 else => {}, 84 else => {},
89 } 85 }
90 } 86 }
87 } else |err| switch (err) {
88 error.EndOfStream => {},
89 else => {
90 return err;
91 },
91 } 92 }
92
93 // Process DerivedGeneralCategory.txt 93 // Process DerivedGeneralCategory.txt
94 var dgc_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); 94 var dgc_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt"));
95 defer dgc_file.close();
96 var dgc_buf = std.io.bufferedReader(dgc_file.reader());
97 const dgc_reader = dgc_buf.reader();
98 95
99 while (try dgc_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 96 while (dgc_reader.takeDelimiterInclusive('\n')) |took| {
97 const line = std.mem.trimRight(u8, took, "\n");
100 if (line.len == 0 or line[0] == '#') continue; 98 if (line.len == 0 or line[0] == '#') continue;
101 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 99 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
102 100
@@ -139,15 +137,20 @@ pub fn main() !void {
139 else => {}, 137 else => {},
140 } 138 }
141 } 139 }
140 } else |err| switch (err) {
141 error.EndOfStream => {},
142 else => {
143 return err;
144 },
142 } 145 }
143 146
144 var blocks_map = BlockMap.init(allocator); 147 var blocks_map = BlockMap.init(allocator);
145 defer blocks_map.deinit(); 148 defer blocks_map.deinit();
146 149
147 var stage1 = std.ArrayList(u16).init(allocator); 150 var stage1 = std.array_list.Managed(u16).init(allocator);
148 defer stage1.deinit(); 151 defer stage1.deinit();
149 152
150 var stage2 = std.ArrayList(i4).init(allocator); 153 var stage2 = std.array_list.Managed(i4).init(allocator);
151 defer stage2.deinit(); 154 defer stage2.deinit();
152 155
153 var block: Block = [_]i4{0} ** block_size; 156 var block: Block = [_]i4{0} ** block_size;
@@ -227,18 +230,17 @@ pub fn main() !void {
227 _ = args_iter.skip(); 230 _ = args_iter.skip();
228 const output_path = args_iter.next() orelse @panic("No output file arg!"); 231 const output_path = args_iter.next() orelse @panic("No output file arg!");
229 232
230 const compressor = std.compress.flate.deflate.compressor; 233 var write_buf: [4096]u8 = undefined;
231 var out_file = try std.fs.cwd().createFile(output_path, .{}); 234 var out_file = try std.fs.cwd().createFile(output_path, .{});
232 defer out_file.close(); 235 defer out_file.close();
233 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 236 var writer = out_file.writer(&write_buf);
234 const writer = out_comp.writer();
235 237
236 const endian = builtin.cpu.arch.endian(); 238 const endian = builtin.cpu.arch.endian();
237 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 239 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
238 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 240 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
239 241
240 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 242 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
241 for (stage2.items) |i| try writer.writeInt(i8, i, endian); 243 for (stage2.items) |i| try writer.interface.writeInt(i8, i, endian);
242 244
243 try out_comp.flush(); 245 try writer.interface.flush();
244} 246}
diff --git a/codegen/fold.zig b/codegen/fold.zig
index cb73cca..366ed79 100644
--- a/codegen/fold.zig
+++ b/codegen/fold.zig
@@ -2,23 +2,19 @@ const std = @import("std");
2const builtin = @import("builtin"); 2const builtin = @import("builtin");
3const mem = std.mem; 3const mem = std.mem;
4 4
5pub fn main() !void { 5pub fn main() anyerror!void {
6 var gpa = std.heap.GeneralPurposeAllocator(.{}){}; 6 var gpa = std.heap.GeneralPurposeAllocator(.{}){};
7 defer std.debug.assert(gpa.deinit() == .ok); 7 defer std.debug.assert(gpa.deinit() == .ok);
8 const allocator = gpa.allocator(); 8 const allocator = gpa.allocator();
9 9
10 // Process DerivedCoreProperties.txt 10 // Process DerivedCoreProperties.txt
11 var props_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); 11 var props_reader = std.io.Reader.fixed(@embedFile("DerivedCoreProperties.txt"));
12 defer props_file.close();
13 var props_buf = std.io.bufferedReader(props_file.reader());
14 const props_reader = props_buf.reader();
15 12
16 var props_map = std.AutoHashMap(u21, void).init(allocator); 13 var props_map = std.AutoHashMap(u21, void).init(allocator);
17 defer props_map.deinit(); 14 defer props_map.deinit();
18 15
19 var line_buf: [4096]u8 = undefined; 16 props_lines: while (props_reader.takeDelimiterInclusive('\n')) |took| {
20 17 const line = std.mem.trimRight(u8, took, "\n");
21 props_lines: while (try props_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
22 if (line.len == 0 or line[0] == '#') continue; 18 if (line.len == 0 or line[0] == '#') continue;
23 19
24 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 20 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -49,18 +45,20 @@ pub fn main() !void {
49 else => {}, 45 else => {},
50 } 46 }
51 } 47 }
48 } else |err| switch (err) {
49 error.EndOfStream => {},
50 else => {
51 return err;
52 },
52 } 53 }
53
54 var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator); 54 var codepoint_mapping = std.AutoArrayHashMap(u21, [3]u21).init(allocator);
55 defer codepoint_mapping.deinit(); 55 defer codepoint_mapping.deinit();
56 56
57 // Process CaseFolding.txt 57 // Process CaseFolding.txt
58 var cp_file = try std.fs.cwd().openFile("data/unicode/CaseFolding.txt", .{});
59 defer cp_file.close();
60 var cp_buf = std.io.bufferedReader(cp_file.reader());
61 const cp_reader = cp_buf.reader();
62 58
63 while (try cp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 59 var cp_reader = std.io.Reader.fixed(@embedFile("CaseFolding.txt"));
60 while (cp_reader.takeDelimiterInclusive('\n')) |took| {
61 const line = std.mem.trimRight(u8, took, "\n");
64 if (line.len == 0 or line[0] == '#') continue; 62 if (line.len == 0 or line[0] == '#') continue;
65 63
66 var field_it = std.mem.splitScalar(u8, line, ';'); 64 var field_it = std.mem.splitScalar(u8, line, ';');
@@ -81,9 +79,14 @@ pub fn main() !void {
81 } 79 }
82 80
83 try codepoint_mapping.putNoClobber(codepoint, mapping_buf); 81 try codepoint_mapping.putNoClobber(codepoint, mapping_buf);
82 } else |err| switch (err) {
83 error.EndOfStream => {},
84 else => {
85 return err;
86 },
84 } 87 }
85 88
86 var changes_when_casefolded_exceptions = std.ArrayList(u21).init(allocator); 89 var changes_when_casefolded_exceptions = std.array_list.Managed(u21).init(allocator);
87 defer changes_when_casefolded_exceptions.deinit(); 90 defer changes_when_casefolded_exceptions.deinit();
88 91
89 { 92 {
@@ -221,32 +224,31 @@ pub fn main() !void {
221 _ = args_iter.skip(); 224 _ = args_iter.skip();
222 const output_path = args_iter.next() orelse @panic("No output file arg!"); 225 const output_path = args_iter.next() orelse @panic("No output file arg!");
223 226
224 const compressor = std.compress.flate.deflate.compressor; 227 var write_buf: [4096]u8 = undefined;
225 var out_file = try std.fs.cwd().createFile(output_path, .{}); 228 var out_file = try std.fs.cwd().createFile(output_path, .{});
226 defer out_file.close(); 229 defer out_file.close();
227 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 230 var writer = out_file.writer(&write_buf);
228 const writer = out_comp.writer();
229 231
230 const endian = builtin.cpu.arch.endian(); 232 const endian = builtin.cpu.arch.endian();
231 // Table metadata. 233 // Table metadata.
232 try writer.writeInt(u24, @intCast(codepoint_cutoff), endian); 234 try writer.interface.writeInt(u24, @intCast(codepoint_cutoff), endian);
233 try writer.writeInt(u24, @intCast(multiple_codepoint_start), endian); 235 try writer.interface.writeInt(u24, @intCast(multiple_codepoint_start), endian);
234 // Stage 1 236 // Stage 1
235 try writer.writeInt(u16, @intCast(meaningful_stage1.len), endian); 237 try writer.interface.writeInt(u16, @intCast(meaningful_stage1.len), endian);
236 try writer.writeAll(meaningful_stage1); 238 try writer.interface.writeAll(meaningful_stage1);
237 // Stage 2 239 // Stage 2
238 try writer.writeInt(u16, @intCast(stage2.len), endian); 240 try writer.interface.writeInt(u16, @intCast(stage2.len), endian);
239 try writer.writeAll(stage2); 241 try writer.interface.writeAll(stage2);
240 // Stage 3 242 // Stage 3
241 try writer.writeInt(u16, @intCast(stage3.len), endian); 243 try writer.interface.writeInt(u16, @intCast(stage3.len), endian);
242 for (stage3) |offset| try writer.writeInt(i24, offset, endian); 244 for (stage3) |offset| try writer.interface.writeInt(i24, offset, endian);
243 // Changes when case folded 245 // Changes when case folded
244 // Min and max 246 // Min and max
245 try writer.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian); 247 try writer.interface.writeInt(u24, std.mem.min(u21, changes_when_casefolded_exceptions.items), endian);
246 try writer.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian); 248 try writer.interface.writeInt(u24, std.mem.max(u21, changes_when_casefolded_exceptions.items), endian);
247 try writer.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian); 249 try writer.interface.writeInt(u16, @intCast(changes_when_casefolded_exceptions.items.len), endian);
248 for (changes_when_casefolded_exceptions.items) |cp| try writer.writeInt(u24, cp, endian); 250 for (changes_when_casefolded_exceptions.items) |cp| try writer.interface.writeInt(u24, cp, endian);
249 251
250 try out_comp.flush(); 252 try writer.interface.flush();
251 } 253 }
252} 254}
diff --git a/codegen/gbp.zig b/codegen/gbp.zig
index 3fc4461..1d06e9a 100644
--- a/codegen/gbp.zig
+++ b/codegen/gbp.zig
@@ -47,7 +47,7 @@ const BlockMap = std.HashMap(
47 std.hash_map.default_max_load_percentage, 47 std.hash_map.default_max_load_percentage,
48); 48);
49 49
50pub fn main() !void { 50pub fn main() anyerror!void {
51 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 51 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
52 defer arena.deinit(); 52 defer arena.deinit();
53 const allocator = arena.allocator(); 53 const allocator = arena.allocator();
@@ -61,15 +61,12 @@ pub fn main() !void {
61 var emoji_set = std.AutoHashMap(u21, void).init(allocator); 61 var emoji_set = std.AutoHashMap(u21, void).init(allocator);
62 defer emoji_set.deinit(); 62 defer emoji_set.deinit();
63 63
64 var line_buf: [4096]u8 = undefined;
65
66 // Process Indic 64 // Process Indic
67 var indic_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); 65 const indic_file = @embedFile("DerivedCoreProperties.txt");
68 defer indic_file.close(); 66 var indic_reader = std.io.Reader.fixed(indic_file);
69 var indic_buf = std.io.bufferedReader(indic_file.reader());
70 const indic_reader = indic_buf.reader();
71 67
72 while (try indic_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 68 while (indic_reader.takeDelimiterInclusive('\n')) |took| {
69 const line = std.mem.trimRight(u8, took, "\n");
73 if (line.len == 0 or line[0] == '#') continue; 70 if (line.len == 0 or line[0] == '#') continue;
74 if (std.mem.indexOf(u8, line, "InCB") == null) continue; 71 if (std.mem.indexOf(u8, line, "InCB") == null) continue;
75 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 72 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -100,15 +97,18 @@ pub fn main() !void {
100 else => {}, 97 else => {},
101 } 98 }
102 } 99 }
100 } else |err| switch (err) {
101 error.EndOfStream => {},
102 else => {
103 return err;
104 },
103 } 105 }
104
105 // Process GBP 106 // Process GBP
106 var gbp_file = try std.fs.cwd().openFile("data/unicode/auxiliary/GraphemeBreakProperty.txt", .{});
107 defer gbp_file.close();
108 var gbp_buf = std.io.bufferedReader(gbp_file.reader());
109 const gbp_reader = gbp_buf.reader();
110 107
111 while (try gbp_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 108 var gbp_reader = std.io.Reader.fixed(@embedFile("GraphemeBreakProperty.txt"));
109
110 while (gbp_reader.takeDelimiterInclusive('\n')) |took| {
111 const line = std.mem.trimRight(u8, took, "\n");
112 if (line.len == 0 or line[0] == '#') continue; 112 if (line.len == 0 or line[0] == '#') continue;
113 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 113 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
114 114
@@ -138,15 +138,18 @@ pub fn main() !void {
138 else => {}, 138 else => {},
139 } 139 }
140 } 140 }
141 } else |err| switch (err) {
142 error.EndOfStream => {},
143 else => {
144 return err;
145 },
141 } 146 }
142
143 // Process Emoji 147 // Process Emoji
144 var emoji_file = try std.fs.cwd().openFile("data/unicode/emoji/emoji-data.txt", .{});
145 defer emoji_file.close();
146 var emoji_buf = std.io.bufferedReader(emoji_file.reader());
147 const emoji_reader = emoji_buf.reader();
148 148
149 while (try emoji_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 149 var emoji_reader = std.io.Reader.fixed(@embedFile("emoji-data.txt"));
150
151 while (emoji_reader.takeDelimiterInclusive('\n')) |took| {
152 const line = std.mem.trimRight(u8, took, "\n");
150 if (line.len == 0 or line[0] == '#') continue; 153 if (line.len == 0 or line[0] == '#') continue;
151 if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue; 154 if (std.mem.indexOf(u8, line, "Extended_Pictographic") == null) continue;
152 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 155 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -170,15 +173,20 @@ pub fn main() !void {
170 else => {}, 173 else => {},
171 } 174 }
172 } 175 }
176 } else |err| switch (err) {
177 error.EndOfStream => {},
178 else => {
179 return err;
180 },
173 } 181 }
174 182
175 var blocks_map = BlockMap.init(allocator); 183 var blocks_map = BlockMap.init(allocator);
176 defer blocks_map.deinit(); 184 defer blocks_map.deinit();
177 185
178 var stage1 = std.ArrayList(u16).init(allocator); 186 var stage1 = std.array_list.Managed(u16).init(allocator);
179 defer stage1.deinit(); 187 defer stage1.deinit();
180 188
181 var stage2 = std.ArrayList(u16).init(allocator); 189 var stage2 = std.array_list.Managed(u16).init(allocator);
182 defer stage2.deinit(); 190 defer stage2.deinit();
183 191
184 var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator); 192 var stage3 = std.AutoArrayHashMap(u8, u16).init(allocator);
@@ -227,22 +235,21 @@ pub fn main() !void {
227 _ = args_iter.skip(); 235 _ = args_iter.skip();
228 const output_path = args_iter.next() orelse @panic("No output file arg!"); 236 const output_path = args_iter.next() orelse @panic("No output file arg!");
229 237
230 const compressor = std.compress.flate.deflate.compressor; 238 var write_buf: [4096]u8 = undefined;
231 var out_file = try std.fs.cwd().createFile(output_path, .{}); 239 var out_file = try std.fs.cwd().createFile(output_path, .{});
232 defer out_file.close(); 240 defer out_file.close();
233 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 241 var writer = out_file.writer(&write_buf);
234 const writer = out_comp.writer();
235 242
236 const endian = builtin.cpu.arch.endian(); 243 const endian = builtin.cpu.arch.endian();
237 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 244 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
238 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 245 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
239 246
240 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 247 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
241 for (stage2.items) |i| try writer.writeInt(u16, i, endian); 248 for (stage2.items) |i| try writer.interface.writeInt(u16, i, endian);
242 249
243 const props_bytes = stage3.keys(); 250 const props_bytes = stage3.keys();
244 try writer.writeInt(u16, @intCast(props_bytes.len), endian); 251 try writer.interface.writeInt(u16, @intCast(props_bytes.len), endian);
245 try writer.writeAll(props_bytes); 252 try writer.interface.writeAll(props_bytes);
246 253
247 try out_comp.flush(); 254 try writer.interface.flush();
248} 255}
diff --git a/codegen/gencat.zig b/codegen/gencat.zig
index fe06bd7..9800f1d 100644
--- a/codegen/gencat.zig
+++ b/codegen/gencat.zig
@@ -62,15 +62,10 @@ pub fn main() !void {
62 var flat_map = std.AutoHashMap(u21, u5).init(allocator); 62 var flat_map = std.AutoHashMap(u21, u5).init(allocator);
63 defer flat_map.deinit(); 63 defer flat_map.deinit();
64 64
65 var line_buf: [4096]u8 = undefined;
66
67 // Process DerivedGeneralCategory.txt 65 // Process DerivedGeneralCategory.txt
68 var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedGeneralCategory.txt", .{}); 66 var in_reader = std.io.Reader.fixed(@embedFile("DerivedGeneralCategory.txt"));
69 defer in_file.close(); 67 while (in_reader.takeDelimiterInclusive('\n')) |took| {
70 var in_buf = std.io.bufferedReader(in_file.reader()); 68 const line = std.mem.trimRight(u8, took, "\n");
71 const in_reader = in_buf.reader();
72
73 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
74 if (line.len == 0 or line[0] == '#') continue; 69 if (line.len == 0 or line[0] == '#') continue;
75 70
76 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 71 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -101,18 +96,23 @@ pub fn main() !void {
101 else => {}, 96 else => {},
102 } 97 }
103 } 98 }
99 } else |err| switch (err) {
100 error.EndOfStream => {},
101 else => {
102 return err;
103 },
104 } 104 }
105 105
106 var blocks_map = BlockMap.init(allocator); 106 var blocks_map = BlockMap.init(allocator);
107 defer blocks_map.deinit(); 107 defer blocks_map.deinit();
108 108
109 var stage1 = std.ArrayList(u16).init(allocator); 109 var stage1 = std.array_list.Managed(u16).init(allocator);
110 defer stage1.deinit(); 110 defer stage1.deinit();
111 111
112 var stage2 = std.ArrayList(u5).init(allocator); 112 var stage2 = std.array_list.Managed(u5).init(allocator);
113 defer stage2.deinit(); 113 defer stage2.deinit();
114 114
115 var stage3 = std.ArrayList(u5).init(allocator); 115 var stage3 = std.array_list.Managed(u5).init(allocator);
116 defer stage3.deinit(); 116 defer stage3.deinit();
117 117
118 var block: Block = [_]u5{0} ** block_size; 118 var block: Block = [_]u5{0} ** block_size;
@@ -151,21 +151,20 @@ pub fn main() !void {
151 _ = args_iter.skip(); 151 _ = args_iter.skip();
152 const output_path = args_iter.next() orelse @panic("No output file arg!"); 152 const output_path = args_iter.next() orelse @panic("No output file arg!");
153 153
154 const compressor = std.compress.flate.deflate.compressor; 154 var write_buf: [4096]u8 = undefined;
155 var out_file = try std.fs.cwd().createFile(output_path, .{}); 155 var out_file = try std.fs.cwd().createFile(output_path, .{});
156 defer out_file.close(); 156 defer out_file.close();
157 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 157 var writer = out_file.writer(&write_buf);
158 const writer = out_comp.writer();
159 158
160 const endian = builtin.cpu.arch.endian(); 159 const endian = builtin.cpu.arch.endian();
161 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 160 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
162 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 161 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
163 162
164 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 163 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
165 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 164 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
166 165
167 try writer.writeInt(u8, @intCast(stage3.items.len), endian); 166 try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian);
168 for (stage3.items) |i| try writer.writeInt(u8, i, endian); 167 for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian);
169 168
170 try out_comp.flush(); 169 try writer.interface.flush();
171} 170}
diff --git a/codegen/hangul.zig b/codegen/hangul.zig
index 2c42bb7..2e4c175 100644
--- a/codegen/hangul.zig
+++ b/codegen/hangul.zig
@@ -30,7 +30,7 @@ const BlockMap = std.HashMap(
30 std.hash_map.default_max_load_percentage, 30 std.hash_map.default_max_load_percentage,
31); 31);
32 32
33pub fn main() !void { 33pub fn main() anyerror!void {
34 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 34 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
35 defer arena.deinit(); 35 defer arena.deinit();
36 const allocator = arena.allocator(); 36 const allocator = arena.allocator();
@@ -38,15 +38,10 @@ pub fn main() !void {
38 var flat_map = std.AutoHashMap(u21, u3).init(allocator); 38 var flat_map = std.AutoHashMap(u21, u3).init(allocator);
39 defer flat_map.deinit(); 39 defer flat_map.deinit();
40 40
41 var line_buf: [4096]u8 = undefined;
42
43 // Process HangulSyllableType.txt 41 // Process HangulSyllableType.txt
44 var in_file = try std.fs.cwd().openFile("data/unicode/HangulSyllableType.txt", .{}); 42 var in_reader = std.io.Reader.fixed(@embedFile("HangulSyllableType.txt"));
45 defer in_file.close(); 43 while (in_reader.takeDelimiterInclusive('\n')) |took| {
46 var in_buf = std.io.bufferedReader(in_file.reader()); 44 const line = std.mem.trimRight(u8, took, "\n");
47 const in_reader = in_buf.reader();
48
49 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
50 if (line.len == 0 or line[0] == '#') continue; 45 if (line.len == 0 or line[0] == '#') continue;
51 46
52 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 47 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -77,15 +72,19 @@ pub fn main() !void {
77 else => {}, 72 else => {},
78 } 73 }
79 } 74 }
75 } else |err| switch (err) {
76 error.EndOfStream => {},
77 else => {
78 return err;
79 },
80 } 80 }
81
82 var blocks_map = BlockMap.init(allocator); 81 var blocks_map = BlockMap.init(allocator);
83 defer blocks_map.deinit(); 82 defer blocks_map.deinit();
84 83
85 var stage1 = std.ArrayList(u16).init(allocator); 84 var stage1 = std.array_list.Managed(u16).init(allocator);
86 defer stage1.deinit(); 85 defer stage1.deinit();
87 86
88 var stage2 = std.ArrayList(u3).init(allocator); 87 var stage2 = std.array_list.Managed(u3).init(allocator);
89 defer stage2.deinit(); 88 defer stage2.deinit();
90 89
91 var block: Block = [_]u3{0} ** block_size; 90 var block: Block = [_]u3{0} ** block_size;
@@ -116,18 +115,17 @@ pub fn main() !void {
116 _ = args_iter.skip(); 115 _ = args_iter.skip();
117 const output_path = args_iter.next() orelse @panic("No output file arg!"); 116 const output_path = args_iter.next() orelse @panic("No output file arg!");
118 117
119 const compressor = std.compress.flate.deflate.compressor; 118 var write_buf: [4096]u8 = undefined;
120 var out_file = try std.fs.cwd().createFile(output_path, .{}); 119 var out_file = try std.fs.cwd().createFile(output_path, .{});
121 defer out_file.close(); 120 defer out_file.close();
122 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 121 var writer = out_file.writer(&write_buf);
123 const writer = out_comp.writer();
124 122
125 const endian = builtin.cpu.arch.endian(); 123 const endian = builtin.cpu.arch.endian();
126 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 124 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
127 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 125 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
128 126
129 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 127 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
130 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 128 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
131 129
132 try out_comp.flush(); 130 try writer.interface.flush();
133} 131}
diff --git a/codegen/lower.zig b/codegen/lower.zig
index a053fe3..91f3ef2 100644
--- a/codegen/lower.zig
+++ b/codegen/lower.zig
@@ -6,27 +6,22 @@ pub fn main() !void {
6 defer arena.deinit(); 6 defer arena.deinit();
7 const allocator = arena.allocator(); 7 const allocator = arena.allocator();
8 8
9 var write_buf: [4096]u8 = undefined;
9 // Process UnicodeData.txt 10 // Process UnicodeData.txt
10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); 11 var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt"));
11 defer in_file.close();
12 var in_buf = std.io.bufferedReader(in_file.reader());
13 const in_reader = in_buf.reader();
14
15 var args_iter = try std.process.argsWithAllocator(allocator); 12 var args_iter = try std.process.argsWithAllocator(allocator);
16 defer args_iter.deinit(); 13 defer args_iter.deinit();
17 _ = args_iter.skip(); 14 _ = args_iter.skip();
18 const output_path = args_iter.next() orelse @panic("No output file arg!"); 15 const output_path = args_iter.next() orelse @panic("No output file arg!");
19 16
20 const compressor = std.compress.flate.deflate.compressor;
21 var out_file = try std.fs.cwd().createFile(output_path, .{}); 17 var out_file = try std.fs.cwd().createFile(output_path, .{});
22 defer out_file.close(); 18 defer out_file.close();
23 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 19 var writer = out_file.writer(&write_buf);
24 const writer = out_comp.writer();
25 20
26 const endian = builtin.cpu.arch.endian(); 21 const endian = builtin.cpu.arch.endian();
27 var line_buf: [4096]u8 = undefined;
28 22
29 lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 23 lines: while (in_reader.takeDelimiterInclusive('\n')) |took| {
24 const line = std.mem.trimRight(u8, took, "\n");
30 if (line.len == 0) continue; 25 if (line.len == 0) continue;
31 26
32 var field_iter = std.mem.splitScalar(u8, line, ';'); 27 var field_iter = std.mem.splitScalar(u8, line, ';');
@@ -42,16 +37,20 @@ pub fn main() !void {
42 13 => { 37 13 => {
43 // Simple lowercase mapping 38 // Simple lowercase mapping
44 if (field.len == 0) continue :lines; 39 if (field.len == 0) continue :lines;
45 try writer.writeInt(i24, cp, endian); 40 try writer.interface.writeInt(i24, cp, endian);
46 const mapping = try std.fmt.parseInt(i24, field, 16); 41 const mapping = try std.fmt.parseInt(i24, field, 16);
47 try writer.writeInt(i24, mapping - cp, endian); 42 try writer.interface.writeInt(i24, mapping - cp, endian);
48 }, 43 },
49 44
50 else => {}, 45 else => {},
51 } 46 }
52 } 47 }
48 } else |err| switch (err) {
49 error.EndOfStream => {},
50 else => {
51 return err;
52 },
53 } 53 }
54 54 try writer.interface.writeInt(u24, 0, endian);
55 try writer.writeInt(u24, 0, endian); 55 try writer.interface.flush();
56 try out_comp.flush();
57} 56}
diff --git a/codegen/normp.zig b/codegen/normp.zig
index 60dabdc..eaf6989 100644
--- a/codegen/normp.zig
+++ b/codegen/normp.zig
@@ -21,7 +21,7 @@ const BlockMap = std.HashMap(
21 std.hash_map.default_max_load_percentage, 21 std.hash_map.default_max_load_percentage,
22); 22);
23 23
24pub fn main() !void { 24pub fn main() anyerror!void {
25 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 25 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
26 defer arena.deinit(); 26 defer arena.deinit();
27 const allocator = arena.allocator(); 27 const allocator = arena.allocator();
@@ -29,15 +29,10 @@ pub fn main() !void {
29 var flat_map = std.AutoHashMap(u21, u3).init(allocator); 29 var flat_map = std.AutoHashMap(u21, u3).init(allocator);
30 defer flat_map.deinit(); 30 defer flat_map.deinit();
31 31
32 var line_buf: [4096]u8 = undefined;
33
34 // Process DerivedNormalizationProps.txt 32 // Process DerivedNormalizationProps.txt
35 var in_file = try std.fs.cwd().openFile("data/unicode/DerivedNormalizationProps.txt", .{}); 33 var in_reader = std.io.Reader.fixed(@embedFile("DerivedNormalizationProps.txt"));
36 defer in_file.close(); 34 while (in_reader.takeDelimiterInclusive('\n')) |took| {
37 var in_buf = std.io.bufferedReader(in_file.reader()); 35 const line = std.mem.trimRight(u8, took, "\n");
38 const in_reader = in_buf.reader();
39
40 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
41 if (line.len == 0 or line[0] == '#') continue; 36 if (line.len == 0 or line[0] == '#') continue;
42 37
43 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 38 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -78,15 +73,19 @@ pub fn main() !void {
78 else => {}, 73 else => {},
79 } 74 }
80 } 75 }
76 } else |err| switch (err) {
77 error.EndOfStream => {},
78 else => {
79 return err;
80 },
81 } 81 }
82
83 var blocks_map = BlockMap.init(allocator); 82 var blocks_map = BlockMap.init(allocator);
84 defer blocks_map.deinit(); 83 defer blocks_map.deinit();
85 84
86 var stage1 = std.ArrayList(u16).init(allocator); 85 var stage1 = std.array_list.Managed(u16).init(allocator);
87 defer stage1.deinit(); 86 defer stage1.deinit();
88 87
89 var stage2 = std.ArrayList(u3).init(allocator); 88 var stage2 = std.array_list.Managed(u3).init(allocator);
90 defer stage2.deinit(); 89 defer stage2.deinit();
91 90
92 var block: Block = [_]u3{0} ** block_size; 91 var block: Block = [_]u3{0} ** block_size;
@@ -117,18 +116,17 @@ pub fn main() !void {
117 _ = args_iter.skip(); 116 _ = args_iter.skip();
118 const output_path = args_iter.next() orelse @panic("No output file arg!"); 117 const output_path = args_iter.next() orelse @panic("No output file arg!");
119 118
120 const compressor = std.compress.flate.deflate.compressor; 119 var write_buf: [4096]u8 = undefined;
121 var out_file = try std.fs.cwd().createFile(output_path, .{}); 120 var out_file = try std.fs.cwd().createFile(output_path, .{});
122 defer out_file.close(); 121 defer out_file.close();
123 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 122 var writer = out_file.writer(&write_buf);
124 const writer = out_comp.writer();
125 123
126 const endian = builtin.cpu.arch.endian(); 124 const endian = builtin.cpu.arch.endian();
127 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 125 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
128 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 126 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
129 127
130 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 128 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
131 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 129 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
132 130
133 try out_comp.flush(); 131 try writer.interface.flush();
134} 132}
diff --git a/codegen/numeric.zig b/codegen/numeric.zig
index 038ac0a..b304349 100644
--- a/codegen/numeric.zig
+++ b/codegen/numeric.zig
@@ -22,7 +22,7 @@ const BlockMap = std.HashMap(
22 std.hash_map.default_max_load_percentage, 22 std.hash_map.default_max_load_percentage,
23); 23);
24 24
25pub fn main() !void { 25pub fn main() anyerror!void {
26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
27 defer arena.deinit(); 27 defer arena.deinit();
28 const allocator = arena.allocator(); 28 const allocator = arena.allocator();
@@ -30,15 +30,10 @@ pub fn main() !void {
30 var flat_map = std.AutoHashMap(u21, u8).init(allocator); 30 var flat_map = std.AutoHashMap(u21, u8).init(allocator);
31 defer flat_map.deinit(); 31 defer flat_map.deinit();
32 32
33 var line_buf: [4096]u8 = undefined;
34
35 // Process DerivedNumericType.txt 33 // Process DerivedNumericType.txt
36 var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); 34 var in_reader = std.io.Reader.fixed(@embedFile("DerivedNumericType.txt"));
37 defer in_file.close(); 35 while (in_reader.takeDelimiterInclusive('\n')) |took| {
38 var in_buf = std.io.bufferedReader(in_file.reader()); 36 const line = std.mem.trimRight(u8, took, "\n");
39 const in_reader = in_buf.reader();
40
41 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
42 if (line.len == 0 or line[0] == '#') continue; 37 if (line.len == 0 or line[0] == '#') continue;
43 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 38 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
44 39
@@ -79,15 +74,20 @@ pub fn main() !void {
79 else => {}, 74 else => {},
80 } 75 }
81 } 76 }
77 } else |err| switch (err) {
78 error.EndOfStream => {},
79 else => {
80 return err;
81 },
82 } 82 }
83 83
84 var blocks_map = BlockMap.init(allocator); 84 var blocks_map = BlockMap.init(allocator);
85 defer blocks_map.deinit(); 85 defer blocks_map.deinit();
86 86
87 var stage1 = std.ArrayList(u16).init(allocator); 87 var stage1 = std.array_list.Managed(u16).init(allocator);
88 defer stage1.deinit(); 88 defer stage1.deinit();
89 89
90 var stage2 = std.ArrayList(u8).init(allocator); 90 var stage2 = std.array_list.Managed(u8).init(allocator);
91 defer stage2.deinit(); 91 defer stage2.deinit();
92 92
93 var block: Block = [_]u8{0} ** block_size; 93 var block: Block = [_]u8{0} ** block_size;
@@ -118,18 +118,17 @@ pub fn main() !void {
118 _ = args_iter.skip(); 118 _ = args_iter.skip();
119 const output_path = args_iter.next() orelse @panic("No output file arg!"); 119 const output_path = args_iter.next() orelse @panic("No output file arg!");
120 120
121 const compressor = std.compress.flate.deflate.compressor; 121 var write_buf: [4096]u8 = undefined;
122 var out_file = try std.fs.cwd().createFile(output_path, .{}); 122 var out_file = try std.fs.cwd().createFile(output_path, .{});
123 defer out_file.close(); 123 defer out_file.close();
124 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 124 var writer = out_file.writer(&write_buf);
125 const writer = out_comp.writer();
126 125
127 const endian = builtin.cpu.arch.endian(); 126 const endian = builtin.cpu.arch.endian();
128 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 127 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
129 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 128 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
130 129
131 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 130 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
132 try writer.writeAll(stage2.items); 131 try writer.interface.writeAll(stage2.items);
133 132
134 try out_comp.flush(); 133 try writer.interface.flush();
135} 134}
diff --git a/codegen/props.zig b/codegen/props.zig
index 24b22e0..35c7dfb 100644
--- a/codegen/props.zig
+++ b/codegen/props.zig
@@ -22,7 +22,7 @@ const BlockMap = std.HashMap(
22 std.hash_map.default_max_load_percentage, 22 std.hash_map.default_max_load_percentage,
23); 23);
24 24
25pub fn main() !void { 25pub fn main() anyerror!void {
26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 26 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
27 defer arena.deinit(); 27 defer arena.deinit();
28 const allocator = arena.allocator(); 28 const allocator = arena.allocator();
@@ -30,15 +30,10 @@ pub fn main() !void {
30 var flat_map = std.AutoHashMap(u21, u8).init(allocator); 30 var flat_map = std.AutoHashMap(u21, u8).init(allocator);
31 defer flat_map.deinit(); 31 defer flat_map.deinit();
32 32
33 var line_buf: [4096]u8 = undefined;
34
35 // Process PropList.txt 33 // Process PropList.txt
36 var in_file = try std.fs.cwd().openFile("data/unicode/PropList.txt", .{}); 34 var in_reader = std.io.Reader.fixed(@embedFile("PropList.txt"));
37 defer in_file.close(); 35 while (in_reader.takeDelimiterInclusive('\n')) |took| {
38 var in_buf = std.io.bufferedReader(in_file.reader()); 36 const line = std.mem.trimRight(u8, took, "\n");
39 const in_reader = in_buf.reader();
40
41 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
42 if (line.len == 0 or line[0] == '#') continue; 37 if (line.len == 0 or line[0] == '#') continue;
43 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 38 const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
44 39
@@ -79,15 +74,20 @@ pub fn main() !void {
79 else => {}, 74 else => {},
80 } 75 }
81 } 76 }
77 } else |err| switch (err) {
78 error.EndOfStream => {},
79 else => {
80 return err;
81 },
82 } 82 }
83 83
84 var blocks_map = BlockMap.init(allocator); 84 var blocks_map = BlockMap.init(allocator);
85 defer blocks_map.deinit(); 85 defer blocks_map.deinit();
86 86
87 var stage1 = std.ArrayList(u16).init(allocator); 87 var stage1 = std.array_list.Managed(u16).init(allocator);
88 defer stage1.deinit(); 88 defer stage1.deinit();
89 89
90 var stage2 = std.ArrayList(u8).init(allocator); 90 var stage2 = std.array_list.Managed(u8).init(allocator);
91 defer stage2.deinit(); 91 defer stage2.deinit();
92 92
93 var block: Block = [_]u8{0} ** block_size; 93 var block: Block = [_]u8{0} ** block_size;
@@ -118,18 +118,16 @@ pub fn main() !void {
118 _ = args_iter.skip(); 118 _ = args_iter.skip();
119 const output_path = args_iter.next() orelse @panic("No output file arg!"); 119 const output_path = args_iter.next() orelse @panic("No output file arg!");
120 120
121 const compressor = std.compress.flate.deflate.compressor; 121 var write_buf: [4096]u8 = undefined;
122 var out_file = try std.fs.cwd().createFile(output_path, .{}); 122 var out_file = try std.fs.cwd().createFile(output_path, .{});
123 defer out_file.close(); 123 defer out_file.close();
124 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 124 var writer = out_file.writer(&write_buf);
125 const writer = out_comp.writer();
126 125
127 const endian = builtin.cpu.arch.endian(); 126 const endian = builtin.cpu.arch.endian();
128 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 127 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
129 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 128 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
130
131 try writer.writeInt(u16, @intCast(stage2.items.len), endian);
132 try writer.writeAll(stage2.items);
133 129
134 try out_comp.flush(); 130 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
131 try writer.interface.writeAll(stage2.items);
132 try writer.interface.flush();
135} 133}
diff --git a/codegen/scripts.zig b/codegen/scripts.zig
index 530205d..0f0194c 100644
--- a/codegen/scripts.zig
+++ b/codegen/scripts.zig
@@ -195,7 +195,7 @@ const BlockMap = std.HashMap(
195 std.hash_map.default_max_load_percentage, 195 std.hash_map.default_max_load_percentage,
196); 196);
197 197
198pub fn main() !void { 198pub fn main() anyerror!void {
199 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 199 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
200 defer arena.deinit(); 200 defer arena.deinit();
201 const allocator = arena.allocator(); 201 const allocator = arena.allocator();
@@ -203,15 +203,10 @@ pub fn main() !void {
203 var flat_map = std.AutoHashMap(u21, u8).init(allocator); 203 var flat_map = std.AutoHashMap(u21, u8).init(allocator);
204 defer flat_map.deinit(); 204 defer flat_map.deinit();
205 205
206 var line_buf: [4096]u8 = undefined; 206 // Process Scripts.txt
207 207 var in_reader = std.io.Reader.fixed(@embedFile("Scripts.txt"));
208 // Process DerivedGeneralCategory.txt 208 while (in_reader.takeDelimiterInclusive('\n')) |took| {
209 var in_file = try std.fs.cwd().openFile("data/unicode/Scripts.txt", .{}); 209 const line = std.mem.trimRight(u8, took, "\n");
210 defer in_file.close();
211 var in_buf = std.io.bufferedReader(in_file.reader());
212 const in_reader = in_buf.reader();
213
214 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
215 if (line.len == 0 or line[0] == '#') continue; 210 if (line.len == 0 or line[0] == '#') continue;
216 211
217 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 212 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -245,18 +240,22 @@ pub fn main() !void {
245 else => {}, 240 else => {},
246 } 241 }
247 } 242 }
243 } else |err| switch (err) {
244 error.EndOfStream => {},
245 else => {
246 return err;
247 },
248 } 248 }
249
250 var blocks_map = BlockMap.init(allocator); 249 var blocks_map = BlockMap.init(allocator);
251 defer blocks_map.deinit(); 250 defer blocks_map.deinit();
252 251
253 var stage1 = std.ArrayList(u16).init(allocator); 252 var stage1 = std.array_list.Managed(u16).init(allocator);
254 defer stage1.deinit(); 253 defer stage1.deinit();
255 254
256 var stage2 = std.ArrayList(u8).init(allocator); 255 var stage2 = std.array_list.Managed(u8).init(allocator);
257 defer stage2.deinit(); 256 defer stage2.deinit();
258 257
259 var stage3 = std.ArrayList(u8).init(allocator); 258 var stage3 = std.array_list.Managed(u8).init(allocator);
260 defer stage3.deinit(); 259 defer stage3.deinit();
261 260
262 var block: Block = [_]u8{0} ** block_size; 261 var block: Block = [_]u8{0} ** block_size;
@@ -295,21 +294,20 @@ pub fn main() !void {
295 _ = args_iter.skip(); 294 _ = args_iter.skip();
296 const output_path = args_iter.next() orelse @panic("No output file arg!"); 295 const output_path = args_iter.next() orelse @panic("No output file arg!");
297 296
298 const compressor = std.compress.flate.deflate.compressor; 297 var write_buf: [4096]u8 = undefined;
299 var out_file = try std.fs.cwd().createFile(output_path, .{}); 298 var out_file = try std.fs.cwd().createFile(output_path, .{});
300 defer out_file.close(); 299 defer out_file.close();
301 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 300 var writer = out_file.writer(&write_buf);
302 const writer = out_comp.writer();
303 301
304 const endian = builtin.cpu.arch.endian(); 302 const endian = builtin.cpu.arch.endian();
305 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 303 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
306 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 304 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
307 305
308 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 306 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
309 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 307 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
310 308
311 try writer.writeInt(u8, @intCast(stage3.items.len), endian); 309 try writer.interface.writeInt(u8, @intCast(stage3.items.len), endian);
312 for (stage3.items) |i| try writer.writeInt(u8, i, endian); 310 for (stage3.items) |i| try writer.interface.writeInt(u8, i, endian);
313 311
314 try out_comp.flush(); 312 try writer.interface.flush();
315} 313}
diff --git a/codegen/upper.zig b/codegen/upper.zig
index 5848911..5eb29e7 100644
--- a/codegen/upper.zig
+++ b/codegen/upper.zig
@@ -1,32 +1,26 @@
1const std = @import("std"); 1const std = @import("std");
2const builtin = @import("builtin"); 2const builtin = @import("builtin");
3 3
4pub fn main() !void { 4pub fn main() anyerror!void {
5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 5 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
6 defer arena.deinit(); 6 defer arena.deinit();
7 const allocator = arena.allocator(); 7 const allocator = arena.allocator();
8 8
9 var write_buf: [4096]u8 = undefined;
9 // Process UnicodeData.txt 10 // Process UnicodeData.txt
10 var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); 11 var in_reader = std.io.Reader.fixed(@embedFile("UnicodeData.txt"));
11 defer in_file.close();
12 var in_buf = std.io.bufferedReader(in_file.reader());
13 const in_reader = in_buf.reader();
14
15 var args_iter = try std.process.argsWithAllocator(allocator); 12 var args_iter = try std.process.argsWithAllocator(allocator);
16 defer args_iter.deinit(); 13 defer args_iter.deinit();
17 _ = args_iter.skip(); 14 _ = args_iter.skip();
18 const output_path = args_iter.next() orelse @panic("No output file arg!"); 15 const output_path = args_iter.next() orelse @panic("No output file arg!");
19 16
20 const compressor = std.compress.flate.deflate.compressor;
21 var out_file = try std.fs.cwd().createFile(output_path, .{}); 17 var out_file = try std.fs.cwd().createFile(output_path, .{});
22 defer out_file.close(); 18 defer out_file.close();
23 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 19 var file_writer = out_file.writer(&write_buf);
24 const writer = out_comp.writer();
25
26 const endian = builtin.cpu.arch.endian(); 20 const endian = builtin.cpu.arch.endian();
27 var line_buf: [4096]u8 = undefined;
28 21
29 lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { 22 lines: while (in_reader.takeDelimiterInclusive('\n')) |took| {
23 const line = std.mem.trimRight(u8, took, "\n");
30 if (line.len == 0) continue; 24 if (line.len == 0) continue;
31 25
32 var field_iter = std.mem.splitScalar(u8, line, ';'); 26 var field_iter = std.mem.splitScalar(u8, line, ';');
@@ -42,16 +36,21 @@ pub fn main() !void {
42 12 => { 36 12 => {
43 // Simple uppercase mapping 37 // Simple uppercase mapping
44 if (field.len == 0) continue :lines; 38 if (field.len == 0) continue :lines;
45 try writer.writeInt(i24, cp, endian); 39 try file_writer.interface.writeInt(i24, cp, endian);
46 const mapping = try std.fmt.parseInt(i24, field, 16); 40 const mapping = try std.fmt.parseInt(i24, field, 16);
47 try writer.writeInt(i24, mapping - cp, endian); 41 try file_writer.interface.writeInt(i24, mapping - cp, endian);
48 }, 42 },
49 43
50 else => {}, 44 else => {},
51 } 45 }
52 } 46 }
47 } else |err| switch (err) {
48 error.EndOfStream => {},
49 else => {
50 return err;
51 },
53 } 52 }
54 53
55 try writer.writeInt(u24, 0, endian); 54 try file_writer.interface.writeInt(u24, 0, endian);
56 try out_comp.flush(); 55 try file_writer.interface.flush();
57} 56}
diff --git a/codegen/wbp.zig b/codegen/wbp.zig
index 741103e..33eeea5 100644
--- a/codegen/wbp.zig
+++ b/codegen/wbp.zig
@@ -43,7 +43,7 @@ const BlockMap = std.HashMap(
43 std.hash_map.default_max_load_percentage, 43 std.hash_map.default_max_load_percentage,
44); 44);
45 45
46pub fn main() !void { 46pub fn main() anyerror!void {
47 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 47 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
48 defer arena.deinit(); 48 defer arena.deinit();
49 const allocator = arena.allocator(); 49 const allocator = arena.allocator();
@@ -51,15 +51,11 @@ pub fn main() !void {
51 var flat_map = std.AutoHashMap(u21, u5).init(allocator); 51 var flat_map = std.AutoHashMap(u21, u5).init(allocator);
52 defer flat_map.deinit(); 52 defer flat_map.deinit();
53 53
54 var line_buf: [4096]u8 = undefined;
55
56 // Process HangulSyllableType.txt 54 // Process HangulSyllableType.txt
57 var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{}); 55 const in_file = @embedFile("WordBreakProperty.txt");
58 defer in_file.close(); 56 var in_reader = std.io.Reader.fixed(in_file);
59 var in_buf = std.io.bufferedReader(in_file.reader()); 57 while (in_reader.takeDelimiterInclusive('\n')) |took| {
60 const in_reader = in_buf.reader(); 58 const line = std.mem.trimRight(u8, took, "\n");
61
62 while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
63 if (line.len == 0 or line[0] == '#') continue; 59 if (line.len == 0 or line[0] == '#') continue;
64 60
65 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; 61 const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
@@ -90,15 +86,19 @@ pub fn main() !void {
90 else => {}, 86 else => {},
91 } 87 }
92 } 88 }
89 } else |err| switch (err) {
90 error.EndOfStream => {},
91 else => {
92 return err;
93 },
93 } 94 }
94
95 var blocks_map = BlockMap.init(allocator); 95 var blocks_map = BlockMap.init(allocator);
96 defer blocks_map.deinit(); 96 defer blocks_map.deinit();
97 97
98 var stage1 = std.ArrayList(u16).init(allocator); 98 var stage1 = std.array_list.Managed(u16).init(allocator);
99 defer stage1.deinit(); 99 defer stage1.deinit();
100 100
101 var stage2 = std.ArrayList(u5).init(allocator); 101 var stage2 = std.array_list.Managed(u5).init(allocator);
102 defer stage2.deinit(); 102 defer stage2.deinit();
103 103
104 var block: Block = [_]u5{0} ** block_size; 104 var block: Block = [_]u5{0} ** block_size;
@@ -129,18 +129,17 @@ pub fn main() !void {
129 _ = args_iter.skip(); 129 _ = args_iter.skip();
130 const output_path = args_iter.next() orelse @panic("No output file arg!"); 130 const output_path = args_iter.next() orelse @panic("No output file arg!");
131 131
132 const compressor = std.compress.flate.deflate.compressor; 132 var write_buf: [4096]u8 = undefined;
133 var out_file = try std.fs.cwd().createFile(output_path, .{}); 133 var out_file = try std.fs.cwd().createFile(output_path, .{});
134 defer out_file.close(); 134 defer out_file.close();
135 var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best }); 135 var writer = out_file.writer(&write_buf);
136 const writer = out_comp.writer();
137 136
138 const endian = builtin.cpu.arch.endian(); 137 const endian = builtin.cpu.arch.endian();
139 try writer.writeInt(u16, @intCast(stage1.items.len), endian); 138 try writer.interface.writeInt(u16, @intCast(stage1.items.len), endian);
140 for (stage1.items) |i| try writer.writeInt(u16, i, endian); 139 for (stage1.items) |i| try writer.interface.writeInt(u16, i, endian);
141 140
142 try writer.writeInt(u16, @intCast(stage2.items.len), endian); 141 try writer.interface.writeInt(u16, @intCast(stage2.items.len), endian);
143 for (stage2.items) |i| try writer.writeInt(u8, i, endian); 142 for (stage2.items) |i| try writer.interface.writeInt(u8, i, endian);
144 143
145 try out_comp.flush(); 144 try writer.interface.flush();
146} 145}