diff options
| author | 2024-03-25 20:05:36 -0400 | |
|---|---|---|
| committer | 2024-03-25 20:05:36 -0400 | |
| commit | 961d2ec2f9075f30e9264ed2db6c394cfa5967f9 (patch) | |
| tree | 4c6988e2de888c17507566c9d90d1e755ec4346f | |
| parent | NumericData (diff) | |
| download | zg-961d2ec2f9075f30e9264ed2db6c394cfa5967f9.tar.gz zg-961d2ec2f9075f30e9264ed2db6c394cfa5967f9.tar.xz zg-961d2ec2f9075f30e9264ed2db6c394cfa5967f9.zip | |
CaseData
| -rw-r--r-- | build.zig | 64 | ||||
| -rw-r--r-- | codegen/case_prop.zig | 136 | ||||
| -rw-r--r-- | codegen/lower.zig | 58 | ||||
| -rw-r--r-- | codegen/numeric.zig | 45 | ||||
| -rw-r--r-- | codegen/title.zig | 58 | ||||
| -rw-r--r-- | codegen/upper.zig | 58 | ||||
| -rw-r--r-- | src/CaseData.zig | 223 |
7 files changed, 613 insertions, 29 deletions
| @@ -107,6 +107,46 @@ pub fn build(b: *std.Build) void { | |||
| 107 | const run_num_gen_exe = b.addRunArtifact(num_gen_exe); | 107 | const run_num_gen_exe = b.addRunArtifact(num_gen_exe); |
| 108 | const num_gen_out = run_num_gen_exe.addOutputFileArg("numeric.bin.z"); | 108 | const num_gen_out = run_num_gen_exe.addOutputFileArg("numeric.bin.z"); |
| 109 | 109 | ||
| 110 | // Letter case properties | ||
| 111 | const case_prop_gen_exe = b.addExecutable(.{ | ||
| 112 | .name = "case_prop", | ||
| 113 | .root_source_file = .{ .path = "codegen/case_prop.zig" }, | ||
| 114 | .target = b.host, | ||
| 115 | .optimize = .Debug, | ||
| 116 | }); | ||
| 117 | const run_case_prop_gen_exe = b.addRunArtifact(case_prop_gen_exe); | ||
| 118 | const case_prop_gen_out = run_case_prop_gen_exe.addOutputFileArg("case_prop.bin.z"); | ||
| 119 | |||
| 120 | // Uppercase mappings | ||
| 121 | const upper_gen_exe = b.addExecutable(.{ | ||
| 122 | .name = "upper", | ||
| 123 | .root_source_file = .{ .path = "codegen/upper.zig" }, | ||
| 124 | .target = b.host, | ||
| 125 | .optimize = .Debug, | ||
| 126 | }); | ||
| 127 | const run_upper_gen_exe = b.addRunArtifact(upper_gen_exe); | ||
| 128 | const upper_gen_out = run_upper_gen_exe.addOutputFileArg("upper.bin.z"); | ||
| 129 | |||
| 130 | // Lowercase mappings | ||
| 131 | const lower_gen_exe = b.addExecutable(.{ | ||
| 132 | .name = "lower", | ||
| 133 | .root_source_file = .{ .path = "codegen/lower.zig" }, | ||
| 134 | .target = b.host, | ||
| 135 | .optimize = .Debug, | ||
| 136 | }); | ||
| 137 | const run_lower_gen_exe = b.addRunArtifact(lower_gen_exe); | ||
| 138 | const lower_gen_out = run_lower_gen_exe.addOutputFileArg("lower.bin.z"); | ||
| 139 | |||
| 140 | // Titlecase mappings | ||
| 141 | const title_gen_exe = b.addExecutable(.{ | ||
| 142 | .name = "title", | ||
| 143 | .root_source_file = .{ .path = "codegen/title.zig" }, | ||
| 144 | .target = b.host, | ||
| 145 | .optimize = .Debug, | ||
| 146 | }); | ||
| 147 | const run_title_gen_exe = b.addRunArtifact(title_gen_exe); | ||
| 148 | const title_gen_out = run_title_gen_exe.addOutputFileArg("title.bin.z"); | ||
| 149 | |||
| 110 | // Modules we provide | 150 | // Modules we provide |
| 111 | // Code points | 151 | // Code points |
| 112 | const code_point = b.addModule("code_point", .{ | 152 | const code_point = b.addModule("code_point", .{ |
| @@ -221,7 +261,7 @@ pub fn build(b: *std.Build) void { | |||
| 221 | }); | 261 | }); |
| 222 | gencat_data.addAnonymousImport("gencat", .{ .root_source_file = gencat_gen_out }); | 262 | gencat_data.addAnonymousImport("gencat", .{ .root_source_file = gencat_gen_out }); |
| 223 | 263 | ||
| 224 | // Case | 264 | // Case folding |
| 225 | const fold_data = b.createModule(.{ | 265 | const fold_data = b.createModule(.{ |
| 226 | .root_source_file = .{ .path = "src/FoldData.zig" }, | 266 | .root_source_file = .{ .path = "src/FoldData.zig" }, |
| 227 | .target = target, | 267 | .target = target, |
| @@ -246,6 +286,18 @@ pub fn build(b: *std.Build) void { | |||
| 246 | }); | 286 | }); |
| 247 | num_data.addAnonymousImport("numeric", .{ .root_source_file = num_gen_out }); | 287 | num_data.addAnonymousImport("numeric", .{ .root_source_file = num_gen_out }); |
| 248 | 288 | ||
| 289 | // Letter case | ||
| 290 | const case_data = b.createModule(.{ | ||
| 291 | .root_source_file = .{ .path = "src/CaseData.zig" }, | ||
| 292 | .target = target, | ||
| 293 | .optimize = optimize, | ||
| 294 | }); | ||
| 295 | case_data.addImport("code_point", code_point); | ||
| 296 | case_data.addAnonymousImport("case_prop", .{ .root_source_file = case_prop_gen_out }); | ||
| 297 | case_data.addAnonymousImport("upper", .{ .root_source_file = upper_gen_out }); | ||
| 298 | case_data.addAnonymousImport("lower", .{ .root_source_file = lower_gen_out }); | ||
| 299 | case_data.addAnonymousImport("title", .{ .root_source_file = title_gen_out }); | ||
| 300 | |||
| 249 | // Benchmark rig | 301 | // Benchmark rig |
| 250 | const exe = b.addExecutable(.{ | 302 | const exe = b.addExecutable(.{ |
| 251 | .name = "zg", | 303 | .name = "zg", |
| @@ -274,12 +326,12 @@ pub fn build(b: *std.Build) void { | |||
| 274 | 326 | ||
| 275 | // Tests | 327 | // Tests |
| 276 | const exe_unit_tests = b.addTest(.{ | 328 | const exe_unit_tests = b.addTest(.{ |
| 277 | .root_source_file = .{ .path = "src/NumericData.zig" }, | 329 | .root_source_file = .{ .path = "src/CaseData.zig" }, |
| 278 | .target = target, | 330 | .target = target, |
| 279 | .optimize = optimize, | 331 | .optimize = optimize, |
| 280 | }); | 332 | }); |
| 281 | // exe_unit_tests.root_module.addImport("ascii", ascii); | 333 | // exe_unit_tests.root_module.addImport("ascii", ascii); |
| 282 | // exe_unit_tests.root_module.addImport("code_point", code_point); | 334 | exe_unit_tests.root_module.addImport("code_point", code_point); |
| 283 | // exe_unit_tests.root_module.addImport("GraphemeData", grapheme_data); | 335 | // exe_unit_tests.root_module.addImport("GraphemeData", grapheme_data); |
| 284 | // exe_unit_tests.root_module.addImport("grapheme", grapheme); | 336 | // exe_unit_tests.root_module.addImport("grapheme", grapheme); |
| 285 | // exe_unit_tests.root_module.addImport("ziglyph", ziglyph.module("ziglyph")); | 337 | // exe_unit_tests.root_module.addImport("ziglyph", ziglyph.module("ziglyph")); |
| @@ -288,7 +340,11 @@ pub fn build(b: *std.Build) void { | |||
| 288 | // exe_unit_tests.root_module.addImport("NormData", norm_data); | 340 | // exe_unit_tests.root_module.addImport("NormData", norm_data); |
| 289 | // exe_unit_tests.root_module.addImport("Normalize", norm); | 341 | // exe_unit_tests.root_module.addImport("Normalize", norm); |
| 290 | // exe_unit_tests.root_module.addImport("FoldData", fold_data); | 342 | // exe_unit_tests.root_module.addImport("FoldData", fold_data); |
| 291 | exe_unit_tests.root_module.addAnonymousImport("numeric", .{ .root_source_file = num_gen_out }); | 343 | // exe_unit_tests.root_module.addAnonymousImport("numeric", .{ .root_source_file = num_gen_out }); |
| 344 | exe_unit_tests.root_module.addAnonymousImport("case_prop", .{ .root_source_file = case_prop_gen_out }); | ||
| 345 | exe_unit_tests.root_module.addAnonymousImport("upper", .{ .root_source_file = upper_gen_out }); | ||
| 346 | exe_unit_tests.root_module.addAnonymousImport("lower", .{ .root_source_file = lower_gen_out }); | ||
| 347 | exe_unit_tests.root_module.addAnonymousImport("title", .{ .root_source_file = title_gen_out }); | ||
| 292 | // exe_unit_tests.filter = "nfd !ASCII"; | 348 | // exe_unit_tests.filter = "nfd !ASCII"; |
| 293 | 349 | ||
| 294 | const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests); | 350 | const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests); |
diff --git a/codegen/case_prop.zig b/codegen/case_prop.zig new file mode 100644 index 0000000..ce7ee0d --- /dev/null +++ b/codegen/case_prop.zig | |||
| @@ -0,0 +1,136 @@ | |||
| 1 | const std = @import("std"); | ||
| 2 | const builtin = @import("builtin"); | ||
| 3 | const mem = std.mem; | ||
| 4 | |||
| 5 | const block_size = 256; | ||
| 6 | const Block = [block_size]u8; | ||
| 7 | |||
| 8 | const BlockMap = std.HashMap( | ||
| 9 | Block, | ||
| 10 | u16, | ||
| 11 | struct { | ||
| 12 | pub fn hash(_: @This(), k: Block) u64 { | ||
| 13 | var hasher = std.hash.Wyhash.init(0); | ||
| 14 | std.hash.autoHashStrat(&hasher, k, .DeepRecursive); | ||
| 15 | return hasher.final(); | ||
| 16 | } | ||
| 17 | |||
| 18 | pub fn eql(_: @This(), a: Block, b: Block) bool { | ||
| 19 | return mem.eql(u8, &a, &b); | ||
| 20 | } | ||
| 21 | }, | ||
| 22 | std.hash_map.default_max_load_percentage, | ||
| 23 | ); | ||
| 24 | |||
| 25 | pub fn main() !void { | ||
| 26 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | ||
| 27 | defer arena.deinit(); | ||
| 28 | const allocator = arena.allocator(); | ||
| 29 | |||
| 30 | var flat_map = std.AutoHashMap(u21, u8).init(allocator); | ||
| 31 | defer flat_map.deinit(); | ||
| 32 | |||
| 33 | var line_buf: [4096]u8 = undefined; | ||
| 34 | |||
| 35 | // Process DerivedCoreProperties.txt | ||
| 36 | var in_file = try std.fs.cwd().openFile("data/unicode/DerivedCoreProperties.txt", .{}); | ||
| 37 | defer in_file.close(); | ||
| 38 | var in_buf = std.io.bufferedReader(in_file.reader()); | ||
| 39 | const in_reader = in_buf.reader(); | ||
| 40 | |||
| 41 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | ||
| 42 | if (line.len == 0 or line[0] == '#') continue; | ||
| 43 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | ||
| 44 | |||
| 45 | var field_iter = mem.tokenizeAny(u8, no_comment, "; "); | ||
| 46 | var current_code: [2]u21 = undefined; | ||
| 47 | |||
| 48 | var i: usize = 0; | ||
| 49 | while (field_iter.next()) |field| : (i += 1) { | ||
| 50 | switch (i) { | ||
| 51 | 0 => { | ||
| 52 | // Code point(s) | ||
| 53 | if (mem.indexOf(u8, field, "..")) |dots| { | ||
| 54 | current_code = .{ | ||
| 55 | try std.fmt.parseInt(u21, field[0..dots], 16), | ||
| 56 | try std.fmt.parseInt(u21, field[dots + 2 ..], 16), | ||
| 57 | }; | ||
| 58 | } else { | ||
| 59 | const code = try std.fmt.parseInt(u21, field, 16); | ||
| 60 | current_code = .{ code, code }; | ||
| 61 | } | ||
| 62 | }, | ||
| 63 | 1 => { | ||
| 64 | // Props | ||
| 65 | var bit: u8 = 0; | ||
| 66 | |||
| 67 | if (mem.eql(u8, field, "Lowercase")) bit = 1; | ||
| 68 | if (mem.eql(u8, field, "Uppercase")) bit = 2; | ||
| 69 | if (mem.eql(u8, field, "Cased")) bit = 4; | ||
| 70 | |||
| 71 | if (bit != 0) { | ||
| 72 | for (current_code[0]..current_code[1] + 1) |cp| { | ||
| 73 | const gop = try flat_map.getOrPut(@intCast(cp)); | ||
| 74 | if (!gop.found_existing) gop.value_ptr.* = 0; | ||
| 75 | gop.value_ptr.* |= bit; | ||
| 76 | } | ||
| 77 | } | ||
| 78 | }, | ||
| 79 | else => {}, | ||
| 80 | } | ||
| 81 | } | ||
| 82 | } | ||
| 83 | |||
| 84 | var blocks_map = BlockMap.init(allocator); | ||
| 85 | defer blocks_map.deinit(); | ||
| 86 | |||
| 87 | var stage1 = std.ArrayList(u16).init(allocator); | ||
| 88 | defer stage1.deinit(); | ||
| 89 | |||
| 90 | var stage2 = std.ArrayList(u8).init(allocator); | ||
| 91 | defer stage2.deinit(); | ||
| 92 | |||
| 93 | var block: Block = [_]u8{0} ** block_size; | ||
| 94 | var block_len: u16 = 0; | ||
| 95 | |||
| 96 | for (0..0x110000) |i| { | ||
| 97 | const cp: u21 = @intCast(i); | ||
| 98 | const prop = flat_map.get(cp) orelse 0; | ||
| 99 | |||
| 100 | // Process block | ||
| 101 | block[block_len] = prop; | ||
| 102 | block_len += 1; | ||
| 103 | |||
| 104 | if (block_len < block_size and cp != 0x10ffff) continue; | ||
| 105 | |||
| 106 | const gop = try blocks_map.getOrPut(block); | ||
| 107 | if (!gop.found_existing) { | ||
| 108 | gop.value_ptr.* = @intCast(stage2.items.len); | ||
| 109 | try stage2.appendSlice(&block); | ||
| 110 | } | ||
| 111 | |||
| 112 | try stage1.append(gop.value_ptr.*); | ||
| 113 | block_len = 0; | ||
| 114 | } | ||
| 115 | |||
| 116 | var args_iter = try std.process.argsWithAllocator(allocator); | ||
| 117 | defer args_iter.deinit(); | ||
| 118 | _ = args_iter.skip(); | ||
| 119 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | ||
| 120 | |||
| 121 | const compressor = std.compress.deflate.compressor; | ||
| 122 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | ||
| 123 | defer out_file.close(); | ||
| 124 | var out_comp = try compressor(allocator, out_file.writer(), .{ .level = .best_compression }); | ||
| 125 | defer out_comp.deinit(); | ||
| 126 | const writer = out_comp.writer(); | ||
| 127 | |||
| 128 | const endian = builtin.cpu.arch.endian(); | ||
| 129 | try writer.writeInt(u16, @intCast(stage1.items.len), endian); | ||
| 130 | for (stage1.items) |i| try writer.writeInt(u16, i, endian); | ||
| 131 | |||
| 132 | try writer.writeInt(u16, @intCast(stage2.items.len), endian); | ||
| 133 | try writer.writeAll(stage2.items); | ||
| 134 | |||
| 135 | try out_comp.flush(); | ||
| 136 | } | ||
diff --git a/codegen/lower.zig b/codegen/lower.zig new file mode 100644 index 0000000..5a1f1b3 --- /dev/null +++ b/codegen/lower.zig | |||
| @@ -0,0 +1,58 @@ | |||
| 1 | const std = @import("std"); | ||
| 2 | const builtin = @import("builtin"); | ||
| 3 | |||
| 4 | pub fn main() !void { | ||
| 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | ||
| 6 | defer arena.deinit(); | ||
| 7 | const allocator = arena.allocator(); | ||
| 8 | |||
| 9 | // Process UnicodeData.txt | ||
| 10 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); | ||
| 11 | defer in_file.close(); | ||
| 12 | var in_buf = std.io.bufferedReader(in_file.reader()); | ||
| 13 | const in_reader = in_buf.reader(); | ||
| 14 | |||
| 15 | var args_iter = try std.process.argsWithAllocator(allocator); | ||
| 16 | defer args_iter.deinit(); | ||
| 17 | _ = args_iter.skip(); | ||
| 18 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | ||
| 19 | |||
| 20 | const compressor = std.compress.deflate.compressor; | ||
| 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | ||
| 22 | defer out_file.close(); | ||
| 23 | var out_comp = try compressor(allocator, out_file.writer(), .{ .level = .best_compression }); | ||
| 24 | defer out_comp.deinit(); | ||
| 25 | const writer = out_comp.writer(); | ||
| 26 | |||
| 27 | const endian = builtin.cpu.arch.endian(); | ||
| 28 | var line_buf: [4096]u8 = undefined; | ||
| 29 | |||
| 30 | lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | ||
| 31 | if (line.len == 0) continue; | ||
| 32 | |||
| 33 | var field_iter = std.mem.splitScalar(u8, line, ';'); | ||
| 34 | var cps: [2]u24 = undefined; | ||
| 35 | |||
| 36 | var i: usize = 0; | ||
| 37 | while (field_iter.next()) |field| : (i += 1) { | ||
| 38 | switch (i) { | ||
| 39 | 0 => cps[0] = try std.fmt.parseInt(u24, field, 16), | ||
| 40 | |||
| 41 | 13 => { | ||
| 42 | // Simple lowercase mapping | ||
| 43 | if (field.len == 0) continue :lines; | ||
| 44 | cps[1] = try std.fmt.parseInt(u24, field, 16); | ||
| 45 | }, | ||
| 46 | |||
| 47 | 2 => if (line[0] == '<') continue :lines, | ||
| 48 | |||
| 49 | else => {}, | ||
| 50 | } | ||
| 51 | } | ||
| 52 | |||
| 53 | for (&cps) |cp| try writer.writeInt(u24, cp, endian); | ||
| 54 | } | ||
| 55 | |||
| 56 | try writer.writeInt(u24, 0, endian); | ||
| 57 | try out_comp.flush(); | ||
| 58 | } | ||
diff --git a/codegen/numeric.zig b/codegen/numeric.zig index 80fc7c2..ad8490c 100644 --- a/codegen/numeric.zig +++ b/codegen/numeric.zig | |||
| @@ -1,5 +1,6 @@ | |||
| 1 | const std = @import("std"); | 1 | const std = @import("std"); |
| 2 | const builtin = @import("builtin"); | 2 | const builtin = @import("builtin"); |
| 3 | const mem = std.mem; | ||
| 3 | 4 | ||
| 4 | const block_size = 256; | 5 | const block_size = 256; |
| 5 | const Block = [block_size]u8; | 6 | const Block = [block_size]u8; |
| @@ -15,7 +16,7 @@ const BlockMap = std.HashMap( | |||
| 15 | } | 16 | } |
| 16 | 17 | ||
| 17 | pub fn eql(_: @This(), a: Block, b: Block) bool { | 18 | pub fn eql(_: @This(), a: Block, b: Block) bool { |
| 18 | return std.mem.eql(u8, &a, &b); | 19 | return mem.eql(u8, &a, &b); |
| 19 | } | 20 | } |
| 20 | }, | 21 | }, |
| 21 | std.hash_map.default_max_load_percentage, | 22 | std.hash_map.default_max_load_percentage, |
| @@ -31,17 +32,17 @@ pub fn main() !void { | |||
| 31 | 32 | ||
| 32 | var line_buf: [4096]u8 = undefined; | 33 | var line_buf: [4096]u8 = undefined; |
| 33 | 34 | ||
| 34 | // Process DerivedCombiningClass.txt | 35 | // Process DerivedNumericType.txt |
| 35 | var num_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); | 36 | var in_file = try std.fs.cwd().openFile("data/unicode/extracted/DerivedNumericType.txt", .{}); |
| 36 | defer num_file.close(); | 37 | defer in_file.close(); |
| 37 | var num_buf = std.io.bufferedReader(num_file.reader()); | 38 | var in_buf = std.io.bufferedReader(in_file.reader()); |
| 38 | const num_reader = num_buf.reader(); | 39 | const in_reader = in_buf.reader(); |
| 39 | 40 | ||
| 40 | while (try num_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | 41 | while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { |
| 41 | if (line.len == 0 or line[0] == '#') continue; | 42 | if (line.len == 0 or line[0] == '#') continue; |
| 42 | const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; | 43 | const no_comment = if (mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line; |
| 43 | 44 | ||
| 44 | var field_iter = std.mem.tokenizeAny(u8, no_comment, "; "); | 45 | var field_iter = mem.tokenizeAny(u8, no_comment, "; "); |
| 45 | var current_code: [2]u21 = undefined; | 46 | var current_code: [2]u21 = undefined; |
| 46 | 47 | ||
| 47 | var i: usize = 0; | 48 | var i: usize = 0; |
| @@ -49,7 +50,7 @@ pub fn main() !void { | |||
| 49 | switch (i) { | 50 | switch (i) { |
| 50 | 0 => { | 51 | 0 => { |
| 51 | // Code point(s) | 52 | // Code point(s) |
| 52 | if (std.mem.indexOf(u8, field, "..")) |dots| { | 53 | if (mem.indexOf(u8, field, "..")) |dots| { |
| 53 | current_code = .{ | 54 | current_code = .{ |
| 54 | try std.fmt.parseInt(u21, field[0..dots], 16), | 55 | try std.fmt.parseInt(u21, field[0..dots], 16), |
| 55 | try std.fmt.parseInt(u21, field[dots + 2 ..], 16), | 56 | try std.fmt.parseInt(u21, field[dots + 2 ..], 16), |
| @@ -61,25 +62,19 @@ pub fn main() !void { | |||
| 61 | }, | 62 | }, |
| 62 | 1 => { | 63 | 1 => { |
| 63 | // Numeric type | 64 | // Numeric type |
| 64 | if (std.mem.eql(u8, field, "Numeric")) { | 65 | var bit: u8 = 0; |
| 65 | for (current_code[0]..current_code[1] + 1) |cp| { | 66 | |
| 66 | const gop = try flat_map.getOrPut(@intCast(cp)); | 67 | if (mem.eql(u8, field, "Numeric")) bit = 1; |
| 67 | if (!gop.found_existing) gop.value_ptr.* = 0; | 68 | if (mem.eql(u8, field, "Digit")) bit = 2; |
| 68 | gop.value_ptr.* |= 1; | 69 | if (mem.eql(u8, field, "Decimal")) bit = 4; |
| 69 | } | 70 | |
| 70 | } else if (std.mem.eql(u8, field, "Digit")) { | 71 | if (bit != 0) { |
| 71 | for (current_code[0]..current_code[1] + 1) |cp| { | ||
| 72 | const gop = try flat_map.getOrPut(@intCast(cp)); | ||
| 73 | if (!gop.found_existing) gop.value_ptr.* = 0; | ||
| 74 | gop.value_ptr.* |= 2; | ||
| 75 | } | ||
| 76 | } else if (std.mem.eql(u8, field, "Decimal")) { | ||
| 77 | for (current_code[0]..current_code[1] + 1) |cp| { | 72 | for (current_code[0]..current_code[1] + 1) |cp| { |
| 78 | const gop = try flat_map.getOrPut(@intCast(cp)); | 73 | const gop = try flat_map.getOrPut(@intCast(cp)); |
| 79 | if (!gop.found_existing) gop.value_ptr.* = 0; | 74 | if (!gop.found_existing) gop.value_ptr.* = 0; |
| 80 | gop.value_ptr.* |= 4; | 75 | gop.value_ptr.* |= bit; |
| 81 | } | 76 | } |
| 82 | } else continue; | 77 | } |
| 83 | }, | 78 | }, |
| 84 | else => {}, | 79 | else => {}, |
| 85 | } | 80 | } |
diff --git a/codegen/title.zig b/codegen/title.zig new file mode 100644 index 0000000..653b812 --- /dev/null +++ b/codegen/title.zig | |||
| @@ -0,0 +1,58 @@ | |||
| 1 | const std = @import("std"); | ||
| 2 | const builtin = @import("builtin"); | ||
| 3 | |||
| 4 | pub fn main() !void { | ||
| 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | ||
| 6 | defer arena.deinit(); | ||
| 7 | const allocator = arena.allocator(); | ||
| 8 | |||
| 9 | // Process UnicodeData.txt | ||
| 10 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); | ||
| 11 | defer in_file.close(); | ||
| 12 | var in_buf = std.io.bufferedReader(in_file.reader()); | ||
| 13 | const in_reader = in_buf.reader(); | ||
| 14 | |||
| 15 | var args_iter = try std.process.argsWithAllocator(allocator); | ||
| 16 | defer args_iter.deinit(); | ||
| 17 | _ = args_iter.skip(); | ||
| 18 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | ||
| 19 | |||
| 20 | const compressor = std.compress.deflate.compressor; | ||
| 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | ||
| 22 | defer out_file.close(); | ||
| 23 | var out_comp = try compressor(allocator, out_file.writer(), .{ .level = .best_compression }); | ||
| 24 | defer out_comp.deinit(); | ||
| 25 | const writer = out_comp.writer(); | ||
| 26 | |||
| 27 | const endian = builtin.cpu.arch.endian(); | ||
| 28 | var line_buf: [4096]u8 = undefined; | ||
| 29 | |||
| 30 | lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | ||
| 31 | if (line.len == 0) continue; | ||
| 32 | |||
| 33 | var field_iter = std.mem.splitScalar(u8, line, ';'); | ||
| 34 | var cps: [2]u24 = undefined; | ||
| 35 | |||
| 36 | var i: usize = 0; | ||
| 37 | while (field_iter.next()) |field| : (i += 1) { | ||
| 38 | switch (i) { | ||
| 39 | 0 => cps[0] = try std.fmt.parseInt(u24, field, 16), | ||
| 40 | |||
| 41 | 14 => { | ||
| 42 | // Simple titlecase mapping | ||
| 43 | if (field.len == 0) continue :lines; | ||
| 44 | cps[1] = try std.fmt.parseInt(u24, field, 16); | ||
| 45 | }, | ||
| 46 | |||
| 47 | 2 => if (line[0] == '<') continue :lines, | ||
| 48 | |||
| 49 | else => {}, | ||
| 50 | } | ||
| 51 | } | ||
| 52 | |||
| 53 | for (&cps) |cp| try writer.writeInt(u24, cp, endian); | ||
| 54 | } | ||
| 55 | |||
| 56 | try writer.writeInt(u24, 0, endian); | ||
| 57 | try out_comp.flush(); | ||
| 58 | } | ||
diff --git a/codegen/upper.zig b/codegen/upper.zig new file mode 100644 index 0000000..6fea608 --- /dev/null +++ b/codegen/upper.zig | |||
| @@ -0,0 +1,58 @@ | |||
| 1 | const std = @import("std"); | ||
| 2 | const builtin = @import("builtin"); | ||
| 3 | |||
| 4 | pub fn main() !void { | ||
| 5 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | ||
| 6 | defer arena.deinit(); | ||
| 7 | const allocator = arena.allocator(); | ||
| 8 | |||
| 9 | // Process UnicodeData.txt | ||
| 10 | var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{}); | ||
| 11 | defer in_file.close(); | ||
| 12 | var in_buf = std.io.bufferedReader(in_file.reader()); | ||
| 13 | const in_reader = in_buf.reader(); | ||
| 14 | |||
| 15 | var args_iter = try std.process.argsWithAllocator(allocator); | ||
| 16 | defer args_iter.deinit(); | ||
| 17 | _ = args_iter.skip(); | ||
| 18 | const output_path = args_iter.next() orelse @panic("No output file arg!"); | ||
| 19 | |||
| 20 | const compressor = std.compress.deflate.compressor; | ||
| 21 | var out_file = try std.fs.cwd().createFile(output_path, .{}); | ||
| 22 | defer out_file.close(); | ||
| 23 | var out_comp = try compressor(allocator, out_file.writer(), .{ .level = .best_compression }); | ||
| 24 | defer out_comp.deinit(); | ||
| 25 | const writer = out_comp.writer(); | ||
| 26 | |||
| 27 | const endian = builtin.cpu.arch.endian(); | ||
| 28 | var line_buf: [4096]u8 = undefined; | ||
| 29 | |||
| 30 | lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| { | ||
| 31 | if (line.len == 0) continue; | ||
| 32 | |||
| 33 | var field_iter = std.mem.splitScalar(u8, line, ';'); | ||
| 34 | var cps: [2]u24 = undefined; | ||
| 35 | |||
| 36 | var i: usize = 0; | ||
| 37 | while (field_iter.next()) |field| : (i += 1) { | ||
| 38 | switch (i) { | ||
| 39 | 0 => cps[0] = try std.fmt.parseInt(u24, field, 16), | ||
| 40 | |||
| 41 | 12 => { | ||
| 42 | // Simple uppercase mapping | ||
| 43 | if (field.len == 0) continue :lines; | ||
| 44 | cps[1] = try std.fmt.parseInt(u24, field, 16); | ||
| 45 | }, | ||
| 46 | |||
| 47 | 2 => if (line[0] == '<') continue :lines, | ||
| 48 | |||
| 49 | else => {}, | ||
| 50 | } | ||
| 51 | } | ||
| 52 | |||
| 53 | for (&cps) |cp| try writer.writeInt(u24, cp, endian); | ||
| 54 | } | ||
| 55 | |||
| 56 | try writer.writeInt(u24, 0, endian); | ||
| 57 | try out_comp.flush(); | ||
| 58 | } | ||
diff --git a/src/CaseData.zig b/src/CaseData.zig new file mode 100644 index 0000000..38830e3 --- /dev/null +++ b/src/CaseData.zig | |||
| @@ -0,0 +1,223 @@ | |||
| 1 | const std = @import("std"); | ||
| 2 | const builtin = @import("builtin"); | ||
| 3 | const compress = std.compress; | ||
| 4 | const mem = std.mem; | ||
| 5 | const testing = std.testing; | ||
| 6 | const unicode = std.unicode; | ||
| 7 | |||
| 8 | const CodePointIterator = @import("code_point").Iterator; | ||
| 9 | |||
| 10 | allocator: mem.Allocator, | ||
| 11 | case_map: [][3]u21, | ||
| 12 | prop_s1: []u16 = undefined, | ||
| 13 | prop_s2: []u8 = undefined, | ||
| 14 | |||
| 15 | const Self = @This(); | ||
| 16 | |||
| 17 | pub fn init(allocator: mem.Allocator) !Self { | ||
| 18 | const decompressor = compress.deflate.decompressor; | ||
| 19 | const endian = builtin.cpu.arch.endian(); | ||
| 20 | |||
| 21 | var self = Self{ | ||
| 22 | .allocator = allocator, | ||
| 23 | .case_map = try allocator.alloc([3]u21, 0x110000), | ||
| 24 | }; | ||
| 25 | errdefer allocator.free(self.case_map); | ||
| 26 | |||
| 27 | for (0..0x110000) |i| { | ||
| 28 | const cp: u21 = @intCast(i); | ||
| 29 | self.case_map[cp] = .{ cp, cp, cp }; | ||
| 30 | } | ||
| 31 | |||
| 32 | // Uppercase | ||
| 33 | const upper_bytes = @embedFile("upper"); | ||
| 34 | var upper_fbs = std.io.fixedBufferStream(upper_bytes); | ||
| 35 | var upper_decomp = try decompressor(allocator, upper_fbs.reader(), null); | ||
| 36 | defer upper_decomp.deinit(); | ||
| 37 | var upper_reader = upper_decomp.reader(); | ||
| 38 | |||
| 39 | while (true) { | ||
| 40 | const cp = try upper_reader.readInt(u24, endian); | ||
| 41 | if (cp == 0) break; | ||
| 42 | self.case_map[cp][0] = @intCast(try upper_reader.readInt(u24, endian)); | ||
| 43 | } | ||
| 44 | |||
| 45 | // Lowercase | ||
| 46 | const lower_bytes = @embedFile("lower"); | ||
| 47 | var lower_fbs = std.io.fixedBufferStream(lower_bytes); | ||
| 48 | var lower_decomp = try decompressor(allocator, lower_fbs.reader(), null); | ||
| 49 | defer lower_decomp.deinit(); | ||
| 50 | var lower_reader = lower_decomp.reader(); | ||
| 51 | |||
| 52 | while (true) { | ||
| 53 | const cp = try lower_reader.readInt(u24, endian); | ||
| 54 | if (cp == 0) break; | ||
| 55 | self.case_map[cp][1] = @intCast(try lower_reader.readInt(u24, endian)); | ||
| 56 | } | ||
| 57 | |||
| 58 | // Titlercase | ||
| 59 | const title_bytes = @embedFile("title"); | ||
| 60 | var title_fbs = std.io.fixedBufferStream(title_bytes); | ||
| 61 | var title_decomp = try decompressor(allocator, title_fbs.reader(), null); | ||
| 62 | defer title_decomp.deinit(); | ||
| 63 | var title_reader = title_decomp.reader(); | ||
| 64 | |||
| 65 | while (true) { | ||
| 66 | const cp = try title_reader.readInt(u24, endian); | ||
| 67 | if (cp == 0) break; | ||
| 68 | self.case_map[cp][2] = @intCast(try title_reader.readInt(u24, endian)); | ||
| 69 | } | ||
| 70 | |||
| 71 | // Case properties | ||
| 72 | const cp_bytes = @embedFile("case_prop"); | ||
| 73 | var cp_fbs = std.io.fixedBufferStream(cp_bytes); | ||
| 74 | var cp_decomp = try decompressor(allocator, cp_fbs.reader(), null); | ||
| 75 | defer cp_decomp.deinit(); | ||
| 76 | var cp_reader = cp_decomp.reader(); | ||
| 77 | |||
| 78 | const stage_1_len: u16 = try cp_reader.readInt(u16, endian); | ||
| 79 | self.prop_s1 = try allocator.alloc(u16, stage_1_len); | ||
| 80 | errdefer allocator.free(self.prop_s1); | ||
| 81 | for (0..stage_1_len) |i| self.prop_s1[i] = try cp_reader.readInt(u16, endian); | ||
| 82 | |||
| 83 | const stage_2_len: u16 = try cp_reader.readInt(u16, endian); | ||
| 84 | self.prop_s2 = try allocator.alloc(u8, stage_2_len); | ||
| 85 | errdefer allocator.free(self.prop_s2); | ||
| 86 | _ = try cp_reader.readAll(self.prop_s2); | ||
| 87 | |||
| 88 | return self; | ||
| 89 | } | ||
| 90 | |||
| 91 | pub fn deinit(self: *Self) void { | ||
| 92 | self.allocator.free(self.case_map); | ||
| 93 | self.allocator.free(self.prop_s1); | ||
| 94 | self.allocator.free(self.prop_s2); | ||
| 95 | } | ||
| 96 | |||
| 97 | // Returns true if `cp` is either upper, lower, or title case. | ||
| 98 | pub inline fn isCased(self: Self, cp: u21) bool { | ||
| 99 | return self.prop_s2[self.prop_s1[cp >> 8] + (cp & 0xff)] & 4 == 4; | ||
| 100 | } | ||
| 101 | |||
| 102 | // Returns true if `cp` is uppercase. | ||
| 103 | pub fn isUpper(self: Self, cp: u21) bool { | ||
| 104 | if (!self.isCased(cp)) return true; | ||
| 105 | return self.prop_s2[self.prop_s1[cp >> 8] + (cp & 0xff)] & 2 == 2; | ||
| 106 | } | ||
| 107 | |||
| 108 | /// Returns true if `str` is all uppercase. | ||
| 109 | pub fn isUpperStr(self: Self, str: []const u8) bool { | ||
| 110 | var iter = CodePointIterator{ .bytes = str }; | ||
| 111 | |||
| 112 | return while (iter.next()) |cp| { | ||
| 113 | if (!self.isUpper(cp.code)) break false; | ||
| 114 | } else true; | ||
| 115 | } | ||
| 116 | |||
| 117 | test "isUpperStr" { | ||
| 118 | var cd = try init(testing.allocator); | ||
| 119 | defer cd.deinit(); | ||
| 120 | |||
| 121 | try testing.expect(cd.isUpperStr("HELLO, WORLD 2112!")); | ||
| 122 | try testing.expect(!cd.isUpperStr("hello, world 2112!")); | ||
| 123 | try testing.expect(!cd.isUpperStr("Hello, World 2112!")); | ||
| 124 | } | ||
| 125 | |||
| 126 | /// Returns a new string with all letters in uppercase. | ||
| 127 | /// Caller must free returned bytes with `allocator`. | ||
| 128 | pub fn toUpperStr( | ||
| 129 | self: Self, | ||
| 130 | allocator: mem.Allocator, | ||
| 131 | str: []const u8, | ||
| 132 | ) ![]u8 { | ||
| 133 | var bytes = std.ArrayList(u8).init(allocator); | ||
| 134 | defer bytes.deinit(); | ||
| 135 | |||
| 136 | var iter = CodePointIterator{ .bytes = str }; | ||
| 137 | var buf: [4]u8 = undefined; | ||
| 138 | |||
| 139 | while (iter.next()) |cp| { | ||
| 140 | const len = try unicode.utf8Encode(self.toUpper(cp.code), &buf); | ||
| 141 | try bytes.appendSlice(buf[0..len]); | ||
| 142 | } | ||
| 143 | |||
| 144 | return try bytes.toOwnedSlice(); | ||
| 145 | } | ||
| 146 | |||
| 147 | test "toUpperStr" { | ||
| 148 | var cd = try init(testing.allocator); | ||
| 149 | defer cd.deinit(); | ||
| 150 | |||
| 151 | const uppered = try cd.toUpperStr(testing.allocator, "Hello, World 2112!"); | ||
| 152 | defer testing.allocator.free(uppered); | ||
| 153 | try testing.expectEqualStrings("HELLO, WORLD 2112!", uppered); | ||
| 154 | } | ||
| 155 | |||
| 156 | /// Returns uppercase mapping for `cp`. | ||
| 157 | pub inline fn toUpper(self: Self, cp: u21) u21 { | ||
| 158 | return self.case_map[cp][0]; | ||
| 159 | } | ||
| 160 | |||
| 161 | // Returns true if `cp` is lowercase. | ||
| 162 | pub fn isLower(self: Self, cp: u21) bool { | ||
| 163 | if (!self.isCased(cp)) return true; | ||
| 164 | return self.prop_s2[self.prop_s1[cp >> 8] + (cp & 0xff)] & 1 == 1; | ||
| 165 | } | ||
| 166 | |||
| 167 | /// Returns lowercase mapping for `cp`. | ||
| 168 | pub inline fn toLower(self: Self, cp: u21) u21 { | ||
| 169 | return self.case_map[cp][1]; | ||
| 170 | } | ||
| 171 | |||
| 172 | /// Returns true if `str` is all lowercase. | ||
| 173 | pub fn isLowerStr(self: Self, str: []const u8) bool { | ||
| 174 | var iter = CodePointIterator{ .bytes = str }; | ||
| 175 | |||
| 176 | return while (iter.next()) |cp| { | ||
| 177 | if (!self.isLower(cp.code)) break false; | ||
| 178 | } else true; | ||
| 179 | } | ||
| 180 | |||
| 181 | test "isLowerStr" { | ||
| 182 | var cd = try init(testing.allocator); | ||
| 183 | defer cd.deinit(); | ||
| 184 | |||
| 185 | try testing.expect(cd.isLowerStr("hello, world 2112!")); | ||
| 186 | try testing.expect(!cd.isLowerStr("HELLO, WORLD 2112!")); | ||
| 187 | try testing.expect(!cd.isLowerStr("Hello, World 2112!")); | ||
| 188 | } | ||
| 189 | |||
| 190 | /// Returns a new string with all letters in lowercase. | ||
| 191 | /// Caller must free returned bytes with `allocator`. | ||
| 192 | pub fn toLowerStr( | ||
| 193 | self: Self, | ||
| 194 | allocator: mem.Allocator, | ||
| 195 | str: []const u8, | ||
| 196 | ) ![]u8 { | ||
| 197 | var bytes = std.ArrayList(u8).init(allocator); | ||
| 198 | defer bytes.deinit(); | ||
| 199 | |||
| 200 | var iter = CodePointIterator{ .bytes = str }; | ||
| 201 | var buf: [4]u8 = undefined; | ||
| 202 | |||
| 203 | while (iter.next()) |cp| { | ||
| 204 | const len = try unicode.utf8Encode(self.toLower(cp.code), &buf); | ||
| 205 | try bytes.appendSlice(buf[0..len]); | ||
| 206 | } | ||
| 207 | |||
| 208 | return try bytes.toOwnedSlice(); | ||
| 209 | } | ||
| 210 | |||
| 211 | test "toLowerStr" { | ||
| 212 | var cd = try init(testing.allocator); | ||
| 213 | defer cd.deinit(); | ||
| 214 | |||
| 215 | const lowered = try cd.toLowerStr(testing.allocator, "Hello, World 2112!"); | ||
| 216 | defer testing.allocator.free(lowered); | ||
| 217 | try testing.expectEqualStrings("hello, world 2112!", lowered); | ||
| 218 | } | ||
| 219 | |||
| 220 | /// Returns titlecase mapping for `cp`. | ||
| 221 | pub inline fn toTitle(self: Self, cp: u21) u21 { | ||
| 222 | return self.case_map[cp][2]; | ||
| 223 | } | ||