From f37875c6c4e42055735f1cda9fdbcb7ab11b80bc Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 13 Dec 2024 15:14:15 -0500 Subject: Add c0 and c1 control width options This allows a build of DisplayWidth to give characters in those classes a width, for cases where they'll be printed with a substitute in the final display. It also raises the size of possible characters from an i3 to an i4, to accommodate printing C1s as e.g. <80> or \u{80}. --- README.md | 4 +++- build.zig | 17 ++++++++++++++++- codegen/dwp.zig | 16 ++++++++-------- src/DisplayWidth.zig | 14 +++++++++----- src/WidthData.zig | 54 ++++++++++++++++++++++++++-------------------------- 5 files changed, 63 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index bad64fc..c4227a2 100644 --- a/README.md +++ b/README.md @@ -441,7 +441,7 @@ test "Display width" { } ``` -This has a build option, `"cjk"`, which will consider [ambiguous characters](https://www.unicode.org/reports/tr11/tr11-6.html) as double-width. +This module has build options. The first is `cjk`, which will consider [ambiguous characters](https://www.unicode.org/reports/tr11/tr11-6.html) as double-width. To choose this option, add it to the dependency like so: @@ -451,6 +451,8 @@ const zg = b.dependency("zg", .{ }); ``` +The other options are `c0_width` and `c1_width`. The standard behavior is to treat C0 and C1 control codes as zero-width, except for delete and backspace, which are -1 (the logic ensures that a `strWidth` is always at least 0). If printing control codes with replacement characters, it's necessary to assign these a width, hence the options. When provided these values must fit in an `i4`, this allows for C1s to be printed as `\u{80}` if desired. + ## Scripts Unicode categorizes code points by the Script in which they belong. A Script diff --git a/build.zig b/build.zig index 4c4a35a..ce362ea 100644 --- a/build.zig +++ b/build.zig @@ -16,10 +16,24 @@ pub fn build(b: *std.Build) void { const gbp_gen_out = run_gbp_gen_exe.addOutputFileArg("gbp.bin.z"); // Display width - const cjk = b.option(bool, "cjk", "Ambiguouse code points are wide (display width: 2).") orelse false; + const cjk = b.option(bool, "cjk", "Ambiguous code points are wide (display width: 2).") orelse false; const options = b.addOptions(); options.addOption(bool, "cjk", cjk); + // Visible Controls + const c0_width = b.option( + i4, + "c0_width", + "C0 controls have this width (default: 0, default -1)", + ); + options.addOption(?i4, "c0_width", c0_width); + const c1_width = b.option( + i4, + "c1_width", + "C1 controls have this width (default: 0)", + ); + options.addOption(?i4, "c1_width", c1_width); + const dwp_gen_exe = b.addExecutable(.{ .name = "dwp", .root_source_file = b.path("codegen/dwp.zig"), @@ -210,6 +224,7 @@ pub fn build(b: *std.Build) void { display_width.addImport("code_point", code_point); display_width.addImport("grapheme", grapheme); display_width.addImport("DisplayWidthData", width_data); + display_width.addOptions("options", options); // For testing // Normalization const ccc_data = b.createModule(.{ diff --git a/codegen/dwp.zig b/codegen/dwp.zig index c581eb6..5e5bf6a 100644 --- a/codegen/dwp.zig +++ b/codegen/dwp.zig @@ -4,7 +4,7 @@ const builtin = @import("builtin"); const options = @import("options"); const block_size = 256; -const Block = [block_size]i3; +const Block = [block_size]i4; const BlockMap = std.HashMap( Block, @@ -17,7 +17,7 @@ const BlockMap = std.HashMap( } pub fn eql(_: @This(), a: Block, b: Block) bool { - return std.mem.eql(i3, &a, &b); + return std.mem.eql(i4, &a, &b); } }, std.hash_map.default_max_load_percentage, @@ -28,7 +28,7 @@ pub fn main() !void { defer arena.deinit(); const allocator = arena.allocator(); - var flat_map = std.AutoHashMap(u21, i3).init(allocator); + var flat_map = std.AutoHashMap(u21, i4).init(allocator); defer flat_map.deinit(); var line_buf: [4096]u8 = undefined; @@ -147,10 +147,10 @@ pub fn main() !void { var stage1 = std.ArrayList(u16).init(allocator); defer stage1.deinit(); - var stage2 = std.ArrayList(i3).init(allocator); + var stage2 = std.ArrayList(i4).init(allocator); defer stage2.deinit(); - var block: Block = [_]i3{0} ** block_size; + var block: Block = [_]i4{0} ** block_size; var block_len: u16 = 0; for (0..0x110000) |i| { @@ -163,8 +163,8 @@ pub fn main() !void { 0x2e3b => width = 3, // C0/C1 control codes - 0...0x20, - 0x80...0xa0, + 0...0x20 => width = if (options.c0_width) |c0| c0 else 0, + 0x80...0x9f => width = if (options.c1_width) |c1| c1 else 0, // Line separator 0x2028, @@ -204,7 +204,7 @@ pub fn main() !void { if (cp == 0xad) width = 1; // Backspace and delete - if (cp == 0x8 or cp == 0x7f) width = -1; + if (cp == 0x8 or cp == 0x7f) width = if (options.c0_width) |c0| c0 else -1; // Process block block[block_len] = width; diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig index 621b8c1..04e6b0c 100644 --- a/src/DisplayWidth.zig +++ b/src/DisplayWidth.zig @@ -1,5 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); +const options = @import("options"); const ArrayList = std.ArrayList; const mem = std.mem; const simd = std.simd; @@ -60,6 +61,7 @@ test "strWidth" { const data = try DisplayWidthData.init(testing.allocator); defer data.deinit(); const self = Self{ .data = &data }; + const c0 = options.c0_width orelse 0; try testing.expectEqual(@as(usize, 5), self.strWidth("Hello\r\n")); try testing.expectEqual(@as(usize, 1), self.strWidth("\u{0065}\u{0301}")); @@ -74,19 +76,21 @@ test "strWidth" { try testing.expectEqual(@as(usize, 1), self.strWidth("\u{2764}")); // Default text presentation try testing.expectEqual(@as(usize, 1), self.strWidth("\u{2764}\u{FE0E}")); // Default text presentation with VS15 selector try testing.expectEqual(@as(usize, 2), self.strWidth("\u{2764}\u{FE0F}")); // Default text presentation with VS16 selector - try testing.expectEqual(@as(usize, 0), self.strWidth("A\x08")); // Backspace - try testing.expectEqual(@as(usize, 0), self.strWidth("\x7FA")); // DEL - try testing.expectEqual(@as(usize, 0), self.strWidth("\x7FA\x08\x08")); // never less than o + const expect_bs: usize = if (c0 == 0) 0 else 1 + c0; + try testing.expectEqual(expect_bs, self.strWidth("A\x08")); // Backspace + try testing.expectEqual(expect_bs, self.strWidth("\x7FA")); // DEL + const expect_long_del: usize = if (c0 == 0) 0 else 1 + (c0 * 3); + try testing.expectEqual(expect_long_del, self.strWidth("\x7FA\x08\x08")); // never less than 0 // wcwidth Python lib tests. See: https://github.com/jquast/wcwidth/blob/master/tests/test_core.py const empty = ""; try testing.expectEqual(@as(usize, 0), self.strWidth(empty)); const with_null = "hello\x00world"; - try testing.expectEqual(@as(usize, 10), self.strWidth(with_null)); + try testing.expectEqual(@as(usize, 10 + c0), self.strWidth(with_null)); const hello_jp = "コンニチハ, セカイ!"; try testing.expectEqual(@as(usize, 19), self.strWidth(hello_jp)); const control = "\x1b[0m"; - try testing.expectEqual(@as(usize, 3), self.strWidth(control)); + try testing.expectEqual(@as(usize, 3 + c0), self.strWidth(control)); const balinese = "\u{1B13}\u{1B28}\u{1B2E}\u{1B44}"; try testing.expectEqual(@as(usize, 3), self.strWidth(balinese)); diff --git a/src/WidthData.zig b/src/WidthData.zig index 1b7fb2e..d77879e 100644 --- a/src/WidthData.zig +++ b/src/WidthData.zig @@ -9,7 +9,7 @@ const GraphemeData = @import("GraphemeData"); allocator: mem.Allocator, g_data: GraphemeData, s1: []u16 = undefined, -s2: []i3 = undefined, +s2: []i4 = undefined, const Self = @This(); @@ -34,7 +34,7 @@ pub fn init(allocator: mem.Allocator) mem.Allocator.Error!Self { for (0..stage_1_len) |i| self.s1[i] = reader.readInt(u16, endian) catch unreachable; const stage_2_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s2 = try allocator.alloc(i3, stage_2_len); + self.s2 = try allocator.alloc(i4, stage_2_len); errdefer allocator.free(self.s2); for (0..stage_2_len) |i| self.s2[i] = @intCast(reader.readInt(i8, endian) catch unreachable); @@ -52,33 +52,33 @@ pub fn deinit(self: *const Self) void { /// 3, where BACKSPACE and DELETE return -1 and 3-em-dash returns 3. C0/C1 /// control codes return 0. If `cjk` is true, ambiguous code points return 2, /// otherwise they return 1. -pub fn codePointWidth(self: Self, cp: u21) i3 { +pub fn codePointWidth(self: Self, cp: u21) i4 { return self.s2[self.s1[cp >> 8] + (cp & 0xff)]; } test "codePointWidth" { - try testing.expectEqual(@as(i3, 0), codePointWidth(0x0000)); // null - try testing.expectEqual(@as(i3, -1), codePointWidth(0x8)); // \b - try testing.expectEqual(@as(i3, -1), codePointWidth(0x7f)); // DEL - try testing.expectEqual(@as(i3, 0), codePointWidth(0x0005)); // Cf - try testing.expectEqual(@as(i3, 0), codePointWidth(0x0007)); // \a BEL - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000A)); // \n LF - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000B)); // \v VT - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000C)); // \f FF - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000D)); // \r CR - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000E)); // SQ - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000F)); // SI - - try testing.expectEqual(@as(i3, 0), codePointWidth(0x070F)); // Cf - try testing.expectEqual(@as(i3, 1), codePointWidth(0x0603)); // Cf Arabic - - try testing.expectEqual(@as(i3, 1), codePointWidth(0x00AD)); // soft-hyphen - try testing.expectEqual(@as(i3, 2), codePointWidth(0x2E3A)); // two-em dash - try testing.expectEqual(@as(i3, 3), codePointWidth(0x2E3B)); // three-em dash - - try testing.expectEqual(@as(i3, 1), codePointWidth(0x00BD)); // ambiguous halfwidth - - try testing.expectEqual(@as(i3, 1), codePointWidth('é')); - try testing.expectEqual(@as(i3, 2), codePointWidth('😊')); - try testing.expectEqual(@as(i3, 2), codePointWidth('统')); + try testing.expectEqual(@as(i4, 0), codePointWidth(0x0000)); // null + try testing.expectEqual(@as(i4, -1), codePointWidth(0x8)); // \b + try testing.expectEqual(@as(i4, -1), codePointWidth(0x7f)); // DEL + try testing.expectEqual(@as(i4, 0), codePointWidth(0x0005)); // Cf + try testing.expectEqual(@as(i4, 0), codePointWidth(0x0007)); // \a BEL + try testing.expectEqual(@as(i4, 0), codePointWidth(0x000A)); // \n LF + try testing.expectEqual(@as(i4, 0), codePointWidth(0x000B)); // \v VT + try testing.expectEqual(@as(i4, 0), codePointWidth(0x000C)); // \f FF + try testing.expectEqual(@as(i4, 0), codePointWidth(0x000D)); // \r CR + try testing.expectEqual(@as(i4, 0), codePointWidth(0x000E)); // SQ + try testing.expectEqual(@as(i4, 0), codePointWidth(0x000F)); // SI + + try testing.expectEqual(@as(i4, 0), codePointWidth(0x070F)); // Cf + try testing.expectEqual(@as(i4, 1), codePointWidth(0x0603)); // Cf Arabic + + try testing.expectEqual(@as(i4, 1), codePointWidth(0x00AD)); // soft-hyphen + try testing.expectEqual(@as(i4, 2), codePointWidth(0x2E3A)); // two-em dash + try testing.expectEqual(@as(i4, 3), codePointWidth(0x2E3B)); // three-em dash + + try testing.expectEqual(@as(i4, 1), codePointWidth(0x00BD)); // ambiguous halfwidth + + try testing.expectEqual(@as(i4, 1), codePointWidth('é')); + try testing.expectEqual(@as(i4, 2), codePointWidth('😊')); + try testing.expectEqual(@as(i4, 2), codePointWidth('统')); } -- cgit v1.2.3