10 files changed, 184 insertions, 106 deletions

diff --git a/build.zig b/build.zig
index 2c3ccdc..fa3660e 100644
--- a/build.zig
+++ b/build.zig
@@ -43,6 +43,15 @@ pub fn build(b: *std.Build) void {
     const run_canon_gen_exe = b.addRunArtifact(canon_gen_exe);
     const canon_gen_out = run_canon_gen_exe.addOutputFileArg("canon.bin.z");
 
+    const compat_gen_exe = b.addExecutable(.{
+        .name = "compat",
+        .root_source_file = .{ .path = "codegen/compat.zig" },
+        .target = b.host,
+        .optimize = .Debug,
+    });
+    const run_compat_gen_exe = b.addRunArtifact(compat_gen_exe);
+    const compat_gen_out = run_compat_gen_exe.addOutputFileArg("compat.bin.z");
+
     const ccc_gen_exe = b.addExecutable(.{
         .name = "ccc",
         .root_source_file = .{ .path = "codegen/ccc.zig" },
@@ -85,7 +94,7 @@ pub fn build(b: *std.Build) void {
 
     // Fixed pitch font display width
     const dw_data = b.createModule(.{
-        .root_source_file = .{ .path = "src/DisplayWidthData.zig" },
+        .root_source_file = .{ .path = "src/WidthData.zig" },
         .target = target,
         .optimize = optimize,
     });
@@ -104,26 +113,34 @@ pub fn build(b: *std.Build) void {
 
     // Normalization
     const ccc_data = b.createModule(.{
-        .root_source_file = .{ .path = "src/CombiningClassData.zig" },
+        .root_source_file = .{ .path = "src/CombiningData.zig" },
         .target = target,
         .optimize = optimize,
     });
     ccc_data.addAnonymousImport("ccc", .{ .root_source_file = ccc_gen_out });
 
     const canon_data = b.createModule(.{
-        .root_source_file = .{ .path = "src/Canonical.zig" },
+        .root_source_file = .{ .path = "src/CanonData.zig" },
         .target = target,
         .optimize = optimize,
     });
     canon_data.addAnonymousImport("canon", .{ .root_source_file = canon_gen_out });
 
+    const compat_data = b.createModule(.{
+        .root_source_file = .{ .path = "src/CompatData.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
+    compat_data.addAnonymousImport("compat", .{ .root_source_file = compat_gen_out });
+
     const norm_data = b.createModule(.{
         .root_source_file = .{ .path = "src/NormData.zig" },
         .target = target,
         .optimize = optimize,
     });
-    norm_data.addImport("CanonicalData", canon_data);
-    norm_data.addImport("CombiningClassData", ccc_data);
+    norm_data.addImport("CanonData", canon_data);
+    norm_data.addImport("CompatData", compat_data);
+    norm_data.addImport("CombiningData", ccc_data);
 
     const norm = b.addModule("Normalizer", .{
         .root_source_file = .{ .path = "src/Normalizer.zig" },
diff --git a/codegen/compat.zig b/codegen/compat.zig
new file mode 100644
index 0000000..916d4d0
--- /dev/null
+++ b/codegen/compat.zig
@@ -0,0 +1,65 @@
+const std = @import("std");
+const builtin = @import("builtin");
+
+pub fn main() !void {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    defer arena.deinit();
+    const allocator = arena.allocator();
+
+    // Process DerivedEastAsianWidth.txt
+    var in_file = try std.fs.cwd().openFile("data/unicode/UnicodeData.txt", .{});
+    defer in_file.close();
+    var in_buf = std.io.bufferedReader(in_file.reader());
+    const in_reader = in_buf.reader();
+
+    var args_iter = try std.process.argsWithAllocator(allocator);
+    defer args_iter.deinit();
+    _ = args_iter.skip();
+    const output_path = args_iter.next() orelse @panic("No output file arg!");
+
+    const compressor = std.compress.deflate.compressor;
+    var out_file = try std.fs.cwd().createFile(output_path, .{});
+    defer out_file.close();
+    var out_comp = try compressor(allocator, out_file.writer(), .{ .level = .best_compression });
+    defer out_comp.deinit();
+    const writer = out_comp.writer();
+
+    const endian = builtin.cpu.arch.endian();
+    var line_buf: [4096]u8 = undefined;
+
+    lines: while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
+        if (line.len == 0) continue;
+
+        var field_iter = std.mem.splitScalar(u8, line, ';');
+        var cps: [19]u24 = undefined;
+        var len: u8 = 1;
+
+        var i: usize = 0;
+        while (field_iter.next()) |field| : (i += 1) {
+            switch (i) {
+                0 => cps[0] = try std.fmt.parseInt(u24, field, 16),
+
+                5 => {
+                    // Not compatibility.
+                    if (field.len == 0 or field[0] != '<') continue :lines;
+                    var cp_iter = std.mem.tokenizeScalar(u8, field, ' ');
+                    _ = cp_iter.next(); // <compat type>
+
+                    while (cp_iter.next()) |cp_str| : (len += 1) {
+                        cps[len] = try std.fmt.parseInt(u24, cp_str, 16);
+                    }
+                },
+
+                2 => if (line[0] == '<') continue :lines,
+
+                else => {},
+            }
+        }
+
+        try writer.writeInt(u8, @intCast(len), endian);
+        for (cps[0..len]) |cp| try writer.writeInt(u24, cp, endian);
+    }
+
+    try writer.writeInt(u16, 0, endian);
+    try out_comp.flush();
+}
diff --git a/src/Canonical.zig b/src/CanonData.zig
index 81d3eec..81d3eec 100644
--- a/src/Canonical.zig
+++ b/src/CanonData.zig
diff --git a/src/CombiningClassData.zig b/src/CombiningData.zig
index 95c947d..95c947d 100644
--- a/src/CombiningClassData.zig
+++ b/src/CombiningData.zig
diff --git a/src/CompatData.zig b/src/CompatData.zig
new file mode 100644
index 0000000..a1f5de6
--- /dev/null
+++ b/src/CompatData.zig
@@ -0,0 +1,50 @@
+const std = @import("std");
+const builtin = @import("builtin");
+const compress = std.compress;
+const mem = std.mem;
+
+allocator: mem.Allocator,
+nfkd: [][]u21 = undefined,
+
+const Self = @This();
+
+pub fn init(allocator: mem.Allocator) !Self {
+    const decompressor = compress.deflate.decompressor;
+    const in_bytes = @embedFile("compat");
+    var in_fbs = std.io.fixedBufferStream(in_bytes);
+    var in_decomp = try decompressor(allocator, in_fbs.reader(), null);
+    defer in_decomp.deinit();
+    var reader = in_decomp.reader();
+
+    const endian = builtin.cpu.arch.endian();
+    var self = Self{
+        .allocator = allocator,
+        .nfkd = try allocator.alloc([]u21, 0x110000),
+    };
+
+    for (0..0x110000) |i| self.nfkd[i] = &.{};
+
+    while (true) {
+        const len: u8 = try reader.readInt(u8, endian);
+        if (len == 0) break;
+        const cp = try reader.readInt(u24, endian);
+        self.nfkd[cp] = try allocator.alloc(u21, len - 1);
+        for (0..len - 1) |i| {
+            self.nfkd[cp][i] = @intCast(try reader.readInt(u24, endian));
+        }
+    }
+
+    return self;
+}
+
+pub fn deinit(self: *Self) void {
+    for (self.nfkd) |slice| {
+        if (slice.len != 0) self.allocator.free(slice);
+    }
+    self.allocator.free(self.nfkd);
+}
+
+/// Returns compatibility decomposition for `cp`.
+pub inline fn toNfkd(self: Self, cp: u21) []u21 {
+    return self.nfkd[cp];
+}
diff --git a/src/NormData.zig b/src/NormData.zig
index c6fa8e8..83110f0 100644
--- a/src/NormData.zig
+++ b/src/NormData.zig
@@ -1,11 +1,13 @@
 const std = @import("std");
 const mem = std.mem;
 
-const CanonData = @import("CanonicalData");
-const CccData = @import("CombiningClassData");
+const CanonData = @import("CanonData");
+const CccData = @import("CombiningData");
+const CompatData = @import("CompatData");
 
 canon_data: CanonData,
 ccc_data: CccData,
+compat_data: CompatData,
 
 const Self = @This();
 
@@ -13,10 +15,12 @@ pub fn init(allocator: std.mem.Allocator) !Self {
     return Self{
         .canon_data = try CanonData.init(allocator),
         .ccc_data = try CccData.init(allocator),
+        .compat_data = try CompatData.init(allocator),
     };
 }
 
 pub fn deinit(self: *Self) void {
     self.canon_data.deinit();
     self.ccc_data.deinit();
+    self.compat_data.deinit();
 }
diff --git a/src/Normalizer.zig b/src/Normalizer.zig
index 2e2e6e4..1434043 100644
--- a/src/Normalizer.zig
+++ b/src/Normalizer.zig
@@ -12,57 +12,10 @@ const norm_props = @import("ziglyph").normalization_props;
 
 pub const NormData = @import("NormData");
 
-nfkd_map: std.AutoHashMap(u21, [18]u21),
 norm_data: *NormData,
 
 const Self = @This();
 
-pub fn init(allocator: std.mem.Allocator, norm_data: *NormData) !Self {
-    var self = Self{
-        .nfkd_map = std.AutoHashMap(u21, [18]u21).init(allocator),
-        .norm_data = norm_data,
-    };
-    errdefer self.deinit();
-
-    // Compatibility decompositions
-    const dekomp_file = @embedFile("autogen/compatibility_decompositions.txt.deflate");
-    var dekomp_stream = std.io.fixedBufferStream(dekomp_file);
-    var dekomp_decomp = try std.compress.deflate.decompressor(allocator, dekomp_stream.reader(), null);
-    defer dekomp_decomp.deinit();
-
-    var dekomp_buf = std.io.bufferedReader(dekomp_decomp.reader());
-    const dekomp_reader = dekomp_buf.reader();
-    var buf: [4096]u8 = undefined;
-
-    while (try dekomp_reader.readUntilDelimiterOrEof(&buf, '\n')) |line| {
-        if (line.len == 0) continue;
-        var fields = std.mem.split(u8, line, ";");
-        const cp_a = try std.fmt.parseInt(u21, fields.next().?, 16);
-        var cps = [_]u21{0} ** 18;
-        var i: usize = 0;
-
-        while (fields.next()) |cp| : (i += 1) {
-            cps[i] = try std.fmt.parseInt(u21, cp, 16);
-        }
-
-        try self.nfkd_map.put(cp_a, cps);
-    }
-
-    return self;
-}
-
-pub fn deinit(self: *Self) void {
-    self.nfkd_map.deinit();
-}
-
-test "init / deinit" {
-    const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
-}
-
 // Hangul processing utilities.
 fn isHangulPrecomposed(cp: u21) bool {
     if (hangul_map.syllableType(cp)) |kind| return kind == .LV or kind == .LVT;
@@ -140,10 +93,11 @@ pub fn mapping(self: Self, cp: u21, form: Form) Decomp {
         @memcpy(dc.cps[0..len], canon_dc[0..len]);
     }
 
-    if (self.nfkd_map.get(cp)) |array| {
+    const compat_dc = self.norm_data.compat_data.toNfkd(cp);
+    if (compat_dc.len != 0) {
         if (form != .nfd) {
             dc.form = .nfkd;
-            @memcpy(dc.cps[0..array.len], &array);
+            @memcpy(dc.cps[0..compat_dc.len], compat_dc);
         }
     }
 
@@ -210,10 +164,9 @@ pub fn decompose(self: Self, cp: u21, form: Form) Decomp {
 
 test "decompose" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var dc = n.decompose('é', .nfd);
     try std.testing.expect(dc.form == .nfd);
@@ -334,10 +287,9 @@ fn nfxd(self: Self, allocator: std.mem.Allocator, str: []const u8, form: Form) !
 
 test "nfd ASCII / no-alloc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfd(allocator, "Hello World!");
     defer result.deinit();
@@ -347,10 +299,9 @@ test "nfd ASCII / no-alloc" {
 
 test "nfd !ASCII / alloc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfd(allocator, "Héllo World! \u{3d3}");
     defer result.deinit();
@@ -360,10 +311,9 @@ test "nfd !ASCII / alloc" {
 
 test "nfkd ASCII / no-alloc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfkd(allocator, "Hello World!");
     defer result.deinit();
@@ -373,10 +323,9 @@ test "nfkd ASCII / no-alloc" {
 
 test "nfkd !ASCII / alloc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfkd(allocator, "Héllo World! \u{3d3}");
     defer result.deinit();
@@ -529,10 +478,9 @@ fn nfxc(self: Self, allocator: std.mem.Allocator, str: []const u8, form: Form) !
 
 test "nfc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfc(allocator, "Complex char: \u{3D2}\u{301}");
     defer result.deinit();
@@ -542,10 +490,9 @@ test "nfc" {
 
 test "nfkc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfkc(allocator, "Complex char: \u{03A5}\u{0301}");
     defer result.deinit();
@@ -603,10 +550,9 @@ pub fn eql(self: Self, allocator: std.mem.Allocator, a: []const u8, b: []const u
 
 test "eql" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     try std.testing.expect(try n.eql(allocator, "foé", "foe\u{0301}"));
     try std.testing.expect(try n.eql(allocator, "foϓ", "fo\u{03D2}\u{0301}"));
@@ -672,10 +618,9 @@ pub fn eqlCaseless(self: Self, allocator: std.mem.Allocator, a: []const u8, b: [
 
 test "eqlCaseless" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     try std.testing.expect(try n.eqlCaseless(allocator, "Foϓ", "fo\u{03D2}\u{0301}"));
     try std.testing.expect(try n.eqlCaseless(allocator, "FOÉ", "foe\u{0301}")); // foÉ == foé
@@ -709,10 +654,9 @@ pub fn isFcd(self: Self, str: []const u8) bool {
 
 test "isFcd" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     const is_nfc = "José \u{3D3}";
     try std.testing.expect(n.isFcd(is_nfc));
@@ -729,10 +673,9 @@ test "Unicode normalization tests" {
     defer arena.deinit();
     var allocator = arena.allocator();
 
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var file = try std.fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{});
     defer file.close();
diff --git a/src/DisplayWidthData.zig b/src/WidthData.zig
index 32f8658..32f8658 100644
--- a/src/DisplayWidthData.zig
+++ b/src/WidthData.zig
diff --git a/src/autogen/compatibility_decompositions.txt.deflate b/src/autogen/compatibility_decompositions.txt.deflate
deleted file mode 100644
index 0370b4c..0000000
--- a/src/autogen/compatibility_decompositions.txt.deflate
+++ /dev/null
diff --git a/src/main.zig b/src/main.zig
index 05c2ea4..2c2cf8c 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -16,9 +16,9 @@ const std = @import("std");
 // const ascii = @import("ascii");
 // const ascii = std.ascii;
 
-// const norm = @import("ziglyph").Normalizer;
+// const Normalizer = @import("ziglyph").Normalizer;
 const NormData = @import("Normalizer").NormData;
-const norm = @import("Normalizer");
+const Normalizer = @import("Normalizer");
 
 pub fn main() !void {
     var args_iter = std.process.args();
@@ -32,10 +32,9 @@ pub fn main() !void {
     const input = try std.fs.cwd().readFileAlloc(allocator, in_path, std.math.maxInt(u32));
     defer allocator.free(input);
 
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try norm.init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Normalizer{ .norm_data = &data };
     // var n = try norm.init(allocator);
     // defer n.deinit();