diff options
| author | Can Celik <ogulcancelik@gmail.com> | 2026-04-04 19:22:04 +0300 |
|---|---|---|
| committer | Can Celik <ogulcancelik@gmail.com> | 2026-04-05 01:23:06 +0300 |
| commit | 41dd733ca7123c9ac4bfc6c781697a660e95b052 (patch) | |
| tree | 3fc91815ca139ef2e0b281bba0abda4ef15175f4 /vendor/libghostty-vt/src/unicode | |
| parent | 55c138263e190a1e79d4b997bfb9397306c0ec1f (diff) | |
feat: finish ghostty pane backend integration
Diffstat (limited to 'vendor/libghostty-vt/src/unicode')
| -rw-r--r-- | vendor/libghostty-vt/src/unicode/grapheme.zig | 180 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/unicode/lut.zig | 188 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/unicode/main.zig | 10 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/unicode/props.zig | 53 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/unicode/props_table.zig | 18 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/unicode/props_uucode.zig | 85 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/unicode/symbols_table.zig | 17 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/unicode/symbols_uucode.zig | 68 |
8 files changed, 619 insertions, 0 deletions
diff --git a/vendor/libghostty-vt/src/unicode/grapheme.zig b/vendor/libghostty-vt/src/unicode/grapheme.zig new file mode 100644 index 00000000..d233dec7 --- /dev/null +++ b/vendor/libghostty-vt/src/unicode/grapheme.zig @@ -0,0 +1,180 @@ +const std = @import("std"); +const table = @import("props_table.zig").table; +const uucode = @import("uucode"); + +/// Determines if there is a grapheme break between two codepoints. This +/// must be called sequentially maintaining the state between calls. +/// +/// This function does NOT work with control characters. Control characters, +/// line feeds, and carriage returns are expected to be filtered out before +/// calling this function. This is because this function is tuned for +/// Ghostty. +pub fn graphemeBreak(cp1: u21, cp2: u21, state: *uucode.grapheme.BreakState) bool { + const value = Precompute.data[ + (Precompute.Key{ + .gb1 = table.get(cp1).grapheme_break, + .gb2 = table.get(cp2).grapheme_break, + .state = state.*, + }).index() + ]; + state.* = value.state; + return value.result; +} + +/// This is all the structures and data for the precomputed lookup table +/// for all possible permutations of state and grapheme break properties. +/// Precomputation requires 2^13 keys of 4 bit values so the whole table is +/// 8KB. +const Precompute = struct { + const Key = packed struct(u13) { + state: uucode.grapheme.BreakState, + gb1: uucode.x.types.GraphemeBreakNoControl, + gb2: uucode.x.types.GraphemeBreakNoControl, + + fn index(self: Key) usize { + return @intCast(@as(u13, @bitCast(self))); + } + }; + + const Value = packed struct(u4) { + result: bool, + state: uucode.grapheme.BreakState, + }; + + const data = precompute: { + var result: [std.math.maxInt(u13) + 1]Value = undefined; + + const max_state_int = blk: { + var max: usize = 0; + for (@typeInfo(uucode.grapheme.BreakState).@"enum".fields) |field| { + if (field.value > max) max = field.value; + } + break :blk max; + }; + + @setEvalBranchQuota(10_000); + const info = @typeInfo(uucode.x.types.GraphemeBreakNoControl).@"enum"; + for (0..max_state_int + 1) |state_int| { + for (info.fields) |field1| { + for (info.fields) |field2| { + var state: uucode.grapheme.BreakState = @enumFromInt(state_int); + + const key: Key = .{ + .gb1 = @field(uucode.x.types.GraphemeBreakNoControl, field1.name), + .gb2 = @field(uucode.x.types.GraphemeBreakNoControl, field2.name), + .state = state, + }; + const v = uucode.x.grapheme.computeGraphemeBreakNoControl( + key.gb1, + key.gb2, + &state, + ); + result[key.index()] = .{ .result = v, .state = state }; + } + } + } + + std.debug.assert(@sizeOf(@TypeOf(result)) == 8192); + break :precompute result; + }; +}; + +/// If you build this file as a binary, we will verify the grapheme break +/// implementation. This iterates over billions of codepoints so it is +/// SLOW. It's not meant to be run in CI, but it's useful for debugging. +/// TODO: this is hard to build with newer zig build, so +/// https://github.com/ghostty-org/ghostty/pull/7806 took the approach of +/// adding a `-Demit-unicode-test` option for `zig build`, but that +/// hasn't been done here. +pub fn main() !void { + // Set the min and max to control the test range. + const min = 0; + const max = uucode.config.max_code_point + 1; + + var state: uucode.grapheme.BreakState = .default; + var uu_state: uucode.grapheme.BreakState = .default; + for (min..max) |cp1| { + if (cp1 % 1000 == 0) std.log.warn("progress cp1={}", .{cp1}); + + if (cp1 == '\r' or cp1 == '\n' or + uucode.get(.grapheme_break, @intCast(cp1)) == .control) continue; + + for (min..max) |cp2| { + if (cp2 == '\r' or cp2 == '\n' or + uucode.get(.grapheme_break, @intCast(cp1)) == .control) continue; + + const gb = graphemeBreak(@intCast(cp1), @intCast(cp2), &state); + const uu_gb = uucode.grapheme.isBreak(@intCast(cp1), @intCast(cp2), &uu_state); + if (gb != uu_gb) { + std.log.warn("cp1={x} cp2={x} gb={} state={} uu_gb={} uu_state={}", .{ + cp1, + cp2, + gb, + state, + uu_gb, + uu_state, + }); + } + } + } +} + +pub const std_options = struct { + pub const log_level: std.log.Level = .info; +}; + +test "grapheme break: emoji modifier" { + const testing = std.testing; + + // Emoji and modifier + { + var state: uucode.grapheme.BreakState = .default; + try testing.expect(!graphemeBreak(0x261D, 0x1F3FF, &state)); + } + + // Non-emoji and emoji modifier + { + var state: uucode.grapheme.BreakState = .default; + try testing.expect(graphemeBreak(0x22, 0x1F3FF, &state)); + } +} + +test "long emoji zwj sequences" { + var state: uucode.grapheme.BreakState = .default; + // ๐ฉโ๐ฉโ๐งโ๐ฆ (family: woman, woman, girl, boy) + var it = uucode.utf8.Iterator.init("\u{1F469}\u{200D}\u{1F469}\u{200D}\u{1F467}\u{200D}\u{1F466}_"); + var cp1 = it.next() orelse unreachable; + var cp2 = it.next() orelse unreachable; + try std.testing.expect(cp1 == 0x1F469); // ๐ฉ + try std.testing.expect(!graphemeBreak(cp1, cp2, &state)); + + cp1 = cp2; + cp2 = it.next() orelse unreachable; + try std.testing.expect(cp1 == 0x200D); + try std.testing.expect(!graphemeBreak(cp1, cp2, &state)); + + cp1 = cp2; + cp2 = it.next() orelse unreachable; + try std.testing.expect(cp1 == 0x1F469); // ๐ฉ + try std.testing.expect(!graphemeBreak(cp1, cp2, &state)); + + cp1 = cp2; + cp2 = it.next() orelse unreachable; + try std.testing.expect(cp1 == 0x200D); + try std.testing.expect(!graphemeBreak(cp1, cp2, &state)); + + cp1 = cp2; + cp2 = it.next() orelse unreachable; + try std.testing.expect(cp1 == 0x1F467); // ๐ง + try std.testing.expect(!graphemeBreak(cp1, cp2, &state)); + + cp1 = cp2; + cp2 = it.next() orelse unreachable; + try std.testing.expect(cp1 == 0x200D); + try std.testing.expect(!graphemeBreak(cp1, cp2, &state)); + + cp1 = cp2; + cp2 = it.next() orelse unreachable; + try std.testing.expect(cp1 == 0x1F466); // ๐ฆ + try std.testing.expect(graphemeBreak(cp1, cp2, &state)); // break +} diff --git a/vendor/libghostty-vt/src/unicode/lut.zig b/vendor/libghostty-vt/src/unicode/lut.zig new file mode 100644 index 00000000..da90f1ee --- /dev/null +++ b/vendor/libghostty-vt/src/unicode/lut.zig @@ -0,0 +1,188 @@ +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; + +// This whole file is based on the algorithm described here: +// https://here-be-braces.com/fast-lookup-of-unicode-properties/ + +/// Creates a type that is able to generate a 3-level lookup table +/// from a Unicode codepoint to a mapping of type Elem. The lookup table +/// generally is expected to be codegen'd and then reloaded, although it +/// can in theory be generated at runtime. +/// +/// Context must have two functions: +/// - `get(Context, u21) Elem`: returns the mapping for a given codepoint +/// - `eql(Context, Elem, Elem) bool`: returns true if two mappings are equal +/// +pub fn Generator( + comptime Elem: type, + comptime Context: type, +) type { + return struct { + const Self = @This(); + + const block_size = 256; + const Block = [block_size]u16; + + /// Mapping of a block to its index in the stage2 array. + const BlockMap = std.HashMap( + Block, + u16, + struct { + pub fn hash(ctx: @This(), k: Block) u64 { + _ = ctx; + var hasher = std.hash.Wyhash.init(0); + std.hash.autoHashStrat(&hasher, k, .DeepRecursive); + return hasher.final(); + } + + pub fn eql(ctx: @This(), a: Block, b: Block) bool { + _ = ctx; + return std.mem.eql(u16, &a, &b); + } + }, + std.hash_map.default_max_load_percentage, + ); + + ctx: Context = undefined, + + /// Generate the lookup tables. The arrays in the return value + /// are owned by the caller and must be freed. + pub fn generate(self: *const Self, alloc: Allocator) !Tables(Elem) { + // Maps block => stage2 index + var blocks_map = BlockMap.init(alloc); + defer blocks_map.deinit(); + + // Our stages + var stage1: std.ArrayList(u16) = .empty; + var stage2: std.ArrayList(u16) = .empty; + var stage3: std.ArrayList(Elem) = .empty; + defer { + stage1.deinit(alloc); + stage2.deinit(alloc); + stage3.deinit(alloc); + } + + var block: Block = undefined; + var block_len: u16 = 0; + for (0..std.math.maxInt(u21) + 1) |cp| { + // Get our block value and find the matching result value + // in our list of possible values in stage3. This way, each + // possible mapping only gets one entry in stage3. + const elem = try self.ctx.get(@as(u21, @intCast(cp))); + const block_idx = block_idx: { + for (stage3.items, 0..) |item, i| { + if (self.ctx.eql(item, elem)) break :block_idx i; + } + + const idx = stage3.items.len; + try stage3.append(alloc, elem); + break :block_idx idx; + }; + + // The block stores the mapping to the stage3 index + block[block_len] = std.math.cast(u16, block_idx) orelse return error.BlockTooLarge; + block_len += 1; + + // If we still have space and we're not done with codepoints, + // we keep building up the block. Conversely: we finalize this + // block if we've filled it or are out of codepoints. + if (block_len < block_size and cp != std.math.maxInt(u21)) continue; + if (block_len < block_size) @memset(block[block_len..block_size], 0); + + // Look for the stage2 index for this block. If it doesn't exist + // we add it to stage2 and update the mapping. + const gop = try blocks_map.getOrPut(block); + if (!gop.found_existing) { + gop.value_ptr.* = std.math.cast( + u16, + stage2.items.len, + ) orelse return error.Stage2TooLarge; + for (block[0..block_len]) |entry| try stage2.append(alloc, entry); + } + + // Map stage1 => stage2 and reset our block + try stage1.append(alloc, gop.value_ptr.*); + block_len = 0; + } + + // All of our lengths must fit in a u16 for this to work + assert(stage1.items.len <= std.math.maxInt(u16)); + assert(stage2.items.len <= std.math.maxInt(u16)); + assert(stage3.items.len <= std.math.maxInt(u16)); + + const stage1_owned = try stage1.toOwnedSlice(alloc); + errdefer alloc.free(stage1_owned); + const stage2_owned = try stage2.toOwnedSlice(alloc); + errdefer alloc.free(stage2_owned); + const stage3_owned = try stage3.toOwnedSlice(alloc); + errdefer alloc.free(stage3_owned); + + return .{ + .stage1 = stage1_owned, + .stage2 = stage2_owned, + .stage3 = stage3_owned, + }; + } + }; +} + +/// Creates a type that given a 3-level lookup table, can be used to +/// look up a mapping for a given codepoint, encode it out to Zig, etc. +pub fn Tables(comptime Elem: type) type { + return struct { + const Self = @This(); + + stage1: []const u16, + stage2: []const u16, + stage3: []const Elem, + + /// Given a codepoint, returns the mapping for that codepoint. + pub inline fn get(self: *const Self, cp: u21) Elem { + const high = cp >> 8; + const low = cp & 0xFF; + return self.stage3[self.stage2[self.stage1[high] + low]]; + } + + /// Writes the lookup table as Zig to the given writer. The + /// written file exports three constants: stage1, stage2, and + /// stage3. These can be used to rebuild the lookup table in Zig. + pub fn writeZig(self: *const Self, writer: *std.Io.Writer) !void { + try writer.print( + \\//! This file is auto-generated. Do not edit. + \\ + \\pub fn Tables(comptime Elem: type) type {{ + \\ return struct {{ + \\pub const stage1: [{}]u16 = .{{ + , .{self.stage1.len}); + for (self.stage1) |entry| try writer.print("{},", .{entry}); + + try writer.print( + \\ + \\}}; + \\ + \\pub const stage2: [{}]u16 = .{{ + , .{self.stage2.len}); + for (self.stage2) |entry| try writer.print("{},", .{entry}); + try writer.writeAll("};"); + + try writer.print( + \\ + \\pub const stage3: [{}]Elem = .{{ + , .{self.stage3.len}); + for (self.stage3) |entry| { + if (@typeInfo(@TypeOf(entry)) == .@"struct" and + @hasDecl(@TypeOf(entry), "format")) + try writer.print("{f},", .{entry}) + else + try writer.print("{},", .{entry}); + } + try writer.writeAll( + \\}; + \\ }; + \\} + \\ + ); + } + }; +} diff --git a/vendor/libghostty-vt/src/unicode/main.zig b/vendor/libghostty-vt/src/unicode/main.zig new file mode 100644 index 00000000..11ecbd90 --- /dev/null +++ b/vendor/libghostty-vt/src/unicode/main.zig @@ -0,0 +1,10 @@ +pub const lut = @import("lut.zig"); + +const grapheme = @import("grapheme.zig"); +pub const table = @import("props_table.zig").table; +pub const Properties = @import("props.zig").Properties; +pub const graphemeBreak = grapheme.graphemeBreak; + +test { + @import("std").testing.refAllDecls(@This()); +} diff --git a/vendor/libghostty-vt/src/unicode/props.zig b/vendor/libghostty-vt/src/unicode/props.zig new file mode 100644 index 00000000..2653f0cc --- /dev/null +++ b/vendor/libghostty-vt/src/unicode/props.zig @@ -0,0 +1,53 @@ +//! Property set per codepoint that Ghostty cares about. +//! +//! Adding to this lets you find new properties but also potentially makes +//! our lookup tables less efficient. Any changes to this should run the +//! benchmarks in src/bench to verify that we haven't regressed. + +const std = @import("std"); +const uucode = @import("uucode"); + +pub const Properties = packed struct { + /// Codepoint width. We clamp to [0, 2] since Ghostty handles control + /// characters and we max out at 2 for wide characters (i.e. 3-em dash + /// becomes a 2-em dash). + width: u2 = 0, + + /// Whether the code point does not contribute to the width of a grapheme + /// cluster (not used for single code point cells). + width_zero_in_grapheme: bool = false, + + /// Grapheme break property. + grapheme_break: uucode.x.types.GraphemeBreakNoControl = .other, + + /// Emoji VS compatibility + emoji_vs_base: bool = false, + + // Needed for lut.Generator + pub fn eql(a: Properties, b: Properties) bool { + return a.width == b.width and + a.width_zero_in_grapheme == b.width_zero_in_grapheme and + a.grapheme_break == b.grapheme_break and + a.emoji_vs_base == b.emoji_vs_base; + } + + // Needed for lut.Generator + pub fn format( + self: Properties, + writer: *std.Io.Writer, + ) !void { + try writer.print( + \\.{{ + \\ .width= {}, + \\ .width_zero_in_grapheme= {}, + \\ .grapheme_break= .{s}, + \\ .emoji_vs_base= {}, + \\}} + , .{ + self.width, + self.width_zero_in_grapheme, + @tagName(self.grapheme_break), + self.emoji_vs_base, + }); + } +}; diff --git a/vendor/libghostty-vt/src/unicode/props_table.zig b/vendor/libghostty-vt/src/unicode/props_table.zig new file mode 100644 index 00000000..dac7e6f5 --- /dev/null +++ b/vendor/libghostty-vt/src/unicode/props_table.zig @@ -0,0 +1,18 @@ +const Properties = @import("props.zig").Properties; +const lut = @import("lut.zig"); + +/// The lookup tables for Ghostty. +pub const table = table: { + // This is only available after running a generator as part of the Ghostty + // build.zig process, but due to Zig's lazy analysis we can still reference + // it here. + // + // An example process is the `main` in `props_uucode.zig` + const generated = @import("unicode_tables").Tables(Properties); + const Tables = lut.Tables(Properties); + break :table Tables{ + .stage1 = &generated.stage1, + .stage2 = &generated.stage2, + .stage3 = &generated.stage3, + }; +}; diff --git a/vendor/libghostty-vt/src/unicode/props_uucode.zig b/vendor/libghostty-vt/src/unicode/props_uucode.zig new file mode 100644 index 00000000..527a757e --- /dev/null +++ b/vendor/libghostty-vt/src/unicode/props_uucode.zig @@ -0,0 +1,85 @@ +const props = @This(); +const std = @import("std"); +const assert = std.debug.assert; +const uucode = @import("uucode"); +const lut = @import("lut.zig"); +const Properties = @import("props.zig").Properties; + +pub fn get(cp: u21) Properties { + if (cp > uucode.config.max_code_point) return .{ + .width = 1, + .width_zero_in_grapheme = true, + .grapheme_break = .other, + .emoji_vs_base = false, + }; + + return .{ + .width = uucode.get(.width, cp), + .width_zero_in_grapheme = uucode.get(.wcwidth_zero_in_grapheme, cp), + .grapheme_break = uucode.get(.grapheme_break_no_control, cp), + .emoji_vs_base = uucode.get(.is_emoji_vs_base, cp), + }; +} + +/// Runnable binary to generate the lookup tables and output to stdout. +pub fn main() !void { + var arena_state = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena_state.deinit(); + const alloc = arena_state.allocator(); + + const gen: lut.Generator( + Properties, + struct { + pub fn get(ctx: @This(), cp: u21) !Properties { + _ = ctx; + return props.get(cp); + } + + pub fn eql(ctx: @This(), a: Properties, b: Properties) bool { + _ = ctx; + return a.eql(b); + } + }, + ) = .{}; + + const t = try gen.generate(alloc); + defer alloc.free(t.stage1); + defer alloc.free(t.stage2); + defer alloc.free(t.stage3); + + var buf: [4096]u8 = undefined; + var stdout = std.fs.File.stdout().writer(&buf); + try t.writeZig(&stdout.interface); + // Use flush instead of end because stdout is a pipe when captured by + // the build system, and pipes cannot be truncated (Windows returns + // INVALID_PARAMETER, Linux returns EINVAL). + try stdout.interface.flush(); + + // Uncomment when manually debugging to see our table sizes. + // std.log.warn("stage1={} stage2={} stage3={}", .{ + // t.stage1.len, + // t.stage2.len, + // t.stage3.len, + // }); +} + +test "unicode props: tables match uucode" { + if (std.valgrind.runningOnValgrind() > 0) return error.SkipZigTest; + + const testing = std.testing; + const table = @import("props_table.zig").table; + + const min = 0xFF + 1; // start outside ascii + const max = std.math.maxInt(u21) + 1; + for (min..max) |cp| { + const t = table.get(@intCast(cp)); + const uu = if (cp > uucode.config.max_code_point) + 1 + else + uucode.get(.width, @intCast(cp)); + if (t.width != uu) { + std.log.warn("mismatch cp=U+{x} t={} uu={}", .{ cp, t.width, uu }); + try testing.expect(false); + } + } +} diff --git a/vendor/libghostty-vt/src/unicode/symbols_table.zig b/vendor/libghostty-vt/src/unicode/symbols_table.zig new file mode 100644 index 00000000..034d3442 --- /dev/null +++ b/vendor/libghostty-vt/src/unicode/symbols_table.zig @@ -0,0 +1,17 @@ +const lut = @import("lut.zig"); + +/// The lookup tables for Ghostty. +pub const table = table: { + // This is only available after running a generator as part of the Ghostty + // build.zig process, but due to Zig's lazy analysis we can still reference + // it here. + // + // An example process is the `main` in `symbols_uucode.zig` + const generated = @import("symbols_tables").Tables(bool); + const Tables = lut.Tables(bool); + break :table Tables{ + .stage1 = &generated.stage1, + .stage2 = &generated.stage2, + .stage3 = &generated.stage3, + }; +}; diff --git a/vendor/libghostty-vt/src/unicode/symbols_uucode.zig b/vendor/libghostty-vt/src/unicode/symbols_uucode.zig new file mode 100644 index 00000000..794ca5ba --- /dev/null +++ b/vendor/libghostty-vt/src/unicode/symbols_uucode.zig @@ -0,0 +1,68 @@ +const std = @import("std"); +const uucode = @import("uucode"); +const lut = @import("lut.zig"); + +/// Runnable binary to generate the lookup tables and output to stdout. +pub fn main() !void { + var arena_state = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena_state.deinit(); + const alloc = arena_state.allocator(); + + const gen: lut.Generator( + bool, + struct { + pub fn get(ctx: @This(), cp: u21) !bool { + _ = ctx; + return if (cp > uucode.config.max_code_point) + false + else + uucode.get(.is_symbol, @intCast(cp)); + } + + pub fn eql(ctx: @This(), a: bool, b: bool) bool { + _ = ctx; + return a == b; + } + }, + ) = .{}; + + const t = try gen.generate(alloc); + defer alloc.free(t.stage1); + defer alloc.free(t.stage2); + defer alloc.free(t.stage3); + + var buf: [4096]u8 = undefined; + var stdout = std.fs.File.stdout().writer(&buf); + try t.writeZig(&stdout.interface); + // Use flush instead of end because stdout is a pipe when captured by + // the build system, and pipes cannot be truncated (Windows returns + // INVALID_PARAMETER, Linux returns EINVAL). + try stdout.interface.flush(); + + // Uncomment when manually debugging to see our table sizes. + // std.log.warn("stage1={} stage2={} stage3={}", .{ + // t.stage1.len, + // t.stage2.len, + // t.stage3.len, + // }); +} + +test "unicode symbols: tables match uucode" { + if (std.valgrind.runningOnValgrind() > 0) return error.SkipZigTest; + + const testing = std.testing; + const table = @import("symbols_table.zig").table; + + for (0..std.math.maxInt(u21)) |cp| { + const t = table.get(@intCast(cp)); + const uu = if (cp > uucode.config.max_code_point) + false + else + uucode.get(.is_symbol, @intCast(cp)); + + if (t != uu) { + std.log.warn("mismatch cp=U+{x} t={} uu={}", .{ cp, t, uu }); + try testing.expect(false); + } + } +} |
