diff options
| author | Can Celik <ogulcancelik@gmail.com> | 2026-04-04 19:22:04 +0300 |
|---|---|---|
| committer | Can Celik <ogulcancelik@gmail.com> | 2026-04-05 01:23:06 +0300 |
| commit | 41dd733ca7123c9ac4bfc6c781697a660e95b052 (patch) | |
| tree | 3fc91815ca139ef2e0b281bba0abda4ef15175f4 /vendor/libghostty-vt/src/font/shaper | |
| parent | 55c138263e190a1e79d4b997bfb9397306c0ec1f (diff) | |
feat: finish ghostty pane backend integration
Diffstat (limited to 'vendor/libghostty-vt/src/font/shaper')
| -rw-r--r-- | vendor/libghostty-vt/src/font/shaper/Cache.zig | 106 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/font/shaper/coretext.zig | 2678 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/font/shaper/feature.zig | 354 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/font/shaper/harfbuzz.zig | 2172 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/font/shaper/noop.zig | 143 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/font/shaper/run.zig | 409 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/font/shaper/testdata/arabic.txt | 3 | ||||
| -rw-r--r-- | vendor/libghostty-vt/src/font/shaper/web_canvas.zig | 305 |
8 files changed, 6170 insertions, 0 deletions
diff --git a/vendor/libghostty-vt/src/font/shaper/Cache.zig b/vendor/libghostty-vt/src/font/shaper/Cache.zig new file mode 100644 index 00000000..2696985a --- /dev/null +++ b/vendor/libghostty-vt/src/font/shaper/Cache.zig @@ -0,0 +1,106 @@ +//! This structure caches the shaped cells for a given text run. +//! +//! At one point, shaping was the most expensive part of rendering text +//! (accounting for 96% of frame time on my machine). To speed it up, this +//! was introduced so that shaping results can be cached depending on the +//! run. +//! +//! The cache key is the text run. The text run builds its own hash value +//! based on the font, style, codepoint, etc. This just utilizes the hash that +//! the text run provides. +pub const Cache = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const font = @import("../main.zig"); +const CacheTable = @import("../../datastruct/main.zig").CacheTable; + +const log = std.log.scoped(.font_shaper_cache); + +/// Context for cache table. +const CellCacheTableContext = struct { + pub fn hash(self: *const CellCacheTableContext, key: u64) u64 { + _ = self; + return key; + } + pub fn eql(self: *const CellCacheTableContext, a: u64, b: u64) bool { + _ = self; + return a == b; + } +}; + +/// Cache table for run hash -> shaped cells. +const CellCacheTable = CacheTable( + u64, + []font.shape.Cell, + CellCacheTableContext, + + // Capacity is slightly arbitrary. These numbers are guesses. + // + // I'd expect then an average of 256 frequently cached runs is a + // safe guess most terminal screens. + 256, + // 8 items per bucket to give decent resiliency to important runs. + 8, +); + +/// The cache table of shaped cells. +map: CellCacheTable, + +pub fn init() Cache { + return .{ .map = .{ .context = .{} } }; +} + +pub fn deinit(self: *Cache, alloc: Allocator) void { + self.clear(alloc); +} + +/// Get the shaped cells for the given text run, +/// or null if they are not in the cache. +pub fn get(self: *Cache, run: font.shape.TextRun) ?[]const font.shape.Cell { + return self.map.get(run.hash); +} + +/// Insert the shaped cells for the given text run into the cache. +/// +/// The cells will be duplicated. +pub fn put( + self: *Cache, + alloc: Allocator, + run: font.shape.TextRun, + cells: []const font.shape.Cell, +) Allocator.Error!void { + const copy = try alloc.dupe(font.shape.Cell, cells); + const evicted = self.map.put(run.hash, copy); + if (evicted) |kv| { + alloc.free(kv.value); + } +} + +fn clear(self: *Cache, alloc: Allocator) void { + for (self.map.buckets, self.map.lengths) |b, l| { + for (b[0..l]) |kv| { + alloc.free(kv.value); + } + } + self.map.clear(); +} + +test Cache { + const testing = std.testing; + const alloc = testing.allocator; + + var c = Cache.init(); + defer c.deinit(alloc); + + var run: font.shape.TextRun = undefined; + run.hash = 1; + try testing.expect(c.get(run) == null); + try c.put(alloc, run, &.{ + .{ .x = 0, .glyph_index = 0 }, + .{ .x = 1, .glyph_index = 1 }, + }); + + const actual = c.get(run).?; + try testing.expect(actual.len == 2); +} diff --git a/vendor/libghostty-vt/src/font/shaper/coretext.zig b/vendor/libghostty-vt/src/font/shaper/coretext.zig new file mode 100644 index 00000000..ff7c6d9d --- /dev/null +++ b/vendor/libghostty-vt/src/font/shaper/coretext.zig @@ -0,0 +1,2678 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const assert = @import("../../quirks.zig").inlineAssert; +const Allocator = std.mem.Allocator; +const macos = @import("macos"); +const font = @import("../main.zig"); +const os = @import("../../os/main.zig"); +const terminal = @import("../../terminal/main.zig"); +const unicode = @import("../../unicode/main.zig"); +const Feature = font.shape.Feature; +const FeatureList = font.shape.FeatureList; +const default_features = font.shape.default_features; +const Face = font.Face; +const Collection = font.Collection; +const DeferredFace = font.DeferredFace; +const Group = font.Group; +const GroupCache = font.GroupCache; +const Library = font.Library; +const SharedGrid = font.SharedGrid; +const Style = font.Style; +const Presentation = font.Presentation; +const CFReleaseThread = os.CFReleaseThread; + +const log = std.log.scoped(.font_shaper); + +/// Shaper that uses CoreText. +/// +/// CoreText shaping differs in subtle ways from HarfBuzz so it may result +/// in inconsistent rendering across platforms. But it also fixes many +/// issues (some macOS specific): +/// +/// - Theta hat offset is incorrect in HarfBuzz but correct by default +/// on macOS applications using CoreText. (See: +/// https://github.com/harfbuzz/harfbuzz/discussions/4525) +/// +/// - Hyphens (U+2010) can be synthesized by CoreText but not by HarfBuzz. +/// See: https://github.com/mitchellh/ghostty/issues/1643 +/// +pub const Shaper = struct { + /// The allocated used for the feature list, font cache, and cell buf. + alloc: Allocator, + + /// The string used for shaping the current run. + run_state: RunState, + + /// CoreFoundation Dictionary which represents our font feature settings. + features: *macos.foundation.Dictionary, + /// A version of the features dictionary with the default features excluded. + features_no_default: *macos.foundation.Dictionary, + + /// The shared memory used for shaping results. + cell_buf: CellBuf, + + /// Cached attributes dict for creating CTTypesetter objects. + /// The values in this never change so we can avoid overhead + /// by just creating it once and saving it for reuse. + typesetter_attr_dict: *macos.foundation.Dictionary, + + /// List where we cache fonts, so we don't have to remake them for + /// every single shaping operation. + /// + /// Fonts are cached as attribute dictionaries to be applied directly to + /// attributed strings. + cached_fonts: std.ArrayListUnmanaged(?*macos.foundation.Dictionary), + + /// The grid that our cached fonts correspond to. + /// If the grid changes then we need to reset our cache. + cached_font_grid: usize, + + /// The list of CoreFoundation objects to release on the dedicated + /// release thread. This is built up over the course of shaping and + /// sent to the release thread when endFrame is called. + cf_release_pool: std.ArrayListUnmanaged(*anyopaque), + + /// Dedicated thread for releasing CoreFoundation objects. Some objects, + /// such as those produced by CoreText, have excessively slow release + /// callback logic. + cf_release_thread: *CFReleaseThread, + cf_release_thr: std.Thread, + + const CellBuf = std.ArrayListUnmanaged(font.shape.Cell); + const CodepointList = std.ArrayListUnmanaged(Codepoint); + const Codepoint = struct { + codepoint: u32, + cluster: u32, + }; + + const RunState = struct { + codepoints: CodepointList, + unichars: std.ArrayListUnmanaged(u16), + + fn init() RunState { + return .{ .codepoints = .{}, .unichars = .{} }; + } + + fn deinit(self: *RunState, alloc: Allocator) void { + self.codepoints.deinit(alloc); + self.unichars.deinit(alloc); + } + + fn reset(self: *RunState) void { + self.codepoints.clearRetainingCapacity(); + self.unichars.clearRetainingCapacity(); + } + }; + + const Offset = struct { + cluster: u32 = 0, + x: f64 = 0, + }; + + /// Create a CoreFoundation Dictionary suitable for + /// settings the font features of a CoreText font. + fn makeFeaturesDict(feats: []const Feature) !*macos.foundation.Dictionary { + const list = try macos.foundation.MutableArray.create(); + // The list will be retained by the dict once we add it to it. + defer list.release(); + + for (feats) |feat| { + const value_num: c_int = @intCast(feat.value); + + // Keys can only be ASCII. + var key = try macos.foundation.String.createWithBytes(&feat.tag, .ascii, false); + defer key.release(); + var value = try macos.foundation.Number.create(.int, &value_num); + defer value.release(); + + const dict = try macos.foundation.Dictionary.create( + &[_]?*const anyopaque{ + macos.text.c.kCTFontOpenTypeFeatureTag, + macos.text.c.kCTFontOpenTypeFeatureValue, + }, + &[_]?*const anyopaque{ key, value }, + ); + defer dict.release(); + + list.appendValue(macos.foundation.Dictionary, dict); + } + + var dict = try macos.foundation.Dictionary.create( + &[_]?*const anyopaque{macos.text.c.kCTFontFeatureSettingsAttribute}, + &[_]?*const anyopaque{list}, + ); + errdefer dict.release(); + + return dict; + } + + /// The cell_buf argument is the buffer to use for storing shaped results. + /// This should be at least the number of columns in the terminal. + pub fn init(alloc: Allocator, opts: font.shape.Options) !Shaper { + var feature_list: FeatureList = .{}; + defer feature_list.deinit(alloc); + for (opts.features) |feature_str| { + try feature_list.appendFromString(alloc, feature_str); + } + + // We need to construct two attrs dictionaries for font features; + // one without the default features included, and one with them. + const feats = feature_list.features.items; + const feats_df = try alloc.alloc(Feature, feats.len + default_features.len); + defer alloc.free(feats_df); + + @memcpy(feats_df[0..default_features.len], &default_features); + @memcpy(feats_df[default_features.len..], feats); + + const features = try makeFeaturesDict(feats_df); + errdefer features.release(); + const features_no_default = try makeFeaturesDict(feats); + errdefer features_no_default.release(); + + var run_state = RunState.init(); + errdefer run_state.deinit(alloc); + + // For now we only support LTR text. If we shape RTL text then + // rendering will be very wrong so we need to explicitly force + // LTR no matter what. + // + // See: https://github.com/mitchellh/ghostty/issues/1737 + // See: https://github.com/mitchellh/ghostty/issues/1442 + // + // We used to do this by setting the writing direction attribute + // on the attributed string we used, but it seems like that will + // still allow some weird results, for example a single space at + // the end of a line composed of RTL characters will be cause it + // to output a run containing just that space, BEFORE it outputs + // the rest of the line as a separate run, very weirdly with the + // "right to left" flag set in the single space run's run status... + // + // So instead what we do is use a CTTypesetter to create our line, + // using the kCTTypesetterOptionForcedEmbeddingLevel attribute to + // force CoreText not to try doing any sort of BiDi, instead just + // treat all text as embedding level 0 (left to right). + const typesetter_attr_dict = dict: { + const num = try macos.foundation.Number.create(.int, &0); + defer num.release(); + break :dict try macos.foundation.Dictionary.create( + &.{macos.c.kCTTypesetterOptionForcedEmbeddingLevel}, + &.{num}, + ); + }; + errdefer typesetter_attr_dict.release(); + + // Create the CF release thread. + var cf_release_thread = try alloc.create(CFReleaseThread); + errdefer alloc.destroy(cf_release_thread); + cf_release_thread.* = try .init(alloc); + errdefer cf_release_thread.deinit(); + + // Start the CF release thread. + var cf_release_thr = try std.Thread.spawn( + .{}, + CFReleaseThread.threadMain, + .{cf_release_thread}, + ); + cf_release_thr.setName("cf_release") catch {}; + + return .{ + .alloc = alloc, + .cell_buf = .{}, + .run_state = run_state, + .features = features, + .features_no_default = features_no_default, + .typesetter_attr_dict = typesetter_attr_dict, + .cached_fonts = .{}, + .cached_font_grid = 0, + .cf_release_pool = .{}, + .cf_release_thread = cf_release_thread, + .cf_release_thr = cf_release_thr, + }; + } + + pub fn deinit(self: *Shaper) void { + self.cell_buf.deinit(self.alloc); + self.run_state.deinit(self.alloc); + self.features.release(); + self.features_no_default.release(); + self.typesetter_attr_dict.release(); + + { + for (self.cached_fonts.items) |ft| { + if (ft) |f| f.release(); + } + self.cached_fonts.deinit(self.alloc); + } + + if (self.cf_release_pool.items.len > 0) { + for (self.cf_release_pool.items) |ref| macos.foundation.CFRelease(ref); + + // For tests this logic is normal because we don't want to + // wait for a release thread. But in production this is a bug + // and we should warn. + if (comptime !builtin.is_test) log.warn( + "BUG: CFRelease pool was not empty, releasing remaining objects", + .{}, + ); + } + self.cf_release_pool.deinit(self.alloc); + + // Stop the CF release thread + { + self.cf_release_thread.stop.notify() catch |err| + log.err("error notifying cf release thread to stop, may stall err={}", .{err}); + self.cf_release_thr.join(); + } + self.cf_release_thread.deinit(); + self.alloc.destroy(self.cf_release_thread); + } + + pub fn endFrame(self: *Shaper) void { + if (self.cf_release_pool.items.len == 0) return; + + // Get all the items in the pool as an owned slice so we can + // send it to the dedicated release thread. + const items = self.cf_release_pool.toOwnedSlice(self.alloc) catch |err| { + log.warn("error converting release pool to owned slice, slow release err={}", .{err}); + for (self.cf_release_pool.items) |ref| macos.foundation.CFRelease(ref); + self.cf_release_pool.clearRetainingCapacity(); + return; + }; + + // Send the items. If the send succeeds then we wake up the + // thread to process the items. If the send fails then do a manual + // cleanup. + if (self.cf_release_thread.mailbox.push(.{ .release = .{ + .refs = items, + .alloc = self.alloc, + } }, .{ .forever = {} }) != 0) { + self.cf_release_thread.wakeup.notify() catch |err| { + log.warn( + "error notifying cf release thread to wake up, may stall err={}", + .{err}, + ); + }; + return; + } + + for (items) |ref| macos.foundation.CFRelease(ref); + self.alloc.free(items); + } + + pub fn runIterator( + self: *Shaper, + opts: font.shape.RunOptions, + ) font.shape.RunIterator { + return .{ + .hooks = .{ .shaper = self }, + .opts = opts, + }; + } + + /// Note that this will accumulate garbage in the release pool. The + /// caller must ensure you're properly calling endFrame to release + /// all the objects. + pub fn shape( + self: *Shaper, + run: font.shape.TextRun, + ) ![]const font.shape.Cell { + const state = &self.run_state; + + // { + // log.debug("shape -----------------------------------", .{}); + // for (state.codepoints.items) |entry| { + // log.debug("cp={X} cluster={}", .{ entry.codepoint, entry.cluster }); + // } + // log.debug("shape end -------------------------------", .{}); + // } + + // Special fonts aren't shaped and their codepoint == glyph so we + // can just return the codepoints as-is. + if (run.font_index.special() != null) { + self.cell_buf.clearRetainingCapacity(); + try self.cell_buf.ensureTotalCapacity(self.alloc, state.codepoints.items.len); + for (state.codepoints.items) |entry| { + // We use null codepoints to pad out our list so indices match + // the UTF-16 string we constructed for CoreText. We don't want + // to emit these if this is a special font, since they're not + // part of the original run. + if (entry.codepoint == 0) continue; + + self.cell_buf.appendAssumeCapacity(.{ + .x = @intCast(entry.cluster), + .glyph_index = @intCast(entry.codepoint), + }); + } + + return self.cell_buf.items; + } + + // Create an arena for any Zig-based allocations we do + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const alloc = arena.allocator(); + + const attr_dict: *macos.foundation.Dictionary = try self.getFont( + run.grid, + run.font_index, + ); + + // Make room for the attributed string, CTTypesetter, and CTLine. + try self.cf_release_pool.ensureUnusedCapacity(self.alloc, 4); + + const str = macos.foundation.String.createWithCharactersNoCopy(state.unichars.items); + self.cf_release_pool.appendAssumeCapacity(str); + + // Create an attributed string from our string + const attr_str = try macos.foundation.AttributedString.create( + str, + attr_dict, + ); + self.cf_release_pool.appendAssumeCapacity(attr_str); + + // Create a typesetter from the attributed string and the cached + // attr dict. (See comment in init for more info on the attr dict.) + const typesetter = + try macos.text.Typesetter.createWithAttributedStringAndOptions( + attr_str, + self.typesetter_attr_dict, + ); + self.cf_release_pool.appendAssumeCapacity(typesetter); + + // Create a line from the typesetter + const line = typesetter.createLine(.{ .location = 0, .length = 0 }); + self.cf_release_pool.appendAssumeCapacity(line); + + // This keeps track of the current x offset (sum of advance.width) and + // the furthest cluster we've seen so far (max). + var run_offset: Offset = .{}; + + // This keeps track of the cell starting x and cluster. + var cell_offset: Offset = .{}; + + // For debugging positions, turn this on: + //var run_offset_y: f64 = 0.0; + //var cell_offset_y: f64 = 0.0; + + // Clear our cell buf and make sure we have enough room for the whole + // line of glyphs, so that we can just assume capacity when appending + // instead of maybe allocating. + self.cell_buf.clearRetainingCapacity(); + try self.cell_buf.ensureTotalCapacity(self.alloc, line.getGlyphCount()); + + // CoreText, despite our insistence with an enforced embedding level, + // may sometimes output runs that are non-monotonic. In order to fix + // this, we check the run status for each run and if any aren't ltr + // we set this to true, which indicates that we must sort our buffer. + var non_ltr: bool = false; + + // CoreText may generate multiple runs even though our input to + // CoreText is already split into runs by our own run iterator. + // The runs as far as I can tell are always sequential to each + // other so we can iterate over them and just append to our + // cell buffer. + const runs = line.getGlyphRuns(); + for (0..runs.getCount()) |run_i| { + const ctrun = runs.getValueAtIndex(macos.text.Run, run_i); + + const status = ctrun.getStatus(); + if (status.non_monotonic or status.right_to_left) non_ltr = true; + + // Get our glyphs and positions + const glyphs = ctrun.getGlyphsPtr() orelse try ctrun.getGlyphs(alloc); + const advances = ctrun.getAdvancesPtr() orelse try ctrun.getAdvances(alloc); + const positions = ctrun.getPositionsPtr() orelse try ctrun.getPositions(alloc); + const indices = ctrun.getStringIndicesPtr() orelse try ctrun.getStringIndices(alloc); + assert(glyphs.len == advances.len); + assert(glyphs.len == positions.len); + assert(glyphs.len == indices.len); + + for ( + glyphs, + advances, + positions, + indices, + ) |glyph, advance, position, index| { + // Our cluster is also our cell X position. If the cluster changes + // then we need to reset our current cell offsets. + const cluster = state.codepoints.items[index].cluster; + if (cell_offset.cluster != cluster) { + // We previously asserted that the new cluster is greater + // than cell_offset.cluster, but this isn't always true. + // See e.g. the "shape Chakma vowel sign with ligature + // (vowel sign renders first)" test. + + const is_after_glyph_from_current_or_next_clusters = + cluster <= run_offset.cluster; + + const is_first_codepoint_in_cluster = blk: { + var i = index; + while (i > 0) { + i -= 1; + const codepoint = state.codepoints.items[i]; + + // Skip surrogate pair padding + if (codepoint.codepoint == 0) continue; + break :blk codepoint.cluster != cluster; + } else break :blk true; + }; + + // We need to reset the `cell_offset` at the start of a new + // cluster, but we do that conditionally if the codepoint + // `is_first_codepoint_in_cluster` and the cluster is not + // `is_after_glyph_from_current_or_next_clusters`, which is + // a heuristic to detect ligatures and avoid positioning + // glyphs that mark ligatures incorrectly. The idea is that + // if the first codepoint in a cluster doesn't appear in + // the stream, it's very likely that it combined with + // codepoints from a previous cluster into a ligature. + // Then, the subsequent codepoints are very likely marking + // glyphs that are placed relative to that ligature, so if + // we were to reset the `cell_offset` to align it with the + // grid, the positions would be off. The + // `!is_after_glyph_from_current_or_next_clusters` check is + // needed in case these marking glyphs come from a later + // cluster but are rendered first (see the Chakma and + // Bengali tests). In that case when we get to the + // codepoint that `is_first_codepoint_in_cluster`, but in a + // cluster that + // `is_after_glyph_from_current_or_next_clusters`, we don't + // want to reset to the grid and cause the positions to be + // off. (Note that we could go back and align the cells to + // the grid starting from the one from the cluster that + // rendered out of order, but that is more complicated so + // we don't do that for now. Also, it's TBD if there are + // exceptions to this heuristic for detecting ligatures, + // but using the logging below seems to show it works + // well.) + if (is_first_codepoint_in_cluster and + !is_after_glyph_from_current_or_next_clusters) + { + cell_offset = .{ + .cluster = cluster, + .x = run_offset.x, + }; + + // For debugging positions, turn this on: + //cell_offset_y = run_offset_y; + } + } + + // For debugging positions, turn this on: + //try self.debugPositions(alloc, run_offset, run_offset_y, cell_offset, cell_offset_y, position, index); + + const x_offset = position.x - cell_offset.x; + + self.cell_buf.appendAssumeCapacity(.{ + .x = @intCast(cell_offset.cluster), + .x_offset = @intFromFloat(@round(x_offset)), + .y_offset = @intFromFloat(@round(position.y)), + .glyph_index = glyph, + }); + + // Add our advances to keep track of our run offsets. + // Advances apply to the NEXT cell. + run_offset.x += advance.width; + run_offset.cluster = @max(run_offset.cluster, cluster); + + // For debugging positions, turn this on: + //run_offset_y += advance.height; + } + } + + // If our buffer contains some non-ltr sections we need to sort it :/ + if (non_ltr) { + // This is EXCEPTIONALLY rare. Only happens for languages with + // complex shaping which we don't even really support properly + // right now, so are very unlikely to be used heavily by users + // of Ghostty. + @branchHint(.cold); + std.mem.sort( + font.shape.Cell, + self.cell_buf.items, + {}, + struct { + fn lt(_: void, a: font.shape.Cell, b: font.shape.Cell) bool { + return a.x < b.x; + } + }.lt, + ); + } + + return self.cell_buf.items; + } + + /// Get an attr dict for a font from a specific index. + /// These items are cached, do not retain or release them. + fn getFont( + self: *Shaper, + grid: *font.SharedGrid, + index: font.Collection.Index, + ) !*macos.foundation.Dictionary { + // If this grid doesn't match the one we've cached fonts for, + // then we reset the cache list since it's no longer valid. + // We use an intFromPtr rather than direct pointer comparison + // because we don't want anyone to inadvertently use the pointer. + const grid_id: usize = @intFromPtr(grid); + if (grid_id != self.cached_font_grid) { + if (self.cached_font_grid > 0) { + // Put all the currently cached fonts in to + // the release pool before clearing the list. + try self.cf_release_pool.ensureUnusedCapacity( + self.alloc, + self.cached_fonts.items.len, + ); + for (self.cached_fonts.items) |ft| { + if (ft) |f| { + self.cf_release_pool.appendAssumeCapacity(f); + } + } + self.cached_fonts.clearRetainingCapacity(); + } + + self.cached_font_grid = grid_id; + } + + const index_int = index.int(); + + // The cached fonts are indexed directly by the font index, since + // this number is usually low. Therefore, we set any index we haven't + // seen to null. + if (self.cached_fonts.items.len <= index_int) { + try self.cached_fonts.ensureTotalCapacity(self.alloc, index_int + 1); + while (self.cached_fonts.items.len <= index_int) { + self.cached_fonts.appendAssumeCapacity(null); + } + } + + // If we have it, return the cached attr dict. + if (self.cached_fonts.items[index_int]) |cached| return cached; + + // Font descriptor, font + try self.cf_release_pool.ensureUnusedCapacity(self.alloc, 2); + + const run_font = font: { + // The CoreText shaper relies on CoreText and CoreText claims + // that CTFonts are threadsafe. See: + // https://developer.apple.com/documentation/coretext/ + // + // Quote: + // All individual functions in Core Text are thread-safe. Font + // objects (CTFont, CTFontDescriptor, and associated objects) can + // be used simultaneously by multiple operations, work queues, or + // threads. However, the layout objects (CTTypesetter, + // CTFramesetter, CTRun, CTLine, CTFrame, and associated objects) + // should be used in a single operation, work queue, or thread. + // + // Because of this, we only acquire the read lock to grab the + // face and set it up, then release it. + grid.lock.lockShared(); + defer grid.lock.unlockShared(); + + const face = try grid.resolver.collection.getFace(index); + const original = face.font; + + const attrs = if (face.quirks_disable_default_font_features) + self.features_no_default + else + self.features; + + const desc = try macos.text.FontDescriptor.createWithAttributes(attrs); + self.cf_release_pool.appendAssumeCapacity(desc); + + const copied = try original.copyWithAttributes(0, null, desc); + errdefer copied.release(); + + break :font copied; + }; + self.cf_release_pool.appendAssumeCapacity(run_font); + + // Get our font and use that get the attributes to set for the + // attributed string so the whole string uses the same font. + const attr_dict = dict: { + break :dict try macos.foundation.Dictionary.create( + &.{macos.text.StringAttribute.font.key()}, + &.{run_font}, + ); + }; + + self.cached_fonts.items[index_int] = attr_dict; + return attr_dict; + } + + /// The hooks for RunIterator. + pub const RunIteratorHook = struct { + shaper: *Shaper, + + pub fn prepare(self: *RunIteratorHook) void { + self.shaper.run_state.reset(); + // log.warn("----------- run reset -------------", .{}); + } + + pub fn addCodepoint(self: RunIteratorHook, cp: u32, cluster: u32) !void { + const state = &self.shaper.run_state; + + // Build our UTF-16 string for CoreText + try state.unichars.ensureUnusedCapacity(self.shaper.alloc, 2); + + state.unichars.appendNTimesAssumeCapacity(0, 2); + + const pair = macos.foundation.stringGetSurrogatePairForLongCharacter( + cp, + state.unichars.items[state.unichars.items.len - 2 ..][0..2], + ); + if (!pair) { + state.unichars.items.len -= 1; + } + + // Build our reverse lookup table for codepoints to clusters + try state.codepoints.append(self.shaper.alloc, .{ + .codepoint = cp, + .cluster = cluster, + }); + // log.warn("run cp={X}", .{cp}); + + // If the UTF-16 codepoint is a pair then we need to insert + // a dummy entry so that the CTRunGetStringIndices() function + // maps correctly. + if (pair) try state.codepoints.append(self.shaper.alloc, .{ + .codepoint = 0, + .cluster = cluster, + }); + } + + pub fn finalize(self: RunIteratorHook) void { + _ = self; + } + }; + + fn debugPositions( + self: *Shaper, + alloc: Allocator, + run_offset: Offset, + run_offset_y: f64, + cell_offset: Offset, + cell_offset_y: f64, + position: macos.graphics.Point, + index: usize, + ) !void { + const state = &self.run_state; + const x_offset = position.x - cell_offset.x; + const advance_x_offset = run_offset.x - cell_offset.x; + const advance_y_offset = run_offset_y - cell_offset_y; + const x_offset_diff = x_offset - advance_x_offset; + const y_offset_diff = position.y - advance_y_offset; + const positions_differ = @abs(x_offset_diff) > 0.0001 or @abs(y_offset_diff) > 0.0001; + const old_offset_y = position.y - cell_offset_y; + const position_y_differs = @abs(cell_offset_y) > 0.0001; + const cluster = state.codepoints.items[index].cluster; + const cluster_differs = cluster != cell_offset.cluster; + + // To debug every loop, flip this to true: + const extra_debugging = false; + + const is_previous_codepoint_prepend = if (cluster_differs or + extra_debugging) + blk: { + var i = index; + while (i > 0) { + i -= 1; + const codepoint = state.codepoints.items[i]; + + // Skip surrogate pair padding + if (codepoint.codepoint == 0) continue; + + break :blk unicode.table.get(@intCast(codepoint.codepoint)).grapheme_boundary_class == .prepend; + } + break :blk false; + } else false; + + const formatted_cps = if (positions_differ or + position_y_differs or + cluster_differs or + extra_debugging) + blk: { + var allocating = std.Io.Writer.Allocating.init(alloc); + const writer = &allocating.writer; + const codepoints = state.codepoints.items; + var last_cluster: ?u32 = null; + for (codepoints, 0..) |cp, i| { + if ((@as(i32, @intCast(cp.cluster)) >= @as(i32, @intCast(cell_offset.cluster)) - 1 and + cp.cluster <= cluster + 1) and + cp.codepoint != 0 // Skip surrogate pair padding + ) { + if (last_cluster) |last| { + if (cp.cluster != last) { + try writer.writeAll(" "); + } + } + if (i == index) { + try writer.writeAll("▸"); + } + // Using Python syntax for easier debugging + if (cp.codepoint > 0xFFFF) { + try writer.print("\\U{x:0>8}", .{cp.codepoint}); + } else { + try writer.print("\\u{x:0>4}", .{cp.codepoint}); + } + last_cluster = cp.cluster; + } + } + try writer.writeAll(" → "); + for (codepoints) |cp| { + if ((@as(i32, @intCast(cp.cluster)) >= @as(i32, @intCast(cell_offset.cluster)) - 1 and + cp.cluster <= cluster + 1) and + cp.codepoint != 0 // Skip surrogate pair padding + ) { + try writer.print("{u}", .{@as(u21, @intCast(cp.codepoint))}); + } + } + break :blk try allocating.toOwnedSlice(); + } else ""; + + if (extra_debugging) { + log.warn("extra debugging of positions index={d} cell_offset.cluster={d} cluster={d} run_offset.cluster={d} diff={d} pos=({d:.2},{d:.2}) run_offset=({d:.2},{d:.2}) cell_offset=({d:.2},{d:.2}) is_prev_prepend={} cps = {s}", .{ + index, + cell_offset.cluster, + cluster, + run_offset.cluster, + @as(isize, @intCast(cluster)) - @as(isize, @intCast(cell_offset.cluster)), + x_offset, + position.y, + run_offset.x, + run_offset_y, + cell_offset.x, + cell_offset_y, + is_previous_codepoint_prepend, + formatted_cps, + }); + } + + if (positions_differ) { + log.warn("position differs from advance: cluster={d} pos=({d:.2},{d:.2}) adv=({d:.2},{d:.2}) diff=({d:.2},{d:.2}) cps = {s}", .{ + cluster, + x_offset, + position.y, + advance_x_offset, + advance_y_offset, + x_offset_diff, + y_offset_diff, + formatted_cps, + }); + } + + if (position_y_differs) { + log.warn("position.y differs from old offset.y: cluster={d} pos=({d:.2},{d:.2}) run_offset=({d:.2},{d:.2}) cell_offset=({d:.2},{d:.2}) old offset.y={d:.2} cps = {s}", .{ + cluster, + x_offset, + position.y, + run_offset.x, + run_offset_y, + cell_offset.x, + cell_offset_y, + old_offset_y, + formatted_cps, + }); + } + + if (cluster_differs) { + log.warn("cell_offset.cluster differs from cluster (potential ligature detected) cell_offset.cluster={d} cluster={d} run_offset.cluster={d} diff={d} pos=({d:.2},{d:.2}) run_offset=({d:.2},{d:.2}) cell_offset=({d:.2},{d:.2}) is_prev_prepend={} cps = {s}", .{ + cell_offset.cluster, + cluster, + run_offset.cluster, + @as(isize, @intCast(cluster)) - @as(isize, @intCast(cell_offset.cluster)), + x_offset, + position.y, + run_offset.x, + run_offset_y, + cell_offset.x, + cell_offset_y, + is_previous_codepoint_prepend, + formatted_cps, + }); + } + } +}; + +test "run iterator" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + { + var t: terminal.Terminal = try .init(alloc, .{ + .cols = 5, + .rows = 3, + }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("ABCD"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |_| count += 1; + try testing.expectEqual(@as(usize, 1), count); + } + + // Spaces should be part of a run + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("ABCD EFG"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |_| count += 1; + try testing.expectEqual(@as(usize, 1), count); + } + + { + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("A😃D"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |_| count += 1; + try testing.expectEqual(@as(usize, 3), count); + } + + // Bad ligatures + for (&[_][]const u8{ "fl", "fi", "st" }) |bad| { + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(bad); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |_| count += 1; + try testing.expectEqual(@as(usize, 2), count); + } +} + +test "run iterator: empty cells with background set" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + { + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // Set red background + s.nextSlice("\x1b[48;2;255;0;0m"); + s.nextSlice("A"); + + // Get our first row + { + const list_cell = t.screens.active.pages.getCell(.{ .active = .{ .x = 1 } }).?; + const cell = list_cell.cell; + cell.* = .{ + .content_tag = .bg_color_rgb, + .content = .{ .color_rgb = .{ .r = 0xFF, .g = 0, .b = 0 } }, + }; + } + { + const list_cell = t.screens.active.pages.getCell(.{ .active = .{ .x = 2 } }).?; + const cell = list_cell.cell; + cell.* = .{ + .content_tag = .bg_color_rgb, + .content = .{ .color_rgb = .{ .r = 0xFF, .g = 0, .b = 0 } }, + }; + } + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + { + const run = (try it.next(alloc)).?; + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 3), cells.len); + } + try testing.expect(try it.next(alloc) == null); + } +} + +test "shape" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain + buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain + buf_idx += try std.unicode.utf8Encode(0x1F3FD, buf[buf_idx..]); // Medium skin tone + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape nerd fonts" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaperWithFont(alloc, .nerd_font); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(' ', buf[buf_idx..]); // space + buf_idx += try std.unicode.utf8Encode(0xF024B, buf[buf_idx..]); // nf-md-folder + buf_idx += try std.unicode.utf8Encode(' ', buf[buf_idx..]); // space + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape inconsolata ligs" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(">="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + try testing.expectEqual(@as(usize, 2), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("==="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + try testing.expectEqual(@as(usize, 3), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +test "shape monaspace ligs" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaperWithFont(alloc, .monaspace_neon); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("==="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + try testing.expectEqual(@as(usize, 3), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +// https://github.com/mitchellh/ghostty/issues/1708 +test "shape left-replaced lig in last run" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaperWithFont(alloc, .geist_mono); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("!=="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + try testing.expectEqual(@as(usize, 3), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +// https://github.com/mitchellh/ghostty/issues/1708 +test "shape left-replaced lig in early run" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaperWithFont(alloc, .geist_mono); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("!==X"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + + const run = (try it.next(alloc)).?; + + try testing.expectEqual(@as(usize, 4), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 2), cells.len); + } +} + +// https://github.com/mitchellh/ghostty/issues/1664 +test "shape U+3C9 with JB Mono" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaperWithFont(alloc, .jetbrains_mono); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("\u{03C9} foo"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + + var run_count: usize = 0; + var cell_count: usize = 0; + while (try it.next(alloc)) |run| { + run_count += 1; + const cells = try shaper.shape(run); + cell_count += cells.len; + } + try testing.expectEqual(@as(usize, 1), run_count); + try testing.expectEqual(@as(usize, 5), cell_count); + } +} + +test "shape emoji width" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("👍"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +test "shape emoji width long" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen and add a long emoji sequence to it. + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + var page = t.screens.active.pages.pages.first.?.data; + var row = page.getRow(1); + const cell = &row.cells.ptr(page.memory)[0]; + cell.* = .{ + .content_tag = .codepoint, + .content = .{ .codepoint = 0x1F9D4 }, // Person with beard + }; + var graphemes = [_]u21{ + 0x1F3FB, // Light skin tone (Fitz 1-2) + 0x200D, // ZWJ + 0x2642, // Male sign + 0xFE0F, // Emoji presentation selector + }; + try page.setGraphemes( + row, + cell, + graphemes[0..], + ); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(1).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + const cells = try shaper.shape(run); + + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape variation selector VS15" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x270C, buf[buf_idx..]); // Victory sign (default text) + buf_idx += try std.unicode.utf8Encode(0xFE0E, buf[buf_idx..]); // ZWJ to force text + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape variation selector VS16" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x270C, buf[buf_idx..]); // Victory sign (default text) + buf_idx += try std.unicode.utf8Encode(0xFE0F, buf[buf_idx..]); // ZWJ to force color + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape with empty cells in between" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("A"); + s.nextSlice("\x1b[5C"); // 5 spaces forward + s.nextSlice("B"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), count); + try testing.expectEqual(@as(usize, 7), cells.len); + } +} + +test "shape Combining characters" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode('n', buf[buf_idx..]); // Combining + buf_idx += try std.unicode.utf8Encode(0x0308, buf[buf_idx..]); // Combining + buf_idx += try std.unicode.utf8Encode(0x0308, buf[buf_idx..]); + buf_idx += try std.unicode.utf8Encode('a', buf[buf_idx..]); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 4), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); + try testing.expectEqual(@as(u16, 1), cells[3].x); + } + try testing.expectEqual(@as(usize, 1), count); +} + +// This test exists because the string it uses causes CoreText to output a +// non-monotonic run, which we need to handle by sorting the resulting buffer. +test "shape Devanagari string" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports devanagari for this to work, if we can't + // find Arial Unicode MS, which is a system font on macOS, we just skip + // the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Arial Unicode MS", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Disable grapheme clustering + t.modes.set(.grapheme_cluster, false); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("अपार्टमेंट"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + + const run = try it.next(alloc); + try testing.expect(run != null); + const cells = try shaper.shape(run.?); + + // To understand the `x`/`cluster` assertions here, run with the "For + // debugging positions" code turned on and `extra_debugging` set to true. + try testing.expectEqual(@as(usize, 8), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 1), cells[1].x); + try testing.expectEqual(@as(u16, 2), cells[2].x); + try testing.expectEqual(@as(u16, 4), cells[3].x); + try testing.expectEqual(@as(u16, 4), cells[4].x); + try testing.expectEqual(@as(u16, 5), cells[5].x); + try testing.expectEqual(@as(u16, 5), cells[6].x); + try testing.expectEqual(@as(u16, 6), cells[7].x); + + try testing.expect(try it.next(alloc) == null); +} + +test "shape Tai Tham vowels (position differs from advance)" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports Tai Tham for this to work, if we can't find + // Noto Sans Tai Tham, which is a system font on macOS, we just skip the + // test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Tai Tham", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x1a2F, buf[buf_idx..]); // ᨯ + buf_idx += try std.unicode.utf8Encode(0x1a70, buf[buf_idx..]); // ᩰ + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + const cell_width = run.grid.metrics.cell_width; + try testing.expectEqual(@as(usize, 2), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + + // The first glyph renders in the next cell + try testing.expectEqual(@as(i16, @intCast(cell_width)), cells[0].x_offset); + try testing.expectEqual(@as(i16, 0), cells[1].x_offset); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape Tai Tham letters (position.y differs from advance)" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports Tai Tham for this to work, if we can't find + // Noto Sans Tai Tham, which is a system font on macOS, we just skip the + // test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Tai Tham", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x1a49, buf[buf_idx..]); // HA + buf_idx += try std.unicode.utf8Encode(0x1a60, buf[buf_idx..]); // SAKOT + // Second grapheme cluster, combining with the first in a ligature: + buf_idx += try std.unicode.utf8Encode(0x1a3f, buf[buf_idx..]); // YA + buf_idx += try std.unicode.utf8Encode(0x1a69, buf[buf_idx..]); // U + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 3), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); // U from second grapheme + + // The U glyph renders at a y below zero + try testing.expectEqual(@as(i16, -3), cells[2].y_offset); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape Javanese ligatures" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports Javanese for this to work, if we can't find + // Noto Sans Javanese Regular, which is a system font on macOS, we just + // skip the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Javanese", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0xa9a4, buf[buf_idx..]); // NA + buf_idx += try std.unicode.utf8Encode(0xa9c0, buf[buf_idx..]); // PANGKON + // Second grapheme cluster, combining with the first in a ligature: + buf_idx += try std.unicode.utf8Encode(0xa9b2, buf[buf_idx..]); // HA + buf_idx += try std.unicode.utf8Encode(0xa9b8, buf[buf_idx..]); // Vowel sign SUKU + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + const cell_width = run.grid.metrics.cell_width; + try testing.expectEqual(@as(usize, 3), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); + + // The vowel sign SUKU renders with correct x_offset + try testing.expect(cells[2].x_offset > 3 * cell_width); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape Chakma vowel sign with ligature (vowel sign renders first)" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports Chakma for this to work, if we can't find + // Noto Sans Chakma Regular, which is a system font on macOS, we just skip + // the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Chakma", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x1111d, buf[buf_idx..]); // BAA + // Second grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x11116, buf[buf_idx..]); // TAA + buf_idx += try std.unicode.utf8Encode(0x11133, buf[buf_idx..]); // Virama + // Third grapheme cluster, combining with the second in a ligature: + buf_idx += try std.unicode.utf8Encode(0x11120, buf[buf_idx..]); // YYAA + buf_idx += try std.unicode.utf8Encode(0x1112c, buf[buf_idx..]); // Vowel Sign U + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 4), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + // See the giant "We need to reset the `cell_offset`" comment, but here + // we should technically have the rest of these be `x` of 1, but that + // would require going back in the stream to adjust past cells, and + // we don't take on that complexity. + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); + try testing.expectEqual(@as(u16, 0), cells[3].x); + + // The vowel sign U renders before the TAA: + try testing.expect(cells[1].x_offset < cells[2].x_offset); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape Bengali ligatures with out of order vowels" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports Bengali for this to work, if we can't find + // Arial Unicode MS, which is a system font on macOS, we just skip the + // test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Arial Unicode MS", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x09b0, buf[buf_idx..]); // RA + buf_idx += try std.unicode.utf8Encode(0x09be, buf[buf_idx..]); // Vowel sign AA + // Second grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x09b7, buf[buf_idx..]); // SSA + buf_idx += try std.unicode.utf8Encode(0x09cd, buf[buf_idx..]); // Virama + // Third grapheme cluster, combining with the second in a ligature: + buf_idx += try std.unicode.utf8Encode(0x099f, buf[buf_idx..]); // TTA + buf_idx += try std.unicode.utf8Encode(0x09cd, buf[buf_idx..]); // Virama + // Fourth grapheme cluster, combining with the previous two in a ligature: + buf_idx += try std.unicode.utf8Encode(0x09b0, buf[buf_idx..]); // RA + buf_idx += try std.unicode.utf8Encode(0x09c7, buf[buf_idx..]); // Vowel sign E + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 8), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + // See the giant "We need to reset the `cell_offset`" comment, but here + // we should technically have the rest of these be `x` of 2, but that + // would require going back in the stream to adjust past cells, and + // we don't take on that complexity. + try testing.expectEqual(@as(u16, 0), cells[2].x); + try testing.expectEqual(@as(u16, 0), cells[3].x); + try testing.expectEqual(@as(u16, 0), cells[4].x); + try testing.expectEqual(@as(u16, 0), cells[5].x); + try testing.expectEqual(@as(u16, 0), cells[6].x); + try testing.expectEqual(@as(u16, 0), cells[7].x); + + // The vowel sign E renders before the SSA: + try testing.expect(cells[2].x_offset < cells[3].x_offset); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape box glyphs" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x2500, buf[buf_idx..]); // horiz line + buf_idx += try std.unicode.utf8Encode(0x2501, buf[buf_idx..]); // + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 2), cells.len); + try testing.expectEqual(@as(u32, 0x2500), cells[0].glyph_index); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u32, 0x2501), cells[1].glyph_index); + try testing.expectEqual(@as(u16, 1), cells[1].x); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape selection boundary" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("a1b2c3d4e5"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Full line selection + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 0, @intCast(t.cols - 1) }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + // Offset x, goes to end of line selection + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 2, @intCast(t.cols - 1) }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + + // Offset x, starts at beginning of line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 0, 3 }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + + // Selection only subset of line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 1, 3 }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 3), count); + } + + // Selection only one character + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 1, 1 }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 3), count); + } +} + +test "shape cursor boundary" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("a1b2c3d4e5"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // No cursor is full line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + { + // Cursor at index 0 is two runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 0, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + } + + { + // Cursor at index 1 is three runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 1, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 3), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + } + { + // Cursor at last col is two runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 9, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + } +} + +test "shape cursor boundary and colored emoji" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("👍🏼"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // No cursor is full line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + // Cursor on emoji does not split it + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 0, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 1, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +test "shape cell attribute change" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Plain >= should shape into 1 run + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(">="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + // Bold vs regular should split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(">"); + s.nextSlice("\x1b[1m"); // Bold + s.nextSlice("="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + + // Changing fg color should split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 + s.nextSlice("\x1b[38;2;1;2;3m"); + s.nextSlice(">"); + // RGB 3, 2, 1 + s.nextSlice("\x1b[38;2;3;2;1m"); + s.nextSlice("="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + + // Changing bg color should NOT split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 bg + s.nextSlice("\x1b[48;2;1;2;3m"); + s.nextSlice(">"); + // RGB 3, 2, 1 bg + s.nextSlice("\x1b[48;2;3;2;1m"); + s.nextSlice("="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + // Same bg color should not split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 bg + s.nextSlice("\x1b[48;2;1;2;3m"); + s.nextSlice(">"); + s.nextSlice("="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +test "shape high plane sprite font codepoint" { + // While creating runs, the CoreText shaper uses `0` codepoints to + // pad its codepoint list to account for high plane characters which + // use two UTF-16 code units. This is so that, after shaping, the string + // indices can be used to find the originating codepoint / cluster. + // + // This is a problem for special (sprite) fonts, which need to be "shaped" + // by simply returning the input codepoints verbatim. We include logic to + // skip `0` codepoints when constructing the shaped run for sprite fonts, + // this test verifies that it works correctly. + + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // U+1FB70: Vertical One Eighth Block-2 + s.nextSlice("\u{1FB70}"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + // We should get one run + const run = (try it.next(alloc)).?; + // The run state should have the UTF-16 encoding of the character. + try testing.expectEqualSlices( + u16, + &.{ 0xD83E, 0xDF70 }, + shaper.run_state.unichars.items, + ); + // The codepoint list should be padded. + try testing.expectEqualSlices( + Shaper.Codepoint, + &.{ + .{ .codepoint = 0x1FB70, .cluster = 0 }, + .{ .codepoint = 0, .cluster = 0 }, + }, + shaper.run_state.codepoints.items, + ); + // And when shape it + const cells = try shaper.shape(run); + // we should have + // - 1 cell + try testing.expectEqual(1, run.cells); + // - at position 0 + try testing.expectEqual(0, run.offset); + // - with 1 glyph in it + try testing.expectEqual(1, cells.len); + // - at position 0 + try testing.expectEqual(0, cells[0].x); + // - the glyph index should be equal to the codepoint + try testing.expectEqual(0x1FB70, cells[0].glyph_index); + // - it should be a sprite font + try testing.expect(run.font_index.special() != null); + // And we should get a null run after that + try testing.expectEqual(null, try it.next(alloc)); +} + +const TestShaper = struct { + alloc: Allocator, + shaper: Shaper, + grid: *SharedGrid, + lib: Library, + + pub fn deinit(self: *TestShaper) void { + self.shaper.deinit(); + self.grid.deinit(self.alloc); + self.alloc.destroy(self.grid); + self.lib.deinit(); + } +}; + +const TestFont = enum { + code_new_roman, + geist_mono, + inconsolata, + jetbrains_mono, + monaspace_neon, + nerd_font, +}; + +/// Helper to return a fully initialized shaper. +fn testShaper(alloc: Allocator) !TestShaper { + return try testShaperWithFont(alloc, .inconsolata); +} + +fn testShaperWithFont(alloc: Allocator, font_req: TestFont) !TestShaper { + const testEmoji = font.embedded.emoji; + const testEmojiText = font.embedded.emoji_text; + const testFont = switch (font_req) { + .code_new_roman => font.embedded.code_new_roman, + .inconsolata => font.embedded.inconsolata, + .geist_mono => font.embedded.geist_mono, + .jetbrains_mono => font.embedded.jetbrains_mono, + .monaspace_neon => font.embedded.monaspace_neon, + .nerd_font => font.embedded.test_nerd_font, + }; + + var lib = try Library.init(alloc); + errdefer lib.deinit(); + + var c = Collection.init(); + c.load_options = .{ .library = lib }; + + // Setup group + _ = try c.add(alloc, try .init( + lib, + testFont, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + + if (font.options.backend != .coretext) { + // Coretext doesn't support Noto's format + _ = try c.add(alloc, try .init( + lib, + testEmoji, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + } else { + // On CoreText we want to load Apple Emoji, we should have it. + var disco = font.Discover.init(); + defer disco.deinit(); + var disco_it = try disco.discover(alloc, .{ + .family = "Apple Color Emoji", + .size = 12, + .monospace = false, + }); + defer disco_it.deinit(); + var face = (try disco_it.next()).?; + errdefer face.deinit(); + _ = try c.addDeferred(alloc, face, .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + } + _ = try c.add(alloc, try .init( + lib, + testEmojiText, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + + const grid_ptr = try alloc.create(SharedGrid); + errdefer alloc.destroy(grid_ptr); + grid_ptr.* = try .init(alloc, .{ .collection = c }); + errdefer grid_ptr.*.deinit(alloc); + + var shaper = try Shaper.init(alloc, .{ + // Some of our tests rely on dlig being enabled by default + .features = &.{"dlig"}, + }); + errdefer shaper.deinit(); + + return TestShaper{ + .alloc = alloc, + .shaper = shaper, + .grid = grid_ptr, + .lib = lib, + }; +} + +/// Return a fully initialized shaper by discovering a named font on the system. +fn testShaperWithDiscoveredFont(alloc: Allocator, font_req: [:0]const u8) !TestShaper { + var lib = try Library.init(alloc); + errdefer lib.deinit(); + + var c = Collection.init(); + c.load_options = .{ .library = lib }; + + // Discover and add our font to the collection. + { + var disco = font.Discover.init(); + defer disco.deinit(); + var disco_it = try disco.discover(alloc, .{ + .family = font_req, + .size = 12, + .monospace = false, + }); + defer disco_it.deinit(); + var face: font.DeferredFace = (try disco_it.next()) orelse return error.FontNotFound; + errdefer face.deinit(); + _ = try c.add( + alloc, + try face.load(lib, .{ .size = .{ .points = 12 } }), + .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }, + ); + } + + const grid_ptr = try alloc.create(SharedGrid); + errdefer alloc.destroy(grid_ptr); + grid_ptr.* = try .init(alloc, .{ .collection = c }); + errdefer grid_ptr.*.deinit(alloc); + + var shaper = try Shaper.init(alloc, .{}); + errdefer shaper.deinit(); + + return TestShaper{ + .alloc = alloc, + .shaper = shaper, + .grid = grid_ptr, + .lib = lib, + }; +} diff --git a/vendor/libghostty-vt/src/font/shaper/feature.zig b/vendor/libghostty-vt/src/font/shaper/feature.zig new file mode 100644 index 00000000..5bd73f97 --- /dev/null +++ b/vendor/libghostty-vt/src/font/shaper/feature.zig @@ -0,0 +1,354 @@ +const std = @import("std"); +const assert = @import("../../quirks.zig").inlineAssert; +const Allocator = std.mem.Allocator; + +const log = std.log.scoped(.font_shaper); + +/// Represents an OpenType font feature setting, which consists of a tag and +/// a numeric parameter >= 0. Most features are boolean, so only parameters +/// of 0 and 1 make sense for them, but some (e.g. 'cv01'..'cv99') can take +/// parameters to choose between multiple variants of a given character or +/// characters. +/// +/// Ref: +/// - https://learn.microsoft.com/en-us/typography/opentype/spec/chapter2#features-and-lookups +/// - https://harfbuzz.github.io/shaping-opentype-features.html +pub const Feature = struct { + tag: [4]u8, + value: u32, + + pub fn fromString(str: []const u8) ?Feature { + var fbs = std.io.fixedBufferStream(str); + const reader = fbs.reader(); + return .fromReader(reader); + } + + /// Parse a single font feature setting from a std.io.Reader, with a version + /// of the syntax of HarfBuzz's font feature strings. Stops at end of stream + /// or when a ',' is encountered. + /// + /// This parsing aims to be as error-tolerant as possible while avoiding any + /// assumptions in ambiguous scenarios. When invalid syntax is encountered, + /// the reader is advanced to the next boundary (end-of-stream or ',') so + /// that further features may be read. + /// + /// Ref: https://harfbuzz.github.io/harfbuzz-hb-common.html#hb-feature-from-string + pub fn fromReader(reader: anytype) ?Feature { + var tag_buf: [4]u8 = undefined; + var tag: []u8 = tag_buf[0..0]; + var value: ?u32 = null; + + state: switch ((enum { + /// Initial state. + start, + /// Parsing the tag. + tag, + /// In the space between the tag and the value. + space, + /// Parsing an integer parameter directly in to `value`. + int, + /// Parsing a boolean keyword parameter ("on"/"off"). + bool, + /// Encountered an unrecoverable syntax error, advancing to boundary. + err, + /// Done parsing feature, skip whitespace until end. + done, + }).start) { + // If we're done then we skip whitespace until we see a ','. + .done => while (true) switch (reader.readByte() catch ',') { + ' ', '\t' => continue, + ',' => break, + // If we see something other than whitespace or a ',' + // then this is an error since the intent is unclear. + else => continue :state .err, + }, + + // If we're fast-forwarding from an error we just wanna + // stop at the first boundary and ignore all other bytes. + .err => { + reader.skipUntilDelimiterOrEof(',') catch {}; + return null; + }, + + .start => while (true) switch (reader.readByte() catch ',') { + // Ignore leading whitespace. + ' ', '\t' => continue, + // Empty feature string. + ',' => return null, + // '+' prefix to explicitly enable feature. + '+' => { + value = 1; + continue :state .tag; + }, + // '-' prefix to explicitly disable feature. + '-' => { + value = 0; + continue :state .tag; + }, + // Quote mark introducing a tag. + '"', '\'' => { + continue :state .tag; + }, + // First letter of tag. + else => |byte| { + tag.len = 1; + tag[0] = byte; + continue :state .tag; + }, + }, + + .tag => while (true) switch (reader.readByte() catch ',') { + // If the tag is interrupted by a comma it's invalid. + ',' => return null, + // Ignore quote marks. This does technically ignore cases like + // "'k'e'r'n' = 0", but it's unambiguous so if someone really + // wants to do that in their config then... sure why not. + '"', '\'' => continue, + // In all other cases we add the byte to our tag. + else => |byte| { + tag.len += 1; + tag[tag.len - 1] = byte; + if (tag.len == 4) continue :state .space; + }, + }, + + .space => while (true) switch (reader.readByte() catch ',') { + ' ', '\t' => continue, + // Ignore quote marks since we might have a + // closing quote from the tag still ahead. + '"', '\'' => continue, + // Allow an '=' (which we can safely ignore) + // only if we don't already have a value due + // to a '+' or '-' prefix. + '=' => if (value != null) continue :state .err, + ',' => { + // Specifying only a tag turns a feature on. + if (value == null) value = 1; + break; + }, + '0'...'9' => |byte| { + // If we already have value because of a + // '+' or '-' prefix then this is an error. + if (value != null) continue :state .err; + value = byte - '0'; + continue :state .int; + }, + 'o', 'O' => { + // If we already have value because of a + // '+' or '-' prefix then this is an error. + if (value != null) continue :state .err; + continue :state .bool; + }, + else => continue :state .err, + }, + + .int => while (true) switch (reader.readByte() catch ',') { + ',' => break, + '0'...'9' => |byte| { + // If our value gets too big while + // parsing we consider it an error. + value = std.math.mul(u32, value.?, 10) catch { + continue :state .err; + }; + value.? += byte - '0'; + }, + else => continue :state .err, + }, + + .bool => while (true) switch (reader.readByte() catch ',') { + ',' => return null, + 'n', 'N' => { + // "ofn" + if (value != null) { + assert(value == 0); + continue :state .err; + } + value = 1; + continue :state .done; + }, + 'f', 'F' => { + // To make sure we consume two 'f's. + if (value == null) { + value = 0; + } else { + assert(value == 0); + continue :state .done; + } + }, + else => continue :state .err, + }, + } + + assert(value != null); + assert(tag.len == 4); + + return .{ + .tag = tag_buf, + .value = value.?, + }; + } + + /// Serialize this feature to the provided buffer. + /// The string that this produces should be valid to parse. + pub fn toString(self: *const Feature, buf: []u8) !void { + var fbs = std.io.fixedBufferStream(buf); + try self.format("", .{}, fbs.writer()); + } + + /// Formatter for logging + pub fn format( + self: Feature, + comptime layout: []const u8, + opts: std.fmt.FormatOptions, + writer: *std.Io.Writer, + ) !void { + _ = layout; + _ = opts; + if (self.value <= 1) { + // Format boolean options as "+tag" for on and "-tag" for off. + try std.fmt.format(writer, "{c}{s}", .{ + "-+"[self.value], + self.tag, + }); + } else { + // Format non-boolean tags as "tag=value". + try std.fmt.format(writer, "{s}={d}", .{ + self.tag, + self.value, + }); + } + } +}; + +/// A list of font feature settings (see `Feature` for more documentation). +pub const FeatureList = struct { + features: std.ArrayListUnmanaged(Feature) = .{}, + + pub fn deinit(self: *FeatureList, alloc: Allocator) void { + self.features.deinit(alloc); + } + + /// Parse a comma separated list of features. + /// See `Feature.fromReader` for more docs. + pub fn fromString(alloc: Allocator, str: []const u8) !FeatureList { + var self: FeatureList = .{}; + try self.appendFromString(alloc, str); + return self; + } + + /// Append features to this list from a string with a comma separated list. + /// See `Feature.fromReader` for more docs. + pub fn appendFromString( + self: *FeatureList, + alloc: Allocator, + str: []const u8, + ) !void { + var fbs = std.io.fixedBufferStream(str); + const reader = fbs.reader(); + while (fbs.pos < fbs.buffer.len) { + const i = fbs.pos; + if (Feature.fromReader(reader)) |feature| { + try self.features.append(alloc, feature); + } else log.warn( + "failed to parse font feature setting: \"{s}\"", + .{fbs.buffer[i..fbs.pos]}, + ); + } + } + + /// Formatter for logging + pub fn format( + self: FeatureList, + comptime layout: []const u8, + opts: std.fmt.FormatOptions, + writer: *std.Io.Writer, + ) !void { + for (self.features.items, 0..) |feature, i| { + try feature.format(layout, opts, writer); + if (i != std.features.items.len - 1) try writer.writeAll(", "); + } + if (self.value <= 1) { + // Format boolean options as "+tag" for on and "-tag" for off. + try std.fmt.format(writer, "{c}{s}", .{ + "-+"[self.value], + self.tag, + }); + } else { + // Format non-boolean tags as "tag=value". + try std.fmt.format(writer, "{s}={d}", .{ + self.tag, + self.value, + }); + } + } +}; + +/// These features are hardcoded to always be on by default. Users +/// can turn them off by setting the features to "-liga" for example. +pub const default_features = [_]Feature{ + .{ .tag = "liga".*, .value = 1 }, +}; + +test "Feature.fromString" { + const testing = std.testing; + + // This is not *complete* coverage of every possible + // combination of syntax, but it covers quite a few. + + // Boolean settings (on) + const kern_on = Feature{ .tag = "kern".*, .value = 1 }; + try testing.expectEqual(kern_on, Feature.fromString("kern")); + try testing.expectEqual(kern_on, Feature.fromString("kern, ")); + try testing.expectEqual(kern_on, Feature.fromString("kern on")); + try testing.expectEqual(kern_on, Feature.fromString("kern on, ")); + try testing.expectEqual(kern_on, Feature.fromString("+kern")); + try testing.expectEqual(kern_on, Feature.fromString("+kern, ")); + try testing.expectEqual(kern_on, Feature.fromString("\"kern\" = 1")); + try testing.expectEqual(kern_on, Feature.fromString("\"kern\" = 1, ")); + + // Boolean settings (off) + const kern_off = Feature{ .tag = "kern".*, .value = 0 }; + try testing.expectEqual(kern_off, Feature.fromString("kern off")); + try testing.expectEqual(kern_off, Feature.fromString("kern off, ")); + try testing.expectEqual(kern_off, Feature.fromString("-'kern'")); + try testing.expectEqual(kern_off, Feature.fromString("-'kern', ")); + try testing.expectEqual(kern_off, Feature.fromString("\"kern\" = 0")); + try testing.expectEqual(kern_off, Feature.fromString("\"kern\" = 0, ")); + + // Non-boolean settings + const aalt_2 = Feature{ .tag = "aalt".*, .value = 2 }; + try testing.expectEqual(aalt_2, Feature.fromString("aalt=2")); + try testing.expectEqual(aalt_2, Feature.fromString("aalt=2, ")); + try testing.expectEqual(aalt_2, Feature.fromString("'aalt' 2")); + try testing.expectEqual(aalt_2, Feature.fromString("'aalt' 2, ")); + + // Various ambiguous/error cases which should be null + try testing.expectEqual(null, Feature.fromString("aalt=2x")); // bad number + try testing.expectEqual(null, Feature.fromString("toolong")); // tag too long + try testing.expectEqual(null, Feature.fromString("sht")); // tag too short + try testing.expectEqual(null, Feature.fromString("-kern 1")); // redundant/conflicting + try testing.expectEqual(null, Feature.fromString("-kern on")); // redundant/conflicting + try testing.expectEqual(null, Feature.fromString("aalt=o,")); // bad keyword + try testing.expectEqual(null, Feature.fromString("aalt=ofn,")); // bad keyword +} + +test "FeatureList.fromString" { + const testing = std.testing; + + const str = + " kern, kern on , +kern, \"kern\" = 1," ++ // Boolean settings (on) + "kern off, -'kern' , \"kern\"=0," ++ // Boolean settings (off) + "aalt=2, 'aalt'\t2," ++ // Non-boolean settings + "aalt=2x, toolong, sht, -kern 1, -kern on, aalt=o, aalt=ofn," ++ // Invalid cases + "last"; // To ensure final element is included correctly. + var feats = try FeatureList.fromString(testing.allocator, str); + defer feats.deinit(testing.allocator); + try testing.expectEqualSlices( + Feature, + &(.{Feature{ .tag = "kern".*, .value = 1 }} ** 4 ++ + .{Feature{ .tag = "kern".*, .value = 0 }} ** 3 ++ + .{Feature{ .tag = "aalt".*, .value = 2 }} ** 2 ++ + .{Feature{ .tag = "last".*, .value = 1 }}), + feats.features.items, + ); +} diff --git a/vendor/libghostty-vt/src/font/shaper/harfbuzz.zig b/vendor/libghostty-vt/src/font/shaper/harfbuzz.zig new file mode 100644 index 00000000..9c78d9de --- /dev/null +++ b/vendor/libghostty-vt/src/font/shaper/harfbuzz.zig @@ -0,0 +1,2172 @@ +const std = @import("std"); +const assert = @import("../../quirks.zig").inlineAssert; +const Allocator = std.mem.Allocator; +const harfbuzz = @import("harfbuzz"); +const font = @import("../main.zig"); +const terminal = @import("../../terminal/main.zig"); +const unicode = @import("../../unicode/main.zig"); +const Feature = font.shape.Feature; +const FeatureList = font.shape.FeatureList; +const default_features = font.shape.default_features; +const Face = font.Face; +const Collection = font.Collection; +const DeferredFace = font.DeferredFace; +const Library = font.Library; +const SharedGrid = font.SharedGrid; +const Style = font.Style; +const Presentation = font.Presentation; + +const log = std.log.scoped(.font_shaper); + +/// Shaper that uses Harfbuzz. +pub const Shaper = struct { + /// The allocated used for the feature list, cell buf, and codepoints. + alloc: Allocator, + + /// The buffer used for text shaping. We reuse it across multiple shaping + /// calls to prevent allocations. + hb_buf: harfbuzz.Buffer, + + /// The shared memory used for shaping results. + cell_buf: CellBuf, + + /// The features to use for shaping. + hb_feats: []harfbuzz.Feature, + + /// The codepoints added to the buffer before shaping. We need to keep + /// these separately because after shaping, HarfBuzz replaces codepoints + /// with glyph indices in the buffer. + codepoints: std.ArrayListUnmanaged(Codepoint) = .{}, + + const Codepoint = struct { + cluster: u32, + codepoint: u32, + }; + + const CellBuf = std.ArrayListUnmanaged(font.shape.Cell); + + const RunOffset = struct { + cluster: u32 = 0, + x: i32 = 0, + y: i32 = 0, + }; + + const CellOffset = struct { + cluster: u32 = 0, + x: i32 = 0, + }; + + /// The cell_buf argument is the buffer to use for storing shaped results. + /// This should be at least the number of columns in the terminal. + pub fn init(alloc: Allocator, opts: font.shape.Options) !Shaper { + // Parse all the features we want to use. + const hb_feats = hb_feats: { + var feature_list: FeatureList = .{}; + defer feature_list.deinit(alloc); + try feature_list.features.appendSlice(alloc, &default_features); + for (opts.features) |feature_str| { + try feature_list.appendFromString(alloc, feature_str); + } + + var list = try alloc.alloc(harfbuzz.Feature, feature_list.features.items.len); + errdefer alloc.free(list); + + for (feature_list.features.items, 0..) |feature, i| { + list[i] = .{ + .tag = std.mem.nativeToBig(u32, @bitCast(feature.tag)), + .value = feature.value, + .start = harfbuzz.c.HB_FEATURE_GLOBAL_START, + .end = harfbuzz.c.HB_FEATURE_GLOBAL_END, + }; + } + + break :hb_feats list; + }; + errdefer alloc.free(hb_feats); + + return Shaper{ + .alloc = alloc, + .hb_buf = try harfbuzz.Buffer.create(), + .cell_buf = .{}, + .hb_feats = hb_feats, + }; + } + + pub fn deinit(self: *Shaper) void { + self.hb_buf.destroy(); + self.cell_buf.deinit(self.alloc); + self.alloc.free(self.hb_feats); + self.codepoints.deinit(self.alloc); + } + + pub fn endFrame(self: *const Shaper) void { + _ = self; + } + + /// Returns an iterator that returns one text run at a time for the + /// given terminal row. Note that text runs are only valid one at a time + /// for a Shaper struct since they share state. + /// + /// The selection must be a row-only selection (height = 1). See + /// Selection.containedRow. The run iterator will ONLY look at X values + /// and assume the y value matches. + pub fn runIterator( + self: *Shaper, + opts: font.shape.RunOptions, + ) font.shape.RunIterator { + return .{ + .hooks = .{ .shaper = self }, + .opts = opts, + }; + } + + /// Shape the given text run. The text run must be the immediately previous + /// text run that was iterated since the text run does share state with the + /// Shaper struct. + /// + /// The return value is only valid until the next shape call is called. + /// + /// If there is not enough space in the cell buffer, an error is returned. + pub fn shape(self: *Shaper, run: font.shape.TextRun) ![]const font.shape.Cell { + // We only do shaping if the font is not a special-case. For special-case + // fonts, the codepoint == glyph_index so we don't need to run any shaping. + if (run.font_index.special() == null) { + // We have to lock the grid to get the face and unfortunately + // freetype faces (typically used with harfbuzz) are not thread + // safe so this has to be an exclusive lock. + run.grid.lock.lock(); + defer run.grid.lock.unlock(); + + const face = try run.grid.resolver.collection.getFace(run.font_index); + const i = if (!face.quirks_disable_default_font_features) 0 else i: { + // If we are disabling default font features we just offset + // our features by the hardcoded items because always + // add those at the beginning. + break :i default_features.len; + }; + + harfbuzz.shape(face.hb_font, self.hb_buf, self.hb_feats[i..]); + } + + // If our buffer is empty, we short-circuit the rest of the work + // return nothing. + if (self.hb_buf.getLength() == 0) return self.cell_buf.items[0..0]; + const info = self.hb_buf.getGlyphInfos(); + const pos = self.hb_buf.getGlyphPositions() orelse return error.HarfbuzzFailed; + + // This is perhaps not true somewhere, but we currently assume it is true. + // If it isn't true, I'd like to catch it and learn more. + assert(info.len == pos.len); + + // This keeps track of the current x and y offsets (sum of advances) + // and the furthest cluster we've seen so far (max). + var run_offset: RunOffset = .{}; + + // This keeps track of the cell starting x and cluster. + var cell_offset: CellOffset = .{}; + + // Convert all our info/pos to cells and set it. + self.cell_buf.clearRetainingCapacity(); + for (info, pos) |info_v, pos_v| { + // info_v.cluster is the index into our codepoints array. We use it + // to get the original cluster. + const index = info_v.cluster; + // Our cluster is also our cell X position. If the cluster changes + // then we need to reset our current cell offsets. + const cluster = self.codepoints.items[index].cluster; + if (cell_offset.cluster != cluster) { + const is_after_glyph_from_current_or_next_clusters = + cluster <= run_offset.cluster; + + const is_first_codepoint_in_cluster = blk: { + var i = index; + while (i > 0) { + i -= 1; + const codepoint = self.codepoints.items[i]; + break :blk codepoint.cluster != cluster; + } else break :blk true; + }; + + // We need to reset the `cell_offset` at the start of a new + // cluster, but we do that conditionally if the codepoint + // `is_first_codepoint_in_cluster` and the cluster is not + // `is_after_glyph_from_current_or_next_clusters`, which is + // a heuristic to detect ligatures and avoid positioning + // glyphs that mark ligatures incorrectly. The idea is that + // if the first codepoint in a cluster doesn't appear in + // the stream, it's very likely that it combined with + // codepoints from a previous cluster into a ligature. + // Then, the subsequent codepoints are very likely marking + // glyphs that are placed relative to that ligature, so if + // we were to reset the `cell_offset` to align it with the + // grid, the positions would be off. The + // `!is_after_glyph_from_current_or_next_clusters` check is + // needed in case these marking glyphs come from a later + // cluster but are rendered first (see the Chakma and + // Bengali tests). In that case when we get to the + // codepoint that `is_first_codepoint_in_cluster`, but in a + // cluster that + // `is_after_glyph_from_current_or_next_clusters`, we don't + // want to reset to the grid and cause the positions to be + // off. (Note that we could go back and align the cells to + // the grid starting from the one from the cluster that + // rendered out of order, but that is more complicated so + // we don't do that for now. Also, it's TBD if there are + // exceptions to this heuristic for detecting ligatures, + // but using the logging below seems to show it works + // well.) + if (is_first_codepoint_in_cluster and + !is_after_glyph_from_current_or_next_clusters) + { + cell_offset = .{ + .cluster = cluster, + .x = run_offset.x, + }; + } + } + + // Under both FreeType and CoreText the harfbuzz scale is + // in 26.6 fixed point units, so we round to the nearest + // whole value here. + const x_offset = run_offset.x - cell_offset.x + ((pos_v.x_offset + 0b100_000) >> 6); + const y_offset = run_offset.y + ((pos_v.y_offset + 0b100_000) >> 6); + + // For debugging positions, turn this on: + //try self.debugPositions(run_offset, cell_offset, pos_v, index); + + try self.cell_buf.append(self.alloc, .{ + .x = @intCast(cell_offset.cluster), + .x_offset = @intCast(x_offset), + .y_offset = @intCast(y_offset), + .glyph_index = info_v.codepoint, + }); + + // Add our advances to keep track of our run offsets. + // Advances apply to the NEXT cell. + // Under both FreeType and CoreText the harfbuzz scale is + // in 26.6 fixed point units, so we round to the nearest + // whole value here. + run_offset.x += (pos_v.x_advance + 0b100_000) >> 6; + run_offset.y += (pos_v.y_advance + 0b100_000) >> 6; + run_offset.cluster = @max(run_offset.cluster, cluster); + + // const i = self.cell_buf.items.len - 1; + // log.warn("i={} info={} pos={} cell={}", .{ i, info_v, pos_v, self.cell_buf.items[i] }); + } + //log.warn("----------------", .{}); + + return self.cell_buf.items; + } + + /// The hooks for RunIterator. + pub const RunIteratorHook = struct { + shaper: *Shaper, + + pub fn prepare(self: RunIteratorHook) void { + // Reset the buffer for our current run + self.shaper.hb_buf.reset(); + self.shaper.hb_buf.setContentType(.unicode); + + // We set the cluster level to `characters` to give us the most + // granularity, matching the CoreText shaper, and allowing us + // to use our same ligature detection heuristics. + self.shaper.hb_buf.setClusterLevel(.characters); + + self.shaper.codepoints.clearRetainingCapacity(); + + // We don't support RTL text because RTL in terminals is messy. + // Its something we want to improve. For now, we force LTR because + // our renderers assume a strictly increasing X value. + self.shaper.hb_buf.setDirection(.ltr); + } + + pub fn addCodepoint(self: RunIteratorHook, cp: u32, cluster: u32) !void { + // log.warn("cluster={} cp={x}", .{ cluster, cp }); + // We pass the index into codepoints as the cluster value to HarfBuzz. + // After shaping, we use info.cluster to get back the index, which + // lets us look up the original cluster value from codepoints. + const index: u32 = @intCast(self.shaper.codepoints.items.len); + self.shaper.hb_buf.add(cp, index); + try self.shaper.codepoints.append(self.shaper.alloc, .{ + .cluster = cluster, + .codepoint = cp, + }); + } + + pub fn finalize(self: RunIteratorHook) void { + self.shaper.hb_buf.guessSegmentProperties(); + } + }; + + fn debugPositions( + self: *Shaper, + run_offset: RunOffset, + cell_offset: CellOffset, + pos_v: harfbuzz.GlyphPosition, + index: u32, + ) !void { + const x_offset = run_offset.x - cell_offset.x + ((pos_v.x_offset + 0b100_000) >> 6); + const y_offset = run_offset.y + ((pos_v.y_offset + 0b100_000) >> 6); + const advance_x_offset = run_offset.x - cell_offset.x; + const advance_y_offset = run_offset.y; + const x_offset_diff = x_offset - advance_x_offset; + const y_offset_diff = y_offset - advance_y_offset; + const positions_differ = @abs(x_offset_diff) > 0 or @abs(y_offset_diff) > 0; + const y_offset_differs = run_offset.y != 0; + const cluster = self.codepoints.items[index].cluster; + const cluster_differs = cluster != cell_offset.cluster; + + // To debug every loop, flip this to true: + const extra_debugging = false; + + const is_previous_codepoint_prepend = if (cluster_differs or + extra_debugging) + blk: { + var i = index; + while (i > 0) { + i -= 1; + const codepoint = self.codepoints.items[i]; + break :blk unicode.table.get(@intCast(codepoint.codepoint)).grapheme_boundary_class == .prepend; + } + break :blk false; + } else false; + + const formatted_cps: ?[]u8 = if (positions_differ or + y_offset_differs or + cluster_differs or + extra_debugging) + blk: { + var allocating = std.Io.Writer.Allocating.init(self.alloc); + defer allocating.deinit(); + const writer = &allocating.writer; + const codepoints = self.codepoints.items; + var last_cluster: ?u32 = null; + for (codepoints, 0..) |cp, i| { + if (@as(i32, @intCast(cp.cluster)) >= @as(i32, @intCast(cell_offset.cluster)) - 1 and + cp.cluster <= cluster + 1) + { + if (last_cluster) |last| { + if (cp.cluster != last) { + try writer.writeAll(" "); + } + } + if (i == index) { + try writer.writeAll("▸"); + } + // Using Python syntax for easier debugging + if (cp.codepoint > 0xFFFF) { + try writer.print("\\U{x:0>8}", .{cp.codepoint}); + } else { + try writer.print("\\u{x:0>4}", .{cp.codepoint}); + } + last_cluster = cp.cluster; + } + } + try writer.writeAll(" → "); + for (codepoints) |cp| { + if (@as(i32, @intCast(cp.cluster)) >= @as(i32, @intCast(cell_offset.cluster)) - 1 and + cp.cluster <= cluster + 1) + { + try writer.print("{u}", .{@as(u21, @intCast(cp.codepoint))}); + } + } + break :blk try allocating.toOwnedSlice(); + } else null; + defer if (formatted_cps) |cps| self.alloc.free(cps); + + if (extra_debugging) { + log.warn("extra debugging of positions index={d} cell_offset.cluster={d} cluster={d} run_offset.cluster={d} diff={d} pos=({d},{d}) run_offset=({d},{d}) cell_offset.x={d} is_prev_prepend={} cps = {s}", .{ + index, + cell_offset.cluster, + cluster, + run_offset.cluster, + @as(isize, @intCast(cluster)) - @as(isize, @intCast(cell_offset.cluster)), + x_offset, + y_offset, + run_offset.x, + run_offset.y, + cell_offset.x, + is_previous_codepoint_prepend, + formatted_cps.?, + }); + } + + if (positions_differ) { + log.warn("position differs from advance: cluster={d} pos=({d},{d}) adv=({d},{d}) diff=({d},{d}) cps = {s}", .{ + cluster, + x_offset, + y_offset, + advance_x_offset, + advance_y_offset, + x_offset_diff, + y_offset_diff, + formatted_cps.?, + }); + } + + if (y_offset_differs) { + log.warn("run_offset.y differs from zero: cluster={d} pos=({d},{d}) run_offset=({d},{d}) cell_offset.x={d} cps = {s}", .{ + cluster, + x_offset, + y_offset, + run_offset.x, + run_offset.y, + cell_offset.x, + formatted_cps.?, + }); + } + + if (cluster_differs) { + log.warn("cell_offset.cluster differs from cluster (potential ligature detected) cell_offset.cluster={d} cluster={d} run_offset.cluster={d} diff={d} pos=({d},{d}) run_offset=({d},{d}) cell_offset.x={d} is_prev_prepend={} cps = {s}", .{ + cell_offset.cluster, + cluster, + run_offset.cluster, + @as(isize, @intCast(cluster)) - @as(isize, @intCast(cell_offset.cluster)), + x_offset, + y_offset, + run_offset.x, + run_offset.y, + cell_offset.x, + is_previous_codepoint_prepend, + formatted_cps.?, + }); + } + } +}; + +test "run iterator" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + { + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("ABCD"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |_| count += 1; + try testing.expectEqual(@as(usize, 1), count); + } + + // Spaces should be part of a run + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("ABCD EFG"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |_| count += 1; + try testing.expectEqual(@as(usize, 1), count); + } + + { + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("A😃D"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |_| { + count += 1; + + // All runs should be exactly length 1 + try testing.expectEqual(@as(u32, 1), shaper.hb_buf.getLength()); + } + try testing.expectEqual(@as(usize, 3), count); + } +} + +test "run iterator: empty cells with background set" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + { + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // Set red background and write A + s.nextSlice("\x1b[48;2;255;0;0mA"); + + // Get our first row + { + const list_cell = t.screens.active.pages.getCell(.{ .active = .{ .x = 1 } }).?; + const cell = list_cell.cell; + cell.* = .{ + .content_tag = .bg_color_rgb, + .content = .{ .color_rgb = .{ .r = 0xFF, .g = 0, .b = 0 } }, + }; + } + { + const list_cell = t.screens.active.pages.getCell(.{ .active = .{ .x = 2 } }).?; + const cell = list_cell.cell; + cell.* = .{ + .content_tag = .bg_color_rgb, + .content = .{ .color_rgb = .{ .r = 0xFF, .g = 0, .b = 0 } }, + }; + } + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + { + const run = (try it.next(alloc)).?; + try testing.expectEqual(@as(u32, 3), shaper.hb_buf.getLength()); + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 3), cells.len); + } + try testing.expect(try it.next(alloc) == null); + } +} + +test "shape" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain + buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain + buf_idx += try std.unicode.utf8Encode(0x1F3FD, buf[buf_idx..]); // Medium skin tone + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + try testing.expectEqual(@as(u32, 3), shaper.hb_buf.getLength()); + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape inconsolata ligs" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(">="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + try testing.expectEqual(@as(usize, 2), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("==="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + try testing.expectEqual(@as(usize, 3), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +test "shape monaspace ligs" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaperWithFont(alloc, .monaspace_neon); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("==="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + try testing.expectEqual(@as(usize, 3), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +// Ghostty doesn't currently support RTL and our renderers assume +// that cells are in strict LTR order. This means that we need to +// force RTL text to be LTR for rendering. This test ensures that +// we are correctly forcing RTL text to be LTR. +test "shape arabic forced LTR" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaperWithFont(alloc, .arabic); + defer testdata.deinit(); + + var t = try terminal.Terminal.init(alloc, .{ .cols = 120, .rows = 30 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(@embedFile("testdata/arabic.txt")); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + try testing.expectEqual(@as(usize, 25), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 25), cells.len); + + var x: u16 = cells[0].x; + for (cells[1..]) |cell| { + try testing.expectEqual(x + 1, cell.x); + x = cell.x; + } + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape emoji width" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 5, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("👍"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + try testing.expectEqual(@as(usize, 2), run.cells); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +test "shape emoji width long" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen and add a long emoji sequence to it. + var t = try terminal.Terminal.init( + alloc, + .{ .cols = 30, .rows = 3 }, + ); + defer t.deinit(alloc); + + var page = t.screens.active.pages.pages.first.?.data; + var row = page.getRow(1); + const cell = &row.cells.ptr(page.memory)[0]; + cell.* = .{ + .content_tag = .codepoint, + .content = .{ .codepoint = 0x1F9D4 }, // Person with beard + }; + var graphemes = [_]u21{ + 0x1F3FB, // Light skin tone (Fitz 1-2) + 0x200D, // ZWJ + 0x2642, // Male sign + 0xFE0F, // Emoji presentation selector + }; + try page.setGraphemes( + row, + cell, + graphemes[0..], + ); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(1).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + try testing.expectEqual(@as(u32, 4), shaper.hb_buf.getLength()); + + const cells = try shaper.shape(run); + + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape variation selector VS15" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x270C, buf[buf_idx..]); // Victory sign (default text) + buf_idx += try std.unicode.utf8Encode(0xFE0E, buf[buf_idx..]); // ZWJ to force text + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + try testing.expectEqual(@as(u32, 1), shaper.hb_buf.getLength()); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape variation selector VS16" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x270C, buf[buf_idx..]); // Victory sign (default text) + buf_idx += try std.unicode.utf8Encode(0xFE0F, buf[buf_idx..]); // ZWJ to force color + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + try testing.expectEqual(@as(u32, 1), shaper.hb_buf.getLength()); + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), cells.len); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape with empty cells in between" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init( + alloc, + .{ .cols = 30, .rows = 3 }, + ); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("A"); + s.nextSlice("\x1b[5C"); + s.nextSlice("B"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 1), count); + try testing.expectEqual(@as(usize, 7), cells.len); + } +} + +test "shape Combining characters" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode('n', buf[buf_idx..]); // Combining + buf_idx += try std.unicode.utf8Encode(0x0308, buf[buf_idx..]); // Combining + buf_idx += try std.unicode.utf8Encode(0x0308, buf[buf_idx..]); + buf_idx += try std.unicode.utf8Encode('a', buf[buf_idx..]); + + // Make a screen with some data + var t = try terminal.Terminal.init( + alloc, + .{ .cols = 30, .rows = 3 }, + ); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 4), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); + try testing.expectEqual(@as(u16, 1), cells[3].x); + } + try testing.expectEqual(@as(usize, 1), count); +} + +// This test exists because the string it uses causes HarfBuzz to output a +// non-monotonic run with our cluster level set to `characters`, which we need +// to handle by tracking the max cluster for the run. +test "shape Devanagari string" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports devanagari for this to work, if we can't + // find Arial Unicode MS, which is a system font on macOS, we just skip + // the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Arial Unicode MS", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Disable grapheme clustering + t.modes.set(.grapheme_cluster, false); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("अपार्टमेंट"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + + const run = try it.next(alloc); + try testing.expect(run != null); + const cells = try shaper.shape(run.?); + + try testing.expectEqual(@as(usize, 8), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 1), cells[1].x); + try testing.expectEqual(@as(u16, 2), cells[2].x); + try testing.expectEqual(@as(u16, 4), cells[3].x); + try testing.expectEqual(@as(u16, 4), cells[4].x); + try testing.expectEqual(@as(u16, 5), cells[5].x); + try testing.expectEqual(@as(u16, 5), cells[6].x); + try testing.expectEqual(@as(u16, 6), cells[7].x); + + try testing.expect(try it.next(alloc) == null); +} + +// This test fails on Linux if you have the "Noto Sans Tai Tham" font installed +// locally. Disabling this test until it can be fixed. +test "shape Tai Tham vowels (position differs from advance)" { + return error.SkipZigTest; + // // Note that while this test was necessary for CoreText, the old logic was + // // working for HarfBuzz. Still we keep it to ensure it has the correct + // // behavior. + // const testing = std.testing; + // const alloc = testing.allocator; + + // // We need a font that supports Tai Tham for this to work, if we can't find + // // Noto Sans Tai Tham, which is a system font on macOS, we just skip the + // // test. + // var testdata = testShaperWithDiscoveredFont( + // alloc, + // "Noto Sans Tai Tham", + // ) catch return error.SkipZigTest; + // defer testdata.deinit(); + + // var buf: [32]u8 = undefined; + // var buf_idx: usize = 0; + // buf_idx += try std.unicode.utf8Encode(0x1a2F, buf[buf_idx..]); // ᨯ + // buf_idx += try std.unicode.utf8Encode(0x1a70, buf[buf_idx..]); // ᩰ + + // // Make a screen with some data + // var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + // defer t.deinit(alloc); + + // // Enable grapheme clustering + // t.modes.set(.grapheme_cluster, true); + + // var s = t.vtStream(); + // defer s.deinit(); + // s.nextSlice(buf[0..buf_idx]); + + // var state: terminal.RenderState = .empty; + // defer state.deinit(alloc); + // try state.update(alloc, &t); + + // // Get our run iterator + // var shaper = &testdata.shaper; + // var it = shaper.runIterator(.{ + // .grid = testdata.grid, + // .cells = state.row_data.get(0).cells.slice(), + // }); + // var count: usize = 0; + // while (try it.next(alloc)) |run| { + // count += 1; + + // const cells = try shaper.shape(run); + // try testing.expectEqual(@as(usize, 2), cells.len); + // try testing.expectEqual(@as(u16, 0), cells[0].x); + // try testing.expectEqual(@as(u16, 0), cells[1].x); + + // // The first glyph renders in the next cell. We expect the x_offset + // // to equal the cell width. However, with FreeType the cell_width is + // // computed from ASCII glyphs, and Noto Sans Tai Tham only has the + // // space character in ASCII (with a 3px advance), so the cell_width + // // metric doesn't match the actual Tai Tham glyph positioning. + // const expected_x_offset: i16 = if (comptime font.options.backend.hasFreetype()) 7 else @intCast(run.grid.metrics.cell_width); + // try testing.expectEqual(expected_x_offset, cells[0].x_offset); + // try testing.expectEqual(@as(i16, 0), cells[1].x_offset); + // } + // try testing.expectEqual(@as(usize, 1), count); +} + +test "shape Tibetan characters" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that has multiple glyphs for this codepoint to reproduce + // the old broken behavior, and Noto Serif Tibetan is one of them. It's not + // a default Mac font, and if we can't find it we just skip the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Serif Tibetan", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x0f00, buf[buf_idx..]); // ༀ + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 2), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + + // The second glyph renders at the correct location + try testing.expect(cells[1].x_offset < 2); + } + try testing.expectEqual(@as(usize, 1), count); +} + +// This test fails on Linux if you have the "Noto Sans Tai Tham" font installed +// locally. Disabling this test until it can be fixed. +test "shape Tai Tham letters (run_offset.y differs from zero)" { + return error.SkipZigTest; + // const testing = std.testing; + // const alloc = testing.allocator; + + // // We need a font that supports Tai Tham for this to work, if we can't find + // // Noto Sans Tai Tham, which is a system font on macOS, we just skip the + // // test. + // var testdata = testShaperWithDiscoveredFont( + // alloc, + // "Noto Sans Tai Tham", + // ) catch return error.SkipZigTest; + // defer testdata.deinit(); + + // var buf: [32]u8 = undefined; + // var buf_idx: usize = 0; + + // // First grapheme cluster: + // buf_idx += try std.unicode.utf8Encode(0x1a49, buf[buf_idx..]); // HA + // buf_idx += try std.unicode.utf8Encode(0x1a60, buf[buf_idx..]); // SAKOT + // // Second grapheme cluster, combining with the first in a ligature: + // buf_idx += try std.unicode.utf8Encode(0x1a3f, buf[buf_idx..]); // YA + // buf_idx += try std.unicode.utf8Encode(0x1a69, buf[buf_idx..]); // U + + // // Make a screen with some data + // var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + // defer t.deinit(alloc); + + // // Enable grapheme clustering + // t.modes.set(.grapheme_cluster, true); + + // var s = t.vtStream(); + // defer s.deinit(); + // s.nextSlice(buf[0..buf_idx]); + + // var state: terminal.RenderState = .empty; + // defer state.deinit(alloc); + // try state.update(alloc, &t); + + // // Get our run iterator + // var shaper = &testdata.shaper; + // var it = shaper.runIterator(.{ + // .grid = testdata.grid, + // .cells = state.row_data.get(0).cells.slice(), + // }); + // var count: usize = 0; + // while (try it.next(alloc)) |run| { + // count += 1; + + // const cells = try shaper.shape(run); + // try testing.expectEqual(@as(usize, 3), cells.len); + // try testing.expectEqual(@as(u16, 0), cells[0].x); + // try testing.expectEqual(@as(u16, 0), cells[1].x); + // try testing.expectEqual(@as(u16, 0), cells[2].x); // U from second grapheme + + // // The U glyph renders at a y below zero + // try testing.expectEqual(@as(i16, -3), cells[2].y_offset); + // } + // try testing.expectEqual(@as(usize, 1), count); +} + +// This test fails on Linux if you have the "Noto Sans Javanese" font installed +// locally. Disabling this test until it can be fixed. +test "shape Javanese ligatures" { + return error.SkipZigTest; + // const testing = std.testing; + // const alloc = testing.allocator; + + // // We need a font that supports Javanese for this to work, if we can't find + // // Noto Sans Javanese Regular, which is a system font on macOS, we just + // // skip the test. + // var testdata = testShaperWithDiscoveredFont( + // alloc, + // "Noto Sans Javanese", + // ) catch return error.SkipZigTest; + // defer testdata.deinit(); + + // var buf: [32]u8 = undefined; + // var buf_idx: usize = 0; + + // // First grapheme cluster: + // buf_idx += try std.unicode.utf8Encode(0xa9a4, buf[buf_idx..]); // NA + // buf_idx += try std.unicode.utf8Encode(0xa9c0, buf[buf_idx..]); // PANGKON + // // Second grapheme cluster, combining with the first in a ligature: + // buf_idx += try std.unicode.utf8Encode(0xa9b2, buf[buf_idx..]); // HA + // buf_idx += try std.unicode.utf8Encode(0xa9b8, buf[buf_idx..]); // Vowel sign SUKU + + // // Make a screen with some data + // var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + // defer t.deinit(alloc); + + // // Enable grapheme clustering + // t.modes.set(.grapheme_cluster, true); + + // var s = t.vtStream(); + // defer s.deinit(); + // s.nextSlice(buf[0..buf_idx]); + + // var state: terminal.RenderState = .empty; + // defer state.deinit(alloc); + // try state.update(alloc, &t); + + // // Get our run iterator + // var shaper = &testdata.shaper; + // var it = shaper.runIterator(.{ + // .grid = testdata.grid, + // .cells = state.row_data.get(0).cells.slice(), + // }); + // var count: usize = 0; + // while (try it.next(alloc)) |run| { + // count += 1; + + // const cells = try shaper.shape(run); + // const cell_width = run.grid.metrics.cell_width; + // try testing.expectEqual(@as(usize, 3), cells.len); + // try testing.expectEqual(@as(u16, 0), cells[0].x); + // try testing.expectEqual(@as(u16, 0), cells[1].x); + // try testing.expectEqual(@as(u16, 0), cells[2].x); + + // // The vowel sign SUKU renders with correct x_offset + // try testing.expect(cells[2].x_offset > 3 * cell_width); + // } + // try testing.expectEqual(@as(usize, 1), count); +} + +test "shape Chakma vowel sign with ligature (vowel sign renders first)" { + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports Chakma for this to work, if we can't find + // Noto Sans Chakma Regular, which is a system font on macOS, we just skip + // the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Chakma", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x1111d, buf[buf_idx..]); // BAA + // Second grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x11116, buf[buf_idx..]); // TAA + buf_idx += try std.unicode.utf8Encode(0x11133, buf[buf_idx..]); // Virama + // Third grapheme cluster, combining with the second in a ligature: + buf_idx += try std.unicode.utf8Encode(0x11120, buf[buf_idx..]); // YYAA + buf_idx += try std.unicode.utf8Encode(0x1112c, buf[buf_idx..]); // Vowel Sign U + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 4), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + // See the giant "We need to reset the `cell_offset`" comment, but here + // we should technically have the rest of these be `x` of 1, but that + // would require going back in the stream to adjust past cells, and + // we don't take on that complexity. + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); + try testing.expectEqual(@as(u16, 0), cells[3].x); + + // The vowel sign U renders before the TAA: + try testing.expect(cells[1].x_offset < cells[2].x_offset); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape Bengali ligatures with out of order vowels" { + // Whereas this test in CoreText had everything shaping into one giant + // ligature, HarfBuzz splits it into a few clusters. It still looks okay + // (see #10332). + + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports Bengali for this to work, if we can't find + // Arial Unicode MS, which is a system font on macOS, we just skip the + // test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Arial Unicode MS", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x09b0, buf[buf_idx..]); // RA + buf_idx += try std.unicode.utf8Encode(0x09be, buf[buf_idx..]); // Vowel sign AA + // Second grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x09b7, buf[buf_idx..]); // SSA + buf_idx += try std.unicode.utf8Encode(0x09cd, buf[buf_idx..]); // Virama + // Third grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x099f, buf[buf_idx..]); // TTA + buf_idx += try std.unicode.utf8Encode(0x09cd, buf[buf_idx..]); // Virama + // Fourth grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x09b0, buf[buf_idx..]); // RA + buf_idx += try std.unicode.utf8Encode(0x09c7, buf[buf_idx..]); // Vowel sign E + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 8), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + + // Whereas CoreText puts everything all into the first cell (see the + // corresponding test), HarfBuzz splits into two clusters. + try testing.expectEqual(@as(u16, 2), cells[2].x); + try testing.expectEqual(@as(u16, 2), cells[3].x); + try testing.expectEqual(@as(u16, 2), cells[4].x); + try testing.expectEqual(@as(u16, 2), cells[5].x); + try testing.expectEqual(@as(u16, 2), cells[6].x); + try testing.expectEqual(@as(u16, 2), cells[7].x); + + // The vowel sign E renders before the SSA: + try testing.expect(cells[2].x_offset < cells[3].x_offset); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape box glyphs" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x2500, buf[buf_idx..]); // horiz line + buf_idx += try std.unicode.utf8Encode(0x2501, buf[buf_idx..]); // + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + try testing.expectEqual(@as(u32, 2), shaper.hb_buf.getLength()); + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 2), cells.len); + try testing.expectEqual(@as(u32, 0x2500), cells[0].glyph_index); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u32, 0x2501), cells[1].glyph_index); + try testing.expectEqual(@as(u16, 1), cells[1].x); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape selection boundary" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("a1b2c3d4e5"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Full line selection + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 0, 9 }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + // Offset x, goes to end of line selection + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 2, 9 }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + + // Offset x, starts at beginning of line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 0, 3 }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + + // Selection only subset of line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 1, 3 }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 3), count); + } + + // Selection only one character + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 1, 1 }, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 3), count); + } +} + +test "shape cursor boundary" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("a1b2c3d4e5"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // No cursor is full line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + { + // Cursor at index 0 is two runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 0, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + } + + { + // Cursor at index 1 is three runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 1, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 3), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + } + { + // Cursor at last col is two runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 9, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + } +} + +test "shape cursor boundary and colored emoji" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init( + alloc, + .{ .cols = 3, .rows = 10 }, + ); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("👍🏼"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // No cursor is full line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + // Cursor on emoji does not split it + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 0, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 1, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +test "shape cell attribute change" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Plain >= should shape into 1 run + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(">="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + // Bold vs regular should split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice(">"); + s.nextSlice("\x1b[1m"); + s.nextSlice("="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + + // Changing fg color should split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 + s.nextSlice("\x1b[38;2;1;2;3m"); + s.nextSlice(">"); + // RGB 3, 2, 1 + s.nextSlice("\x1b[38;2;3;2;1m"); + s.nextSlice("="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + + // Changing bg color should not split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 bg + s.nextSlice("\x1b[48;2;1;2;3m"); + s.nextSlice(">"); + // RGB 3, 2, 1 bg + s.nextSlice("\x1b[48;2;3;2;1m"); + s.nextSlice("="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + + // Same bg color should not split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 bg + s.nextSlice("\x1b[48;2;1;2;3m"); + s.nextSlice(">"); + s.nextSlice("="); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } +} + +const TestShaper = struct { + alloc: Allocator, + shaper: Shaper, + grid: *SharedGrid, + lib: Library, + + pub fn deinit(self: *TestShaper) void { + self.shaper.deinit(); + self.grid.deinit(self.alloc); + self.alloc.destroy(self.grid); + self.lib.deinit(); + } +}; + +const TestFont = enum { + inconsolata, + monaspace_neon, + arabic, +}; + +/// Helper to return a fully initialized shaper. +fn testShaper(alloc: Allocator) !TestShaper { + return try testShaperWithFont(alloc, .inconsolata); +} + +fn testShaperWithFont(alloc: Allocator, font_req: TestFont) !TestShaper { + const testEmoji = font.embedded.emoji; + const testEmojiText = font.embedded.emoji_text; + const testFont = switch (font_req) { + .inconsolata => font.embedded.inconsolata, + .monaspace_neon => font.embedded.monaspace_neon, + .arabic => font.embedded.arabic, + }; + + var lib = try Library.init(alloc); + errdefer lib.deinit(); + + var c = Collection.init(); + c.load_options = .{ .library = lib }; + + // Setup group + _ = try c.add(alloc, try .init( + lib, + testFont, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + + if (comptime !font.options.backend.hasCoretext()) { + // Coretext doesn't support Noto's format + _ = try c.add(alloc, try .init( + lib, + testEmoji, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + } else { + // On CoreText we want to load Apple Emoji, we should have it. + var disco = font.Discover.init(); + defer disco.deinit(); + var disco_it = try disco.discover(alloc, .{ + .family = "Apple Color Emoji", + .size = 12, + .monospace = false, + }); + defer disco_it.deinit(); + var face = (try disco_it.next()).?; + errdefer face.deinit(); + _ = try c.addDeferred(alloc, face, .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + } + _ = try c.add(alloc, try .init( + lib, + testEmojiText, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + + const grid_ptr = try alloc.create(SharedGrid); + errdefer alloc.destroy(grid_ptr); + grid_ptr.* = try .init(alloc, .{ .collection = c }); + errdefer grid_ptr.*.deinit(alloc); + + var shaper = try Shaper.init(alloc, .{ + // Some of our tests rely on dlig being enabled by default + .features = &.{"dlig"}, + }); + errdefer shaper.deinit(); + + return TestShaper{ + .alloc = alloc, + .shaper = shaper, + .grid = grid_ptr, + .lib = lib, + }; +} + +fn testShaperWithDiscoveredFont(alloc: Allocator, font_req: [:0]const u8) !TestShaper { + if (font.Discover == void) return error.SkipZigTest; + var lib = try Library.init(alloc); + errdefer lib.deinit(); + + var c = Collection.init(); + c.load_options = .{ .library = lib }; + + // Discover and add our font to the collection. + { + var disco = font.Discover.init(); + defer disco.deinit(); + var disco_it = try disco.discover(alloc, .{ + .family = font_req, + .size = 12, + .monospace = false, + }); + defer disco_it.deinit(); + var face: font.DeferredFace = (try disco_it.next()) orelse return error.FontNotFound; + errdefer face.deinit(); + + // Check which font was discovered - skip if it doesn't match the request + var name_buf: [256]u8 = undefined; + const face_name = face.name(&name_buf) catch "(unknown)"; + if (std.mem.indexOf(u8, face_name, font_req) == null) { + return error.SkipZigTest; + } + + _ = try c.add( + alloc, + try face.load(lib, .{ .size = .{ .points = 12 } }), + .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }, + ); + } + + const grid_ptr = try alloc.create(SharedGrid); + errdefer alloc.destroy(grid_ptr); + grid_ptr.* = try .init(alloc, .{ .collection = c }); + errdefer grid_ptr.*.deinit(alloc); + + var shaper = try Shaper.init(alloc, .{}); + errdefer shaper.deinit(); + + return TestShaper{ + .alloc = alloc, + .shaper = shaper, + .grid = grid_ptr, + .lib = lib, + }; +} diff --git a/vendor/libghostty-vt/src/font/shaper/noop.zig b/vendor/libghostty-vt/src/font/shaper/noop.zig new file mode 100644 index 00000000..e5a08653 --- /dev/null +++ b/vendor/libghostty-vt/src/font/shaper/noop.zig @@ -0,0 +1,143 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const font = @import("../main.zig"); +const Face = font.Face; +const Collection = font.Collection; +const DeferredFace = font.DeferredFace; +const Group = font.Group; +const GroupCache = font.GroupCache; +const Library = font.Library; +const SharedGrid = font.SharedGrid; +const Style = font.Style; +const Presentation = font.Presentation; +const terminal = @import("../../terminal/main.zig"); + +const log = std.log.scoped(.font_shaper); + +/// Shaper that doesn't do any shaping. Each individual codepoint is mapped +/// directly to the detected text run font's glyph index. +pub const Shaper = struct { + /// The allocated used for the feature list and cell buf. + alloc: Allocator, + + /// The string used for shaping the current run. + run_state: RunState, + + /// The shared memory used for shaping results. + cell_buf: CellBuf, + + const CellBuf = std.ArrayListUnmanaged(font.shape.Cell); + const CodepointList = std.ArrayListUnmanaged(Codepoint); + const Codepoint = struct { + codepoint: u32, + cluster: u32, + }; + + const RunState = struct { + codepoints: CodepointList = .{}, + + fn deinit(self: *RunState, alloc: Allocator) void { + self.codepoints.deinit(alloc); + } + + fn reset(self: *RunState) !void { + self.codepoints.clearRetainingCapacity(); + } + }; + + /// The cell_buf argument is the buffer to use for storing shaped results. + /// This should be at least the number of columns in the terminal. + pub fn init(alloc: Allocator, opts: font.shape.Options) !Shaper { + _ = opts; + + return Shaper{ + .alloc = alloc, + .cell_buf = .{}, + .run_state = .{}, + }; + } + + pub fn deinit(self: *Shaper) void { + self.cell_buf.deinit(self.alloc); + self.run_state.deinit(self.alloc); + } + + pub fn endFrame(self: *const Shaper) void { + _ = self; + } + + pub fn runIterator( + self: *Shaper, + opts: font.shape.RunOptions, + ) font.shape.RunIterator { + return .{ + .hooks = .{ .shaper = self }, + .opts = opts, + }; + } + + pub fn shape(self: *Shaper, run: font.shape.TextRun) ![]const font.shape.Cell { + const state = &self.run_state; + + // Special fonts aren't shaped and their codepoint == glyph so we + // can just return the codepoints as-is. + if (run.font_index.special() != null) { + self.cell_buf.clearRetainingCapacity(); + try self.cell_buf.ensureTotalCapacity(self.alloc, state.codepoints.items.len); + for (state.codepoints.items) |entry| { + self.cell_buf.appendAssumeCapacity(.{ + .x = @intCast(entry.cluster), + .glyph_index = @intCast(entry.codepoint), + }); + } + + return self.cell_buf.items; + } + + // Go through the run and map each codepoint to a glyph index. + self.cell_buf.clearRetainingCapacity(); + + // Note: this is digging into some internal details, we should maybe + // expose a public API for this. + const face = try run.grid.resolver.collection.getFace(run.font_index); + for (state.codepoints.items) |entry| { + const glyph_index = face.glyphIndex(entry.codepoint) orelse { + // The run iterator shared logic should guarantee that + // there is a glyph index for all codepoints in the run. + // This is not well tested because we don't use the noop + // shaper in any release builds. + unreachable; + }; + try self.cell_buf.append(self.alloc, .{ + .x = @intCast(entry.cluster), + .glyph_index = glyph_index, + }); + } + + return self.cell_buf.items; + } + + /// The hooks for RunIterator. + pub const RunIteratorHook = struct { + shaper: *Shaper, + + pub fn prepare(self: *RunIteratorHook) !void { + try self.shaper.run_state.reset(); + } + + pub fn addCodepoint(self: RunIteratorHook, cp: u32, cluster: u32) !void { + try self.shaper.run_state.codepoints.append(self.shaper.alloc, .{ + .codepoint = cp, + .cluster = cluster, + }); + } + + pub fn finalize(self: RunIteratorHook) !void { + _ = self; + } + }; +}; + +test { + @import("std").testing.refAllDecls(@This()); +} diff --git a/vendor/libghostty-vt/src/font/shaper/run.zig b/vendor/libghostty-vt/src/font/shaper/run.zig new file mode 100644 index 00000000..45c5d38c --- /dev/null +++ b/vendor/libghostty-vt/src/font/shaper/run.zig @@ -0,0 +1,409 @@ +const std = @import("std"); +const assert = @import("../../quirks.zig").inlineAssert; +const Allocator = std.mem.Allocator; +const font = @import("../main.zig"); +const shape = @import("../shape.zig"); +const terminal = @import("../../terminal/main.zig"); +const autoHash = std.hash.autoHash; +const Hasher = std.hash.Wyhash; + +/// A single text run. A text run is only valid for one Shaper instance and +/// until the next run is created. A text run never goes across multiple +/// rows in a terminal, so it is guaranteed to always be one line. +pub const TextRun = struct { + /// A unique hash for this run. This can be used to cache the shaping + /// results. We don't provide a means to compare actual values if the + /// hash is the same, so we should continue to improve this hash to + /// lower the chance of hash collisions if they become a problem. If + /// there are hash collisions, it would result in rendering issues but + /// the core data would be correct. + /// + /// The hash is position-independent within the row by using relative + /// cluster positions. This allows identical runs in different positions + /// to share the same cache entry, improving cache efficiency. + hash: u64, + + /// The offset in the row where this run started. This is added to the + /// X position of the final shaped cells to get the absolute position + /// in the row where they belong. + offset: u16, + + /// The total number of cells produced by this run. + cells: u16, + + /// The font grid that built this run. + grid: *font.SharedGrid, + + /// The font index to use for the glyphs of this run. + font_index: font.Collection.Index, +}; + +/// RunIterator is an iterator that yields text runs. +pub const RunIterator = struct { + hooks: font.Shaper.RunIteratorHook, + opts: shape.RunOptions, + i: usize = 0, + + pub fn next(self: *RunIterator, alloc: Allocator) !?TextRun { + const slice = &self.opts.cells; + const cells: []const terminal.page.Cell = slice.items(.raw); + const graphemes: []const []const u21 = slice.items(.grapheme); + const styles: []const terminal.Style = slice.items(.style); + + // Trim the right side of a row that might be empty + const max: usize = max: { + for (0..cells.len) |i| { + const rev_i = cells.len - i - 1; + if (!cells[rev_i].isEmpty()) break :max rev_i + 1; + } + + break :max 0; + }; + + // Invisible cells don't have any glyphs rendered, + // so we explicitly skip them in the shaping process. + while (self.i < max and + (cells[self.i].hasStyling() and + styles[self.i].flags.invisible)) self.i += 1; + + // We're over at the max + if (self.i >= max) return null; + + // Track the font for our current run + var current_font: font.Collection.Index = .{}; + + // Allow the hook to prepare + self.hooks.prepare(); + + // Initialize our hash for this run. + var hasher = Hasher.init(0); + + // Let's get our style that we'll expect for the run. + const style: terminal.Style = if (cells[self.i].hasStyling()) styles[self.i] else .{}; + + // Go through cell by cell and accumulate while we build our run. + var j: usize = self.i; + while (j < max) : (j += 1) { + // Use relative cluster positions (offset from run start) to make + // the shaping cache position-independent. This ensures that runs + // with identical content but different starting positions in the + // row produce the same hash, enabling cache reuse. + const cluster = j - self.i; + const cell: *const terminal.page.Cell = &cells[j]; + + // If we have a selection and we're at a boundary point, then + // we break the run here. + if (self.opts.selection) |bounds| { + if (j > self.i) { + if (bounds[0] > 0 and j == bounds[0]) break; + if (bounds[1] > 0 and j == bounds[1] + 1) break; + } + } + + // If we're a spacer, then we ignore it + switch (cell.wide) { + .narrow, .wide => {}, + .spacer_head, .spacer_tail => continue, + } + + // If our cell attributes are changing, then we split the run. + // This prevents a single glyph for ">=" to be rendered with + // one color when the two components have different styling. + if (j > self.i) style: { + const prev_cell = cells[j - 1]; + + // If the prev cell and this cell are both plain + // codepoints then we check if they are commonly "bad" + // ligatures and spit the run if they are. + if (prev_cell.content_tag == .codepoint and + cell.content_tag == .codepoint) + { + const prev_cp = prev_cell.codepoint(); + switch (prev_cp) { + // fl, fi + 'f' => { + const cp = cell.codepoint(); + if (cp == 'l' or cp == 'i') break; + }, + + // st + 's' => { + const cp = cell.codepoint(); + if (cp == 't') break; + }, + + else => {}, + } + } + + // If the style is exactly the change then fast path out. + if (prev_cell.style_id == cell.style_id) break :style; + + // The style is different. We allow differing background + // styles but any other change results in a new run. + const c1 = comparableStyle(style); + const c2 = comparableStyle(if (cell.hasStyling()) styles[j] else .{}); + if (!c1.eql(c2)) break; + } + + // Text runs break when font styles change so we need to get + // the proper style. + const font_style: font.Style = style: { + if (style.flags.bold) { + if (style.flags.italic) break :style .bold_italic; + break :style .bold; + } + + if (style.flags.italic) break :style .italic; + break :style .regular; + }; + + // Determine the presentation format for this glyph. + const presentation: ?font.Presentation = if (cell.hasGrapheme()) p: { + // We only check the FIRST codepoint because I believe the + // presentation format must be directly adjacent to the codepoint. + const cps = graphemes[j]; + assert(cps.len > 0); + if (cps[0] == 0xFE0E) break :p .text; + if (cps[0] == 0xFE0F) break :p .emoji; + break :p null; + } else emoji: { + // If we're not a grapheme, our individual char could be + // an emoji so we want to check if we expect emoji presentation. + // The font grid indexForCodepoint we use below will do this + // automatically. + break :emoji null; + }; + + // If our cursor is on this line then we break the run around the + // cursor. This means that any row with a cursor has at least + // three breaks: before, exactly the cursor, and after. + // + // We do not break a cell that is exactly the grapheme. If there + // are cells following that contain joiners, we allow those to + // break. This creates an effect where hovering over an emoji + // such as a skin-tone emoji is fine, but hovering over the + // joiners will show the joiners allowing you to modify the + // emoji. + if (!cell.hasGrapheme()) { + if (self.opts.cursor_x) |cursor_x| { + // Exactly: self.i is the cursor and we iterated once. This + // means that we started exactly at the cursor and did at + // exactly one iteration. Why exactly one? Because we may + // start at our cursor but do many if our cursor is exactly + // on an emoji. + if (self.i == cursor_x and j == self.i + 1) break; + + // Before: up to and not including the cursor. This means + // that we started before the cursor (self.i < cursor_x) + // and j is now at the cursor meaning we haven't yet processed + // the cursor. + if (self.i < cursor_x and j == cursor_x) { + assert(j > 0); + break; + } + + // After: after the cursor. We don't need to do anything + // special, we just let the run complete. + } + } + + // We need to find a font that supports this character. If + // there are additional zero-width codepoints (to form a single + // grapheme, i.e. combining characters), we need to find a font + // that supports all of them. + const font_info: struct { + idx: font.Collection.Index, + fallback: ?u32 = null, + } = font_info: { + // If we find a font that supports this entire grapheme + // then we use that. + if (try self.indexForCell( + alloc, + cell, + graphemes[j], + font_style, + presentation, + )) |idx| break :font_info .{ .idx = idx }; + + // Otherwise we need a fallback character. Prefer the + // official replacement character. + if (try self.opts.grid.getIndex( + alloc, + 0xFFFD, // replacement char + font_style, + presentation, + )) |idx| break :font_info .{ .idx = idx, .fallback = 0xFFFD }; + + // Fallback to space + if (try self.opts.grid.getIndex( + alloc, + ' ', + font_style, + presentation, + )) |idx| break :font_info .{ .idx = idx, .fallback = ' ' }; + + // We can't render at all. This is a bug, we should always + // have a font that can render a space. + unreachable; + }; + + //log.warn("char={x} info={}", .{ cell.char, font_info }); + if (j == self.i) current_font = font_info.idx; + + // If our fonts are not equal, then we're done with our run. + if (font_info.idx != current_font) break; + + // If we're a fallback character, add that and continue; we + // don't want to add the entire grapheme. + if (font_info.fallback) |cp| { + try self.addCodepoint(&hasher, cp, @intCast(cluster)); + continue; + } + + // If we're a Kitty unicode placeholder then we add a blank. + if (cell.codepoint() == terminal.kitty.graphics.unicode.placeholder) { + try self.addCodepoint(&hasher, ' ', @intCast(cluster)); + continue; + } + + // Add all the codepoints for our grapheme + try self.addCodepoint( + &hasher, + if (cell.codepoint() == 0) ' ' else cell.codepoint(), + @intCast(cluster), + ); + if (cell.hasGrapheme()) { + for (graphemes[j]) |cp| { + // Do not send presentation modifiers + if (cp == 0xFE0E or cp == 0xFE0F) continue; + try self.addCodepoint(&hasher, cp, @intCast(cluster)); + } + } + } + + // Finalize our buffer + self.hooks.finalize(); + + // Add our length to the hash as an additional mechanism to avoid collisions + autoHash(&hasher, j - self.i); + + // Add our font index + autoHash(&hasher, current_font); + + // Move our cursor. Must defer since we use self.i below. + defer self.i = j; + + return .{ + .hash = hasher.final(), + .offset = @intCast(self.i), + .cells = @intCast(j - self.i), + .grid = self.opts.grid, + .font_index = current_font, + }; + } + + fn addCodepoint(self: *RunIterator, hasher: anytype, cp: u32, cluster: u32) !void { + autoHash(hasher, cp); + autoHash(hasher, cluster); + try self.hooks.addCodepoint(cp, cluster); + } + + /// Find a font index that supports the grapheme for the given cell, + /// or null if no such font exists. + /// + /// This is used to find a font that supports the entire grapheme. + /// We look for fonts that support each individual codepoint and then + /// find the common font amongst all candidates. + fn indexForCell( + self: *RunIterator, + alloc: Allocator, + cell: *const terminal.Cell, + graphemes: []const u21, + style: font.Style, + presentation: ?font.Presentation, + ) !?font.Collection.Index { + if (cell.isEmpty() or + cell.codepoint() == 0 or + cell.codepoint() == terminal.kitty.graphics.unicode.placeholder) + { + return try self.opts.grid.getIndex( + alloc, + ' ', + style, + presentation, + ); + } + + // Get the font index for the primary codepoint. + const primary_cp: u32 = cell.codepoint(); + const primary = try self.opts.grid.getIndex( + alloc, + primary_cp, + style, + presentation, + ) orelse return null; + + // Easy, and common: we aren't a multi-codepoint grapheme, so + // we just return whatever index for the cell codepoint. + if (!cell.hasGrapheme()) return primary; + + // If this is a grapheme, we need to find a font that supports + // all of the codepoints in the grapheme. + var candidates: std.ArrayList(font.Collection.Index) = try .initCapacity( + alloc, + graphemes.len + 1, + ); + defer candidates.deinit(alloc); + candidates.appendAssumeCapacity(primary); + + for (graphemes) |cp| { + // Ignore Emoji ZWJs + if (cp == 0xFE0E or cp == 0xFE0F or cp == 0x200D) continue; + + // Find a font that supports this codepoint. If none support this + // then the whole grapheme can't be rendered so we return null. + // + // We explicitly do not require the additional grapheme components + // to support the base presentation, since it is common for emoji + // fonts to support the base emoji with emoji presentation but not + // certain ZWJ-combined characters like the male and female signs. + const idx = try self.opts.grid.getIndex( + alloc, + cp, + style, + null, + ) orelse return null; + candidates.appendAssumeCapacity(idx); + } + + // We need to find a candidate that has ALL of our codepoints + for (candidates.items) |idx| { + if (!self.opts.grid.hasCodepoint(idx, primary_cp, presentation)) continue; + for (graphemes) |cp| { + // Ignore Emoji ZWJs + if (cp == 0xFE0E or cp == 0xFE0F or cp == 0x200D) continue; + if (!self.opts.grid.hasCodepoint(idx, cp, null)) break; + } else { + // If the while completed, then we have a candidate that + // supports all of our codepoints. + return idx; + } + } + + return null; + } +}; + +/// Returns a style that when compared must be identical for a run to +/// continue. +fn comparableStyle(style: terminal.Style) terminal.Style { + var s = style; + + // We allow background colors to differ because we'll just paint the + // cell background whatever the style is, and wherever the glyph + // lands on top of it will be the color of the glyph. + s.bg_color = .none; + + return s; +} diff --git a/vendor/libghostty-vt/src/font/shaper/testdata/arabic.txt b/vendor/libghostty-vt/src/font/shaper/testdata/arabic.txt new file mode 100644 index 00000000..d450c762 --- /dev/null +++ b/vendor/libghostty-vt/src/font/shaper/testdata/arabic.txt @@ -0,0 +1,3 @@ +غريبه لاني عربي أبا عن جد +واتكلم الانجليزية بطلاقة اكثر من ٢٥ سنه +ومع هذا اجد العربيه افضل لان فيها الكثير من المفردات الاكثر دقه بالوصف diff --git a/vendor/libghostty-vt/src/font/shaper/web_canvas.zig b/vendor/libghostty-vt/src/font/shaper/web_canvas.zig new file mode 100644 index 00000000..d16bd968 --- /dev/null +++ b/vendor/libghostty-vt/src/font/shaper/web_canvas.zig @@ -0,0 +1,305 @@ +const std = @import("std"); +const assert = @import("../../quirks.zig").inlineAssert; +const Allocator = std.mem.Allocator; +const font = @import("../main.zig"); +const terminal = @import("../../terminal/main.zig"); +const unicode = @import("../../unicode/main.zig"); +const uucode = @import("uucode"); + +const log = std.log.scoped(.font_shaper); + +pub const Shaper = struct { + const RunBuf = std.MultiArrayList(struct { + /// The codepoint for this cell. This must be used in conjunction + /// with cluster to find the total set of codepoints for a given + /// cell. See cluster for more information. + codepoint: u32, + + /// Cluster is set to the X value of the cell that this codepoint + /// is part of. Note that a cell can have multiple codepoints + /// with zero-width joiners (ZWJ) and such. Note that terminals + /// do NOT handle full extended grapheme clustering well so it + /// is possible a single grapheme extends multiple clusters. + /// For example, skin tone emoji thumbs up may show up as two + /// clusters: one with thumbs up and the ZWJ, and a second + /// cluster with the tone block. It is up to the shaper to handle + /// shaping these together into a single glyph, if it wishes. + cluster: u32, + }); + + /// The allocator used for run_buf. + alloc: Allocator, + + /// The shared memory used for shaping results. + cell_buf: []font.shape.Cell, + + /// The shared memory used for storing information about a run. + run_buf: RunBuf, + + /// The cell_buf argument is the buffer to use for storing shaped results. + /// This should be at least the number of columns in the terminal. + pub fn init(alloc: Allocator, opts: font.shape.Options) !Shaper { + // Note: we do not support opts.font_features + + return Shaper{ + .alloc = alloc, + .cell_buf = opts.cell_buf, + .run_buf = .{}, + }; + } + + pub fn deinit(self: *Shaper) void { + self.run_buf.deinit(self.alloc); + self.* = undefined; + } + + pub fn endFrame(self: *const Shaper) void { + _ = self; + } + + /// Returns an iterator that returns one text run at a time for the + /// given terminal row. Note that text runs are only valid one at a time + /// for a Shaper struct since they share state. + pub fn runIterator( + self: *Shaper, + opts: font.shape.RunOptions, + ) font.shape.RunIterator { + return .{ + .hooks = .{ .shaper = self }, + .opts = opts, + }; + } + + /// Shape the given text run. The text run must be the immediately + /// previous text run that was iterated since the text run does share + /// state with the Shaper struct. + /// + /// The return value is only valid until the next shape call is called. + /// + /// If there is not enough space in the cell buffer, an error is + /// returned. + pub fn shape(self: *Shaper, run: font.shape.TextRun) ![]font.shape.Cell { + // TODO: memory check that cell_buf can fit results + + const codepoints = self.run_buf.items(.codepoint); + const clusters = self.run_buf.items(.cluster); + assert(codepoints.len == clusters.len); + + switch (codepoints.len) { + // Special cases: if we have no codepoints (is this possible?) + // then our result is also an empty cell run. + 0 => return self.cell_buf[0..0], + + // If we have only 1 codepoint, then we assume that it is + // a single grapheme and just let it through. At this point, + // we can't have any more information to do anything else. + 1 => { + self.cell_buf[0] = .{ + .x = @intCast(clusters[0]), + .glyph_index = codepoints[0], + }; + + return self.cell_buf[0..1]; + }, + + else => {}, + } + + // We know we have at least two codepoints, so we now go through + // each and perform grapheme clustering. + // + // Note that due to limitations of canvas, we can NOT support + // font ligatures. However, we do support grapheme clustering. + // This means we can render things like skin tone emoji but + // we can't render things like single glyph "=>". + var break_state: uucode.grapheme.BreakState = .default; + var cp1: u21 = @intCast(codepoints[0]); + + var start: usize = 0; + var i: usize = 1; + var cur: usize = 0; + while (i <= codepoints.len) : (i += 1) { + // We loop to codepoints.len so that we can handle the end + // case. In the end case, we always assume it is a grapheme + // break. This isn't strictly true but its how terminals + // work today. + const grapheme_break = i == codepoints.len or blk: { + const cp2: u21 = @intCast(codepoints[i]); + defer cp1 = cp2; + + break :blk unicode.graphemeBreak( + cp1, + cp2, + &break_state, + ); + }; + + // If this is NOT a grapheme break, cp2 is part of a single + // grapheme cluster and we expect there could be more. We + // move on to the next codepoint to try again. + if (!grapheme_break) continue; + + // This IS a grapheme break, meaning that cp2 is NOT part + // of cp1. So we need to render the prior grapheme. + const len = i - start; + assert(len > 0); + switch (len) { + // If we have only a single codepoint then just render it + // as-is. + 1 => self.cell_buf[cur] = .{ + .x = @intCast(clusters[start]), + .glyph_index = codepoints[start], + }, + + // We must have multiple codepoints (see assert above). In + // this case we UTF-8 encode the codepoints and send them + // to the face to reserve a private glyph index. + else => { + // UTF-8 encode the codepoints in this cluster. + const cluster = cluster: { + const cluster_points = codepoints[start..i]; + assert(cluster_points.len == len); + + const buf_len = buf_len: { + var acc: usize = 0; + for (cluster_points) |cp| { + acc += try std.unicode.utf8CodepointSequenceLength( + @intCast(cp), + ); + } + + break :buf_len acc; + }; + + var buf = try self.alloc.alloc(u8, buf_len); + errdefer self.alloc.free(buf); + var buf_i: usize = 0; + for (cluster_points) |cp| { + buf_i += try std.unicode.utf8Encode( + @intCast(cp), + buf[buf_i..], + ); + } + + break :cluster buf; + }; + defer self.alloc.free(cluster); + + var face = try run.group.group.faceFromIndex(run.font_index); + const index = try face.graphemeGlyphIndex(cluster); + + self.cell_buf[cur] = .{ + .x = @intCast(clusters[start]), + .glyph_index = index, + }; + }, + } + + start = i; + cur += 1; + } + + return self.cell_buf[0..cur]; + } + + /// The hooks for RunIterator. + pub const RunIteratorHook = struct { + shaper: *Shaper, + + pub fn prepare(self: RunIteratorHook) !void { + // Reset the buffer for our current run + self.shaper.run_buf.shrinkRetainingCapacity(0); + } + + pub fn addCodepoint( + self: RunIteratorHook, + cp: u32, + cluster: u32, + ) !void { + try self.shaper.run_buf.append(self.shaper.alloc, .{ + .codepoint = cp, + .cluster = cluster, + }); + } + + pub fn finalize(self: RunIteratorHook) !void { + _ = self; + } + }; +}; + +/// The wasm-compatible API. +pub const Wasm = struct { + const wasm = @import("../../os/wasm.zig"); + const alloc = wasm.alloc; + + export fn shaper_new(cap: usize) ?*Shaper { + return shaper_new_(cap) catch null; + } + + fn shaper_new_(cap: usize) !*Shaper { + const cell_buf = try alloc.alloc(font.shape.Cell, cap); + errdefer alloc.free(cell_buf); + + var shaper = try Shaper.init(alloc, .{ .cell_buf = cell_buf }); + errdefer shaper.deinit(); + + const result = try alloc.create(Shaper); + errdefer alloc.destroy(result); + result.* = shaper; + return result; + } + + export fn shaper_free(ptr: ?*Shaper) void { + if (ptr) |v| { + alloc.free(v.cell_buf); + v.deinit(); + alloc.destroy(v); + } + } + + /// Runs a test to verify shaping works properly. + export fn shaper_test( + self: *Shaper, + group: *font.GroupCache, + str: [*]const u8, + len: usize, + ) void { + shaper_test_(self, group, str[0..len]) catch |err| { + log.warn("error during shaper test err={}", .{err}); + }; + } + + fn shaper_test_(self: *Shaper, group: *font.GroupCache, str: []const u8) !void { + // Create a terminal and print all our characters into it. + var term = try terminal.Terminal.init(alloc, self.cell_buf.len, 80); + defer term.deinit(alloc); + + // Iterate over unicode codepoints and add to terminal + { + const view = try std.unicode.Utf8View.init(str); + var iter = view.iterator(); + while (iter.nextCodepoint()) |c| { + try term.print(c); + } + } + + // Iterate over the rows and print out all the runs we get. + var rowIter = term.screen.rowIterator(.viewport); + var y: usize = 0; + while (rowIter.next()) |row| { + defer y += 1; + + var iter = self.runIterator(group, row, null, null); + while (try iter.next(alloc)) |run| { + const cells = try self.shape(run); + log.info("y={} run={d} shape={any} idx={}", .{ + y, + run.cells, + cells, + run.font_index, + }); + } + } + } +}; |
