chore: update vendored libghostty-vt

refs #276

2 files changed, 215 insertions, 40 deletions

diff --git a/vendor/libghostty-vt/src/simd/vt.cpp b/vendor/libghostty-vt/src/simd/vt.cpp
index 5bf4147d..15d46966 100644
--- a/vendor/libghostty-vt/src/simd/vt.cpp
+++ b/vendor/libghostty-vt/src/simd/vt.cpp
@@ -198,35 +198,55 @@ size_t DecodeUTF8(const uint8_t* HWY_RESTRICT input,
   return static_cast<size_t>(out - output);
 }
 
-/// Decode the UTF-8 text in input into output until an escape
-/// character is found. This returns the number of bytes consumed
-/// from input and writes the number of decoded characters into
-/// output_count.
-///
-/// This may return a value less than count even with no escape
-/// character if the input ends with an incomplete UTF-8 sequence.
-/// The caller should check the next byte manually to determine
-/// if it is incomplete.
+// Widen the N uint8 lanes of v into N uint32 values stored at out.
+// This is the UTF-8 to UTF-32 "decode" for ASCII bytes.
 template <class D>
-size_t DecodeUTF8UntilControlSeqImpl(D d,
-                                     const T* HWY_RESTRICT input,
-                                     size_t count,
-                                     char32_t* output,
-                                     size_t* output_count) {
-  const size_t N = hn::Lanes(d);
+static HWY_INLINE void WidenAsciiStore(D d,
+                                       hn::Vec<D> v,
+                                       char32_t* HWY_RESTRICT out) {
+  uint32_t* HWY_RESTRICT out32 = reinterpret_cast<uint32_t*>(out);
+#if HWY_TARGET == HWY_SCALAR
+  // The scalar fallback target has single-lane vectors, which cannot
+  // be halved; widen the one lane directly.
+  (void)d;
+  out32[0] = hn::GetLane(v);
+#else
+  const hn::Half<D> dh;
+  const hn::Half<hn::Half<D>> dq;
+  const hn::Rebind<uint32_t, decltype(dq)> d32;
+  const size_t N4 = hn::Lanes(dq);
+  const auto lo = hn::LowerHalf(dh, v);
+  const auto hi = hn::UpperHalf(dh, v);
+  hn::StoreU(hn::PromoteTo(d32, hn::LowerHalf(dq, lo)), d32, out32 + 0 * N4);
+  hn::StoreU(hn::PromoteTo(d32, hn::UpperHalf(dq, lo)), d32, out32 + 1 * N4);
+  hn::StoreU(hn::PromoteTo(d32, hn::LowerHalf(dq, hi)), d32, out32 + 2 * N4);
+  hn::StoreU(hn::PromoteTo(d32, hn::UpperHalf(dq, hi)), d32, out32 + 3 * N4);
+#endif
+}
 
-  // Create a vector containing ESC since that denotes a control sequence.
+// The general (non-ASCII) portion of DecodeUTF8UntilControlSeqImpl.
+// Continues scanning for ESC starting at byte offset `base` and decodes
+// input[base..stop) via simdutf. The caller must have already decoded
+// input[0..base) as ASCII into output[0..base) (one codepoint per byte).
+template <class D>
+static HWY_NOINLINE size_t DecodeNonAsciiUntilControlSeq(
+    D d,
+    const T* HWY_RESTRICT input,
+    size_t count,
+    size_t base,
+    char32_t* output,
+    size_t* output_count) {
+  const size_t N = hn::Lanes(d);
   const hn::Vec<D> esc_vec = Set(d, 0x1B);
 
   // Compare N elements at a time.
-  size_t i = 0;
+  size_t i = base;
   for (; i + N <= count; i += N) {
     // Load the N elements from our input into a vector.
     const hn::Vec<D> input_vec = hn::LoadU(d, input + i);
 
     // If we don't have any escapes we keep going. We want to accumulate
     // the largest possible valid UTF-8 sequence before decoding.
-    // TODO(mitchellh): benchmark this vs decoding every time
     const size_t esc_idx = IndexOfChunk(d, esc_vec, input_vec);
     if (esc_idx == kNotFound) {
       continue;
@@ -235,25 +255,17 @@ size_t DecodeUTF8UntilControlSeqImpl(D d,
     // We have an ESC char, decode up to this point. We start by assuming
     // a valid UTF-8 sequence and slow-path into error handling if we find
     // an invalid sequence.
-    *output_count = DecodeUTF8(input, i + esc_idx, output);
+    *output_count = base + DecodeUTF8(input + base, i + esc_idx - base,
+                                      output + base);
     return i + esc_idx;
   }
 
-  // If we have leftover input then we decode it one byte at a time (slow!)
+  // If we have leftover input then we scan it one byte at a time (slow!)
   // using pretty much the same logic as above.
-  if (i != count) {
-    const hn::CappedTag<T, 1> d1;
-    using D1 = decltype(d1);
-    const hn::Vec<D1> esc1 = Set(d1, hn::GetLane(esc_vec));
-    for (; i < count; ++i) {
-      const hn::Vec<D1> input_vec = hn::LoadU(d1, input + i);
-      const size_t esc_idx = IndexOfChunk(d1, esc1, input_vec);
-      if (esc_idx == kNotFound) {
-        continue;
-      }
-
-      *output_count = DecodeUTF8(input, i + esc_idx, output);
-      return i + esc_idx;
+  for (; i < count; ++i) {
+    if (input[i] == 0x1B) {
+      *output_count = base + DecodeUTF8(input + base, i - base, output + base);
+      return i;
     }
   }
 
@@ -267,9 +279,86 @@ size_t DecodeUTF8UntilControlSeqImpl(D d,
   // like C0, C1, F5-FF should NOT be trimmed — they should be passed
   // through to DecodeUTF8 which will replace them with U+FFFD per the
   // maximal subpart algorithm.
-  const size_t trimmed_len = TrimValidPartialUTF8(input, i);
-  *output_count = DecodeUTF8(input, trimmed_len, output);
-  return trimmed_len;
+  const size_t trimmed_len = TrimValidPartialUTF8(input + base, count - base);
+  *output_count = base + DecodeUTF8(input + base, trimmed_len, output + base);
+  return base + trimmed_len;
+}
+
+/// Decode the UTF-8 text in input into output until an escape
+/// character is found. This returns the number of bytes consumed
+/// from input and writes the number of decoded characters into
+/// output_count.
+///
+/// This may return a value less than count even with no escape
+/// character if the input ends with an incomplete UTF-8 sequence.
+/// The caller should check the next byte manually to determine
+/// if it is incomplete.
+template <class D>
+size_t DecodeUTF8UntilControlSeqImpl(D d,
+                                     const T* HWY_RESTRICT input,
+                                     size_t count,
+                                     char32_t* output,
+                                     size_t* output_count) {
+  const size_t N = hn::Lanes(d);
+
+  // Create a vector containing ESC since that denotes a control sequence.
+  const hn::Vec<D> esc_vec = Set(d, 0x1B);
+  // Any byte >= 0x80 is part of a multi-byte UTF-8 sequence.
+  const hn::Vec<D> high_vec = Set(d, 0x80);
+
+  // ASCII fast path: terminal input is overwhelmingly ASCII, for which
+  // UTF-8 decoding is a simple widening of each byte to 32 bits. We
+  // fuse the ESC scan with the decode, one chunk at a time, and only
+  // fall back to the full UTF-8 decoder (simdutf) when we encounter a
+  // non-ASCII byte. This avoids a second pass over the input and, for
+  // the common short runs between escape sequences, avoids the fixed
+  // overhead of the general-purpose decoder.
+  size_t i = 0;
+  for (; i + N <= count; i += N) {
+    const hn::Vec<D> input_vec = hn::LoadU(d, input + i);
+
+    // Find the first byte that stops the ASCII fast path: an ESC or
+    // any non-ASCII byte.
+    const hn::Mask<D> stop_mask =
+        hn::Or(hn::Eq(input_vec, esc_vec), hn::Ge(input_vec, high_vec));
+    const intptr_t stop = hn::FindFirstTrue(d, stop_mask);
+
+    // Widen the whole chunk unconditionally: output is guaranteed to
+    // be at least as large as input, and if we stop mid-chunk only
+    // the prefix is reported (the rest is scratch that the caller
+    // never reads).
+    WidenAsciiStore(d, input_vec, output + i);
+    if (stop < 0) continue;
+
+    const size_t stop_idx = i + static_cast<size_t>(stop);
+    if (input[stop_idx] == 0x1B) {
+      // ESC: everything before it was ASCII, one codepoint per byte.
+      *output_count = stop_idx;
+      return stop_idx;
+    }
+
+    // Non-ASCII: decode the rest (up to an ESC) with the full decoder.
+    return DecodeNonAsciiUntilControlSeq(d, input, count, stop_idx, output,
+                                         output_count);
+  }
+
+  // Leftover input (< N bytes): process one byte at a time.
+  for (; i < count; ++i) {
+    const T b = input[i];
+    if (b == 0x1B) {
+      *output_count = i;
+      return i;
+    }
+    if (b >= 0x80) {
+      return DecodeNonAsciiUntilControlSeq(d, input, count, i, output,
+                                           output_count);
+    }
+    output[i] = b;
+  }
+
+  // The entire input was ASCII (no ESC, no partial sequences possible).
+  *output_count = count;
+  return count;
 }
 
 size_t DecodeUTF8UntilControlSeq(const uint8_t* HWY_RESTRICT input,
diff --git a/vendor/libghostty-vt/src/simd/vt.zig b/vendor/libghostty-vt/src/simd/vt.zig
index 4230665f..807285f7 100644
--- a/vendor/libghostty-vt/src/simd/vt.zig
+++ b/vendor/libghostty-vt/src/simd/vt.zig
@@ -80,13 +80,18 @@ fn utf8DecodeUntilControlSeqScalar(
         var valid: usize = 1; // lead byte is valid
         for (0..seq.len - 1) |ci| {
             if (decode_offset + valid >= decode.len) {
-                // Truncated at end of buffer: treat as incomplete
-                // input that may be completed later. Stop decoding
-                // without consuming these bytes.
-                return .{
+                // The sequence is cut off by the end of the decode
+                // region. If the region ends at the true end of the
+                // input then it may be completed by future input, so
+                // stop without consuming these bytes. If the region
+                // was bounded by an ESC then the sequence can never
+                // be completed; the valid-so-far prefix is a maximal
+                // subpart which maps to a single U+FFFD below.
+                if (decode.len == input.len) return .{
                     .consumed = decode_offset,
                     .decoded = decode_count,
                 };
+                break;
             }
             const cb = decode[decode_offset + valid];
             if (cb < seq.ranges[ci][0] or cb > seq.ranges[ci][1]) {
@@ -148,6 +153,58 @@ fn utf8SeqInfo(lead: u8) Utf8SeqInfo {
     };
 }
 
+// Differential test: the SIMD implementation must agree with the
+// scalar implementation on any input. Exercises random mixtures of
+// ASCII, escapes, controls, valid and invalid UTF-8, at various
+// lengths (including chunk-boundary straddling cases).
+test "decode simd matches scalar" {
+    if (comptime !options.simd) return error.SkipZigTest;
+
+    const testing = std.testing;
+    var prng = std.Random.DefaultPrng.init(0xf00dface);
+    const rand = prng.random();
+
+    var input: [257]u8 = undefined;
+    var out_simd: [input.len]u32 = undefined;
+    var out_scalar: [input.len]u32 = undefined;
+
+    for (0..10_000) |_| {
+        const len = rand.intRangeAtMost(usize, 0, input.len);
+        const style = rand.intRangeAtMost(u8, 0, 2);
+        for (input[0..len]) |*b| {
+            b.* = switch (style) {
+                // Mostly ASCII with occasional specials.
+                0 => switch (rand.intRangeAtMost(u8, 0, 20)) {
+                    0 => 0x1B,
+                    1 => rand.intRangeAtMost(u8, 0, 0x1F),
+                    2 => rand.int(u8),
+                    else => rand.intRangeAtMost(u8, 0x20, 0x7E),
+                },
+                // Heavy multi-byte/invalid UTF-8.
+                1 => switch (rand.intRangeAtMost(u8, 0, 3)) {
+                    0 => rand.intRangeAtMost(u8, 0x80, 0xBF),
+                    1 => rand.intRangeAtMost(u8, 0xC0, 0xFF),
+                    2 => 0x1B,
+                    else => rand.intRangeAtMost(u8, 0x20, 0x7E),
+                },
+                // Fully random bytes.
+                else => rand.int(u8),
+            };
+        }
+
+        const res_simd = utf8DecodeUntilControlSeq(input[0..len], &out_simd);
+        const res_scalar = utf8DecodeUntilControlSeqScalar(input[0..len], &out_scalar);
+        errdefer std.debug.print("input={x}\n", .{input[0..len]});
+        try testing.expectEqual(res_scalar.consumed, res_simd.consumed);
+        try testing.expectEqual(res_scalar.decoded, res_simd.decoded);
+        try testing.expectEqualSlices(
+            u32,
+            out_scalar[0..res_scalar.decoded],
+            out_simd[0..res_simd.decoded],
+        );
+    }
+}
+
 test "decode no escape" {
     const testing = std.testing;
 
@@ -399,6 +456,35 @@ test "decode valid multibyte surrounded by invalid" {
     }
 }
 
+test "decode partial UTF-8 before escape" {
+    const testing = std.testing;
+
+    // A valid-so-far but incomplete sequence cut off by an ESC can
+    // never be completed, so it is consumed and replaced by a single
+    // U+FFFD (maximal subpart) rather than left pending. Only
+    // sequences cut off by the true end of input are left pending.
+    var output: [64]u32 = undefined;
+
+    // 2-byte lead cut off by ESC.
+    {
+        const str = "hi\xc2\x1b[0m";
+        const result = utf8DecodeUntilControlSeq(str, &output);
+        try testing.expectEqual(@as(usize, 3), result.consumed);
+        try testing.expectEqual(@as(usize, 3), result.decoded);
+        try testing.expectEqual(@as(u32, 0xFFFD), output[2]);
+    }
+
+    // 3-byte lead plus one valid continuation cut off by ESC:
+    // the whole prefix is one maximal subpart, one U+FFFD.
+    {
+        const str = "\xe0\xa0\x1bX";
+        const result = utf8DecodeUntilControlSeq(str, &output);
+        try testing.expectEqual(@as(usize, 2), result.consumed);
+        try testing.expectEqual(@as(usize, 1), result.decoded);
+        try testing.expectEqual(@as(u32, 0xFFFD), output[0]);
+    }
+}
+
 test "decode invalid byte before escape" {
     const testing = std.testing;