aboutsummaryrefslogtreecommitdiff
path: root/vendor/libghostty-vt/src/simd
diff options
context:
space:
mode:
authorOgulcan Celik <ogulcancelik@gmail.com>2026-05-07 18:29:21 +0300
committerOgulcan Celik <ogulcancelik@gmail.com>2026-05-07 18:29:21 +0300
commitdcfe11a7588fcd42471159453810abc9b7720a57 (patch)
treec32a77266028e73ca2382b96c96f8438a5fcd621 /vendor/libghostty-vt/src/simd
parent2dfed860103c4e4505ac7f25def9fe37da1e1e6a (diff)
build: update libghostty-vt vendor
vendor ghostty 063ac3e and remove the musl libc++ link workaround. expose the new selection formatter and hyperlink uri apis for follow-up rendering work. add manual artifact validation for static linux builds.
Diffstat (limited to 'vendor/libghostty-vt/src/simd')
-rw-r--r--vendor/libghostty-vt/src/simd/codepoint_width.cpp113
-rw-r--r--vendor/libghostty-vt/src/simd/index_of.cpp2
-rw-r--r--vendor/libghostty-vt/src/simd/index_of.h24
-rw-r--r--vendor/libghostty-vt/src/simd/vt.cpp213
-rw-r--r--vendor/libghostty-vt/src/simd/vt.zig373
5 files changed, 600 insertions, 125 deletions
diff --git a/vendor/libghostty-vt/src/simd/codepoint_width.cpp b/vendor/libghostty-vt/src/simd/codepoint_width.cpp
index 294922c6..7f0dfd87 100644
--- a/vendor/libghostty-vt/src/simd/codepoint_width.cpp
+++ b/vendor/libghostty-vt/src/simd/codepoint_width.cpp
@@ -3,10 +3,41 @@
#define HWY_TARGET_INCLUDE "simd/codepoint_width.cpp" // this file
#include <hwy/foreach_target.h> // must come before highway.h
#include <hwy/highway.h>
-#include <hwy/print-inl.h>
-#include <cassert>
-#include <iterator>
+#ifndef GHOSTTY_SIMD_CPW_HELPERS_
+#define GHOSTTY_SIMD_CPW_HELPERS_
+
+#ifdef NDEBUG
+#define GHOSTTY_SIMD_ASSERT(cond) ((void)0)
+#else
+#define GHOSTTY_SIMD_ASSERT(cond) \
+ do { \
+ if (!(cond)) __builtin_trap();\
+ } while (0)
+#endif
+
+// Replacement for std::size() that works without libc++.
+template <typename T, size_t N>
+constexpr size_t array_size(const T (&)[N]) { return N; }
+
+// Constexpr min/max element over a C array (replaces std::min_element/
+// std::max_element).
+template <typename T, size_t N>
+constexpr T array_min(const T (&a)[N]) {
+ T m = a[0];
+ for (size_t i = 1; i < N; ++i)
+ if (a[i] != 0 && (m == 0 || a[i] < m)) m = a[i];
+ return m;
+}
+template <typename T, size_t N>
+constexpr T array_max(const T (&a)[N]) {
+ T m = a[0];
+ for (size_t i = 1; i < N; ++i)
+ if (a[i] > m) m = a[i];
+ return m;
+}
+
+#endif // GHOSTTY_SIMD_CPW_HELPERS_
HWY_BEFORE_NAMESPACE();
namespace ghostty {
@@ -214,18 +245,18 @@ HWY_ALIGN constexpr uint16_t nsm_lte16[] = {
};
// All our tables must be identically sized
-static_assert(std::size(eaw_gte32) == std::size(eaw_lte32));
-static_assert(std::size(eaw_gte16) == std::size(eaw_lte16));
-static_assert(std::size(zero_gte32) == std::size(zero_lte32));
-static_assert(std::size(zero_gte16) == std::size(zero_lte16));
-static_assert(std::size(nsm_gte32) == std::size(nsm_lte32));
-static_assert(std::size(nsm_gte16) == std::size(nsm_lte16));
+static_assert(array_size(eaw_gte32) == array_size(eaw_lte32));
+static_assert(array_size(eaw_gte16) == array_size(eaw_lte16));
+static_assert(array_size(zero_gte32) == array_size(zero_lte32));
+static_assert(array_size(zero_gte16) == array_size(zero_lte16));
+static_assert(array_size(nsm_gte32) == array_size(nsm_lte32));
+static_assert(array_size(nsm_gte16) == array_size(nsm_lte16));
/// Handles 16-bit codepoints.
template <class D, typename T = uint16_t>
int8_t CodepointWidth16(D d, uint16_t input) {
- assert(input > 0xFF);
- assert(input <= 0xFFFF);
+ GHOSTTY_SIMD_ASSERT(input > 0xFF);
+ GHOSTTY_SIMD_ASSERT(input <= 0xFFFF);
const size_t N = hn::Lanes(d);
const hn::Vec<D> input_vec = Set(d, input);
@@ -245,10 +276,10 @@ int8_t CodepointWidth16(D d, uint16_t input) {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
- static_assert(std::size(gte_keys) == std::size(lte_keys));
- static_assert(std::size(gte_keys) >= 32);
+ static_assert(array_size(gte_keys) == array_size(lte_keys));
+ static_assert(array_size(gte_keys) >= 32);
size_t i = 0;
- for (; i + N <= std::size(lte_keys) && lte_keys[i] != 0; i += N) {
+ for (; i + N <= array_size(lte_keys) && lte_keys[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, lte_keys + i);
const hn::Vec<D> gte_vec = hn::Load(d, gte_keys + i);
const intptr_t idx = hn::FindFirstTrue(
@@ -262,17 +293,17 @@ int8_t CodepointWidth16(D d, uint16_t input) {
return 2;
}
}
- assert(i >= 7); // We should have checked all the ranges.
+ GHOSTTY_SIMD_ASSERT(i >= 7); // We should have checked all the ranges.
}
{
constexpr T zero_gte_min =
- *std::min_element(zero_gte16, zero_gte16 + std::size(zero_gte16));
+ array_min(zero_gte16);
constexpr T zero_lte_max =
- *std::max_element(zero_lte16, zero_lte16 + std::size(zero_lte16));
+ array_max(zero_lte16);
if (input >= zero_gte_min && input <= zero_lte_max) {
size_t i = 0;
- for (; i + N <= std::size(zero_gte16) && zero_gte16[i] != 0; i += N) {
+ for (; i + N <= array_size(zero_gte16) && zero_gte16[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, zero_lte16 + i);
const hn::Vec<D> gte_vec = hn::Load(d, zero_gte16 + i);
const intptr_t idx = hn::FindFirstTrue(
@@ -286,12 +317,12 @@ int8_t CodepointWidth16(D d, uint16_t input) {
{
constexpr T eaw_gte_min =
- *std::min_element(eaw_gte16, eaw_gte16 + std::size(eaw_gte16));
+ array_min(eaw_gte16);
constexpr T eaw_lte_max =
- *std::max_element(eaw_lte16, eaw_lte16 + std::size(eaw_lte16));
+ array_max(eaw_lte16);
if (input >= eaw_gte_min && input <= eaw_lte_max) {
size_t i = 0;
- for (; i + N <= std::size(eaw_lte16) && eaw_lte16[i] != 0; i += N) {
+ for (; i + N <= array_size(eaw_lte16) && eaw_lte16[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, eaw_lte16 + i);
const hn::Vec<D> gte_vec = hn::Load(d, eaw_gte16 + i);
const intptr_t idx = hn::FindFirstTrue(
@@ -305,12 +336,12 @@ int8_t CodepointWidth16(D d, uint16_t input) {
{
constexpr T nsm_gte_min =
- *std::min_element(nsm_gte16, nsm_gte16 + std::size(nsm_gte16));
+ array_min(nsm_gte16);
constexpr T nsm_lte_max =
- *std::max_element(nsm_lte16, nsm_lte16 + std::size(nsm_lte16));
+ array_max(nsm_lte16);
if (input >= nsm_gte_min && input <= nsm_lte_max) {
size_t i = 0;
- for (; i + N <= std::size(nsm_lte16) && nsm_lte16[i] != 0; i += N) {
+ for (; i + N <= array_size(nsm_lte16) && nsm_lte16[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, nsm_lte16 + i);
const hn::Vec<D> gte_vec = hn::Load(d, nsm_gte16 + i);
const intptr_t idx = hn::FindFirstTrue(
@@ -328,7 +359,7 @@ int8_t CodepointWidth16(D d, uint16_t input) {
/// Handles codepoints larger than 16-bit.
template <class D, typename T = uint32_t>
int8_t CodepointWidth32(D d, T input) {
- assert(input > 0xFFFF);
+ GHOSTTY_SIMD_ASSERT(input > 0xFFFF);
const size_t N = hn::Lanes(d);
const hn::Vec<D> input_vec = Set(d, input);
@@ -342,10 +373,10 @@ int8_t CodepointWidth32(D d, T input) {
HWY_ALIGN constexpr T lte_keys[] = {
0x1f1ff, 0x2FFFD, 0x3FFFD, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
- static_assert(std::size(gte_keys) == std::size(lte_keys));
- static_assert(std::size(gte_keys) >= 16);
+ static_assert(array_size(gte_keys) == array_size(lte_keys));
+ static_assert(array_size(gte_keys) >= 16);
size_t i = 0;
- for (; i + N <= std::size(lte_keys) && lte_keys[i] != 0; i += N) {
+ for (; i + N <= array_size(lte_keys) && lte_keys[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, lte_keys + i);
const hn::Vec<D> gte_vec = hn::Load(d, gte_keys + i);
const intptr_t idx = hn::FindFirstTrue(
@@ -354,7 +385,7 @@ int8_t CodepointWidth32(D d, T input) {
return 2;
}
}
- assert(i >= 2); // We should have checked all the ranges.
+ GHOSTTY_SIMD_ASSERT(i >= 2); // We should have checked all the ranges.
}
{
@@ -364,10 +395,10 @@ int8_t CodepointWidth32(D d, T input) {
HWY_ALIGN constexpr T lte_keys[] = {
0xE0FFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
- static_assert(std::size(gte_keys) == std::size(lte_keys));
- static_assert(std::size(gte_keys) >= 16);
+ static_assert(array_size(gte_keys) == array_size(lte_keys));
+ static_assert(array_size(gte_keys) >= 16);
size_t i = 0;
- for (; i + N <= std::size(lte_keys) && lte_keys[i] != 0; i += N) {
+ for (; i + N <= array_size(lte_keys) && lte_keys[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, lte_keys + i);
const hn::Vec<D> gte_vec = hn::Load(d, gte_keys + i);
const intptr_t idx = hn::FindFirstTrue(
@@ -380,12 +411,12 @@ int8_t CodepointWidth32(D d, T input) {
{
constexpr T zero_gte_min =
- *std::min_element(zero_gte32, zero_gte32 + std::size(zero_gte32));
+ array_min(zero_gte32);
constexpr T zero_lte_max =
- *std::max_element(zero_lte32, zero_lte32 + std::size(zero_lte32));
+ array_max(zero_lte32);
if (input >= zero_gte_min && input <= zero_lte_max) {
size_t i = 0;
- for (; i + N <= std::size(zero_gte32) && zero_gte32[i] != 0; i += N) {
+ for (; i + N <= array_size(zero_gte32) && zero_gte32[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, zero_lte32 + i);
const hn::Vec<D> gte_vec = hn::Load(d, zero_gte32 + i);
const intptr_t idx = hn::FindFirstTrue(
@@ -399,12 +430,12 @@ int8_t CodepointWidth32(D d, T input) {
{
constexpr T eaw_gte_min =
- *std::min_element(eaw_gte32, eaw_gte32 + std::size(eaw_gte32));
+ array_min(eaw_gte32);
constexpr T eaw_lte_max =
- *std::max_element(eaw_lte32, eaw_lte32 + std::size(eaw_lte32));
+ array_max(eaw_lte32);
if (input >= eaw_gte_min && input <= eaw_lte_max) {
size_t i = 0;
- for (; i + N <= std::size(eaw_lte32) && eaw_lte32[i] != 0; i += N) {
+ for (; i + N <= array_size(eaw_lte32) && eaw_lte32[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, eaw_lte32 + i);
const hn::Vec<D> gte_vec = hn::Load(d, eaw_gte32 + i);
const intptr_t idx = hn::FindFirstTrue(
@@ -418,12 +449,12 @@ int8_t CodepointWidth32(D d, T input) {
{
constexpr T nsm_gte_min =
- *std::min_element(nsm_gte32, nsm_gte32 + std::size(nsm_gte32));
+ array_min(nsm_gte32);
constexpr T nsm_lte_max =
- *std::max_element(nsm_lte32, nsm_lte32 + std::size(nsm_lte32));
+ array_max(nsm_lte32);
if (input >= nsm_gte_min && input <= nsm_lte_max) {
size_t i = 0;
- for (; i + N <= std::size(nsm_lte32) && nsm_lte32[i] != 0; i += N) {
+ for (; i + N <= array_size(nsm_lte32) && nsm_lte32[i] != 0; i += N) {
const hn::Vec<D> lte_vec = hn::Load(d, nsm_lte32 + i);
const hn::Vec<D> gte_vec = hn::Load(d, nsm_gte32 + i);
const intptr_t idx = hn::FindFirstTrue(
diff --git a/vendor/libghostty-vt/src/simd/index_of.cpp b/vendor/libghostty-vt/src/simd/index_of.cpp
index b070ae3b..8dad4bc4 100644
--- a/vendor/libghostty-vt/src/simd/index_of.cpp
+++ b/vendor/libghostty-vt/src/simd/index_of.cpp
@@ -6,8 +6,6 @@
#include <simd/index_of.h>
-#include <optional>
-
HWY_BEFORE_NAMESPACE();
namespace ghostty {
namespace HWY_NAMESPACE {
diff --git a/vendor/libghostty-vt/src/simd/index_of.h b/vendor/libghostty-vt/src/simd/index_of.h
index 531af9f8..172c9357 100644
--- a/vendor/libghostty-vt/src/simd/index_of.h
+++ b/vendor/libghostty-vt/src/simd/index_of.h
@@ -7,8 +7,7 @@
#include <hwy/highway.h>
-#include <cstddef>
-#include <optional>
+#include <stddef.h>
HWY_BEFORE_NAMESPACE();
namespace ghostty {
@@ -16,12 +15,16 @@ namespace HWY_NAMESPACE {
namespace hn = hwy::HWY_NAMESPACE;
+// Sentinel value returned by IndexOfChunk when no match is found.
+static constexpr size_t kNotFound = static_cast<size_t>(-1);
+
// Return the index of the first occurrence of `needle` in `input`, where
-// the input and needle are already loaded into vectors.
+// the input and needle are already loaded into vectors. Returns kNotFound
+// if no match is found.
template <class D, typename T = hn::TFromD<D>>
-std::optional<size_t> IndexOfChunk(D d,
- hn::Vec<D> needle_vec,
- hn::Vec<D> input_vec) {
+size_t IndexOfChunk(D d,
+ hn::Vec<D> needle_vec,
+ hn::Vec<D> input_vec) {
// Compare the input vector with the needle vector. This produces
// a vector where each lane is 0xFF if the corresponding lane in
// `input_vec` is equal to the corresponding lane in `needle_vec`.
@@ -32,9 +35,9 @@ std::optional<size_t> IndexOfChunk(D d,
// If we found a match, return the index into the input.
if (pos >= 0) {
- return std::optional<size_t>(static_cast<size_t>(pos));
+ return static_cast<size_t>(pos);
} else {
- return std::nullopt;
+ return kNotFound;
}
}
@@ -58,8 +61,9 @@ size_t IndexOfImpl(D d, T needle, const T* HWY_RESTRICT input, size_t count) {
for (; i + N <= count; i += N) {
// Load the N elements from our input into a vector and check the chunk.
const hn::Vec<D> input_vec = hn::LoadU(d, input + i);
- if (auto pos = IndexOfChunk(d, needle_vec, input_vec)) {
- return i + pos.value();
+ const size_t pos = IndexOfChunk(d, needle_vec, input_vec);
+ if (pos != kNotFound) {
+ return i + pos;
}
}
diff --git a/vendor/libghostty-vt/src/simd/vt.cpp b/vendor/libghostty-vt/src/simd/vt.cpp
index d8133abf..5bf4147d 100644
--- a/vendor/libghostty-vt/src/simd/vt.cpp
+++ b/vendor/libghostty-vt/src/simd/vt.cpp
@@ -5,8 +5,6 @@
#include <hwy/highway.h>
#include <simdutf.h>
-#include <utf8.h>
-#include <vector>
#include <simd/index_of.h>
#include <simd/vt.h>
@@ -19,12 +17,144 @@ namespace hn = hwy::HWY_NAMESPACE;
using T = uint8_t;
+// Compute the length of the maximal subpart of an ill-formed UTF-8
+// subsequence starting at p[0], per Unicode Table 3-7 and the W3C
+// "U+FFFD Substitution of Maximal Subparts" algorithm.
+//
+// The maximal subpart is the longest initial subsequence that is either:
+// (a) the start of a well-formed sequence, or
+// (b) a single byte.
+// Each maximal subpart maps to exactly one U+FFFD.
+static size_t MaximalSubpart(const unsigned char* p, size_t len) {
+ if (len == 0) return 0;
+
+ unsigned char b0 = p[0];
+
+ // Continuation bytes (80-BF), overlong leads (C0-C1), or invalid (F5-FF):
+ // each is its own maximal subpart of length 1.
+ if (b0 < 0xC2 || b0 > 0xF4) return 1;
+
+ // Determine the expected sequence length and the valid range for each
+ // continuation byte per Unicode Table 3-7.
+ size_t seq_len;
+ unsigned char lo[3], hi[3];
+
+ if (b0 <= 0xDF) {
+ seq_len = 2;
+ lo[0] = 0x80; hi[0] = 0xBF;
+ } else if (b0 == 0xE0) {
+ seq_len = 3;
+ lo[0] = 0xA0; hi[0] = 0xBF;
+ lo[1] = 0x80; hi[1] = 0xBF;
+ } else if (b0 <= 0xEC) {
+ seq_len = 3;
+ lo[0] = 0x80; hi[0] = 0xBF;
+ lo[1] = 0x80; hi[1] = 0xBF;
+ } else if (b0 == 0xED) {
+ seq_len = 3;
+ lo[0] = 0x80; hi[0] = 0x9F;
+ lo[1] = 0x80; hi[1] = 0xBF;
+ } else if (b0 <= 0xEF) {
+ seq_len = 3;
+ lo[0] = 0x80; hi[0] = 0xBF;
+ lo[1] = 0x80; hi[1] = 0xBF;
+ } else if (b0 == 0xF0) {
+ seq_len = 4;
+ lo[0] = 0x90; hi[0] = 0xBF;
+ lo[1] = 0x80; hi[1] = 0xBF;
+ lo[2] = 0x80; hi[2] = 0xBF;
+ } else if (b0 <= 0xF3) {
+ seq_len = 4;
+ lo[0] = 0x80; hi[0] = 0xBF;
+ lo[1] = 0x80; hi[1] = 0xBF;
+ lo[2] = 0x80; hi[2] = 0xBF;
+ } else { // b0 == 0xF4
+ seq_len = 4;
+ lo[0] = 0x80; hi[0] = 0x8F;
+ lo[1] = 0x80; hi[1] = 0xBF;
+ lo[2] = 0x80; hi[2] = 0xBF;
+ }
+
+ // Check continuation bytes against their specific valid ranges.
+ // The maximal subpart extends as far as bytes match.
+ size_t valid = 1; // lead byte counts
+ for (size_t i = 0; i < seq_len - 1 && valid < len; i++) {
+ unsigned char cb = p[valid];
+ if (cb < lo[i] || cb > hi[i]) break;
+ valid++;
+ }
+
+ // If we matched all bytes, the sequence is structurally valid
+ // (shouldn't happen since we're called on an error), but cap
+ // to avoid skipping a valid sequence.
+ if (valid == seq_len) return valid;
+
+ return valid;
+}
+
+// Trim trailing bytes that form a valid-but-incomplete UTF-8 sequence.
+// Only trims sequences whose bytes so far match Table 3-7 ranges (i.e.,
+// truly partial sequences that could be completed by future input).
+// Invalid lead bytes (C0, C1, F5-FF) or mismatched continuations are NOT
+// trimmed — they will be handled as errors by DecodeUTF8.
+static size_t TrimValidPartialUTF8(const uint8_t* input, size_t len) {
+ if (len == 0) return 0;
+
+ // Find the start of a potential trailing partial sequence by scanning
+ // backwards from the end. We look for a lead byte (C2-F4) that could
+ // start a multi-byte sequence, possibly followed by continuation bytes.
+ //
+ // We check up to the last 4 bytes (max UTF-8 sequence length).
+ size_t check_start = len > 4 ? len - 4 : 0;
+ for (size_t pos = len; pos > check_start; pos--) {
+ unsigned char b = input[pos - 1];
+
+ // Skip continuation bytes — they might belong to the partial sequence.
+ if ((b & 0xC0) == 0x80) continue;
+
+ // Found a non-continuation byte. Only valid multi-byte leads (C2-F4)
+ // can start a partial sequence worth trimming. Anything else (ASCII,
+ // C0, C1, F5-FF) should be consumed by DecodeUTF8.
+ if (b < 0xC2 || b > 0xF4) return len;
+
+ // Determine expected sequence length from the lead byte.
+ size_t expected;
+ if (b <= 0xDF)
+ expected = 2;
+ else if (b <= 0xEF)
+ expected = 3;
+ else
+ expected = 4;
+
+ size_t seq_remaining = len - (pos - 1);
+
+ // If we have all expected bytes, the sequence is complete (not partial).
+ if (seq_remaining >= expected) return len;
+
+ // Check if the trailing bytes form a valid prefix using MaximalSubpart.
+ const unsigned char* seq_start = input + pos - 1;
+ size_t subpart = MaximalSubpart(seq_start, seq_remaining);
+
+ // Only trim if ALL trailing bytes are part of the valid prefix
+ // (the sequence is valid-so-far but incomplete).
+ if (subpart == seq_remaining) {
+ return pos - 1;
+ }
+
+ // The sequence is ill-formed, don't trim — let DecodeUTF8 handle it.
+ return len;
+ }
+
+ return len;
+}
+
// Decode the UTF-8 text in input into output. Returns the number of decoded
// characters. This function assumes output is large enough.
//
// This function handles malformed UTF-8 sequences by inserting a
-// replacement character (U+FFFD) and continuing to decode. This function
-// will consume the entire input no matter what.
+// replacement character (U+FFFD) following the W3C/Unicode "U+FFFD
+// Substitution of Maximal Subparts" algorithm and continuing to decode.
+// This function will consume the entire input no matter what.
size_t DecodeUTF8(const uint8_t* HWY_RESTRICT input,
size_t count,
char32_t* output) {
@@ -34,27 +164,38 @@ size_t DecodeUTF8(const uint8_t* HWY_RESTRICT input,
return 0;
}
- // Assume no errors for fast path.
- const size_t decoded = simdutf::convert_utf8_to_utf32(
- reinterpret_cast<const char*>(input), count, output);
- if (decoded > 0) {
- return decoded;
+ // Decode UTF-8 to UTF-32, replacing invalid sequences with U+FFFD.
+ const char* in = reinterpret_cast<const char*>(input);
+ size_t remaining = count;
+ char32_t* out = output;
+ while (remaining > 0) {
+ auto r = simdutf::convert_utf8_to_utf32_with_errors(in, remaining, out);
+
+ // If the decode was a full success then we're done!
+ if (r.error == simdutf::SUCCESS) {
+ out += r.count;
+ break;
+ }
+
+ // On error, r.count is the input byte position of the error.
+ // The output buffer is already written up to that point, but
+ // we need count_utf8 to find how many char32_t that produced.
+ out += simdutf::count_utf8(in, r.count);
+
+ // Compute the maximal subpart at the error position and emit
+ // a single U+FFFD for it.
+ const unsigned char* err_pos =
+ reinterpret_cast<const unsigned char*>(in + r.count);
+ size_t err_remaining = remaining - r.count;
+ size_t skip = r.count + MaximalSubpart(err_pos, err_remaining);
+
+ *out++ = 0xFFFD;
+
+ in += skip;
+ remaining -= skip;
}
- // Errors in the UTF input, take a slow path and do a decode with
- // replacement (with U+FFFD). Note that simdutf doesn't have a
- // decode with replacement API:
- // https://github.com/simdutf/simdutf/issues/147
- //
- // Because of this, we use a separate library with heap allocation
- // that is much, much slower (the allocation is slower, the algorithm
- // is slower, etc.) This is just so we have something that works.
- // I want to replace this.
- std::vector<char> replacement_result;
- utf8::replace_invalid(input, input + count,
- std::back_inserter(replacement_result), 0xFFFD);
- return DecodeUTF8(reinterpret_cast<const uint8_t*>(replacement_result.data()),
- replacement_result.size(), output);
+ return static_cast<size_t>(out - output);
}
/// Decode the UTF-8 text in input into output until an escape
@@ -86,16 +227,16 @@ size_t DecodeUTF8UntilControlSeqImpl(D d,
// If we don't have any escapes we keep going. We want to accumulate
// the largest possible valid UTF-8 sequence before decoding.
// TODO(mitchellh): benchmark this vs decoding every time
- const auto esc_idx = IndexOfChunk(d, esc_vec, input_vec);
- if (!esc_idx) {
+ const size_t esc_idx = IndexOfChunk(d, esc_vec, input_vec);
+ if (esc_idx == kNotFound) {
continue;
}
// We have an ESC char, decode up to this point. We start by assuming
// a valid UTF-8 sequence and slow-path into error handling if we find
// an invalid sequence.
- *output_count = DecodeUTF8(input, i + esc_idx.value(), output);
- return i + esc_idx.value();
+ *output_count = DecodeUTF8(input, i + esc_idx, output);
+ return i + esc_idx;
}
// If we have leftover input then we decode it one byte at a time (slow!)
@@ -106,21 +247,27 @@ size_t DecodeUTF8UntilControlSeqImpl(D d,
const hn::Vec<D1> esc1 = Set(d1, hn::GetLane(esc_vec));
for (; i < count; ++i) {
const hn::Vec<D1> input_vec = hn::LoadU(d1, input + i);
- const auto esc_idx = IndexOfChunk(d1, esc1, input_vec);
- if (!esc_idx) {
+ const size_t esc_idx = IndexOfChunk(d1, esc1, input_vec);
+ if (esc_idx == kNotFound) {
continue;
}
- *output_count = DecodeUTF8(input, i + esc_idx.value(), output);
- return i + esc_idx.value();
+ *output_count = DecodeUTF8(input, i + esc_idx, output);
+ return i + esc_idx;
}
}
// If we reached this point, its possible for our input to have an
// incomplete sequence because we're consuming the full input. We need
// to trim any incomplete sequences from the end of the input.
- const size_t trimmed_len =
- simdutf::trim_partial_utf8(reinterpret_cast<const char*>(input), i);
+ //
+ // We use our own trim instead of simdutf::trim_partial_utf8 because
+ // we only want to trim sequences that are valid-so-far (true partial
+ // sequences that may be completed by future input). Invalid bytes
+ // like C0, C1, F5-FF should NOT be trimmed — they should be passed
+ // through to DecodeUTF8 which will replace them with U+FFFD per the
+ // maximal subpart algorithm.
+ const size_t trimmed_len = TrimValidPartialUTF8(input, i);
*output_count = DecodeUTF8(input, trimmed_len, output);
return trimmed_len;
}
diff --git a/vendor/libghostty-vt/src/simd/vt.zig b/vendor/libghostty-vt/src/simd/vt.zig
index fa8754fa..4230665f 100644
--- a/vendor/libghostty-vt/src/simd/vt.zig
+++ b/vendor/libghostty-vt/src/simd/vt.zig
@@ -45,36 +45,79 @@ fn utf8DecodeUntilControlSeqScalar(
const idx = indexOf(input, 0x1B) orelse input.len;
const decode = input[0..idx];
- // Go through and decode one item at a time.
+ // Go through and decode one item at a time, following the W3C/Unicode
+ // "U+FFFD Substitution of Maximal Subparts" algorithm for ill-formed
+ // subsequences.
var decode_offset: usize = 0;
var decode_count: usize = 0;
while (decode_offset < decode.len) {
- const decode_rem = decode[decode_offset..];
- const cp_len = std.unicode.utf8ByteSequenceLength(decode_rem[0]) catch {
- // Note, this is matching our SIMD behavior, but it is admittedly
- // a bit weird. See our "decode invalid leading byte" test too.
- // SIMD should be our source of truth then we copy behavior here.
- break;
- };
-
- // If we don't have that number of bytes available. we finish. We
- // assume this is a partial input and we defer to the future.
- if (decode_rem.len < cp_len) break;
-
- // We have the bytes available, so move forward
- const cp_bytes = decode_rem[0..cp_len];
- decode_offset += cp_len;
- if (std.unicode.utf8Decode(cp_bytes)) |cp| {
- output[decode_count] = @intCast(cp);
+ const b0 = decode[decode_offset];
+
+ // ASCII fast path
+ if (b0 < 0x80) {
+ output[decode_count] = b0;
decode_count += 1;
- } else |_| {
- // If decoding failed, we replace the leading byte with the
- // replacement char and then continue decoding after that
- // byte. This matches the SIMD behavior and is tested by the
- // "invalid UTF-8" tests.
+ decode_offset += 1;
+ continue;
+ }
+
+ // Continuation byte (80-BF) or invalid byte (C0-C1, F5-FF)
+ // as lead: each is its own maximal subpart → one FFFD per byte.
+ if (b0 < 0xC2 or b0 > 0xF4) {
+ output[decode_count] = 0xFFFD;
+ decode_count += 1;
+ decode_offset += 1;
+ continue;
+ }
+
+ // Multi-byte sequence. Determine expected length and the valid
+ // range for each continuation byte per Unicode Table 3-7.
+ const seq = utf8SeqInfo(b0);
+
+ // Check how many continuation bytes form a valid prefix (the
+ // maximal subpart). We check each byte against its specific
+ // valid range.
+ var valid: usize = 1; // lead byte is valid
+ for (0..seq.len - 1) |ci| {
+ if (decode_offset + valid >= decode.len) {
+ // Truncated at end of buffer: treat as incomplete
+ // input that may be completed later. Stop decoding
+ // without consuming these bytes.
+ return .{
+ .consumed = decode_offset,
+ .decoded = decode_count,
+ };
+ }
+ const cb = decode[decode_offset + valid];
+ if (cb < seq.ranges[ci][0] or cb > seq.ranges[ci][1]) {
+ // Byte doesn't match expected range. The maximal
+ // subpart ends here.
+ break;
+ }
+ valid += 1;
+ }
+
+ if (valid == seq.len) {
+ // Full sequence present and structurally valid. Decode it.
+ // (Structural validity per Table 3-7 guarantees decode success.)
+ const cp_bytes = decode[decode_offset..][0..seq.len];
+ if (std.unicode.utf8Decode(cp_bytes)) |cp| {
+ output[decode_count] = @intCast(cp);
+ decode_count += 1;
+ decode_offset += seq.len;
+ } else |_| {
+ // Should not happen given Table 3-7 validation, but
+ // be safe: emit FFFD for the lead byte.
+ output[decode_count] = 0xFFFD;
+ decode_count += 1;
+ decode_offset += 1;
+ }
+ } else {
+ // Incomplete/ill-formed: the maximal subpart (valid bytes)
+ // maps to a single FFFD.
output[decode_count] = 0xFFFD;
decode_count += 1;
- decode_offset -= cp_len - 1;
+ decode_offset += valid;
}
}
@@ -84,6 +127,27 @@ fn utf8DecodeUntilControlSeqScalar(
};
}
+const Utf8SeqInfo = struct {
+ len: u3,
+ ranges: [3][2]u8,
+};
+
+/// Returns the expected byte count and valid continuation byte ranges
+/// for a UTF-8 sequence based on its lead byte, per Unicode Table 3-7.
+fn utf8SeqInfo(lead: u8) Utf8SeqInfo {
+ return switch (lead) {
+ 0xC2...0xDF => .{ .len = 2, .ranges = .{ .{ 0x80, 0xBF }, .{ 0, 0 }, .{ 0, 0 } } },
+ 0xE0 => .{ .len = 3, .ranges = .{ .{ 0xA0, 0xBF }, .{ 0x80, 0xBF }, .{ 0, 0 } } },
+ 0xE1...0xEC => .{ .len = 3, .ranges = .{ .{ 0x80, 0xBF }, .{ 0x80, 0xBF }, .{ 0, 0 } } },
+ 0xED => .{ .len = 3, .ranges = .{ .{ 0x80, 0x9F }, .{ 0x80, 0xBF }, .{ 0, 0 } } },
+ 0xEE...0xEF => .{ .len = 3, .ranges = .{ .{ 0x80, 0xBF }, .{ 0x80, 0xBF }, .{ 0, 0 } } },
+ 0xF0 => .{ .len = 4, .ranges = .{ .{ 0x90, 0xBF }, .{ 0x80, 0xBF }, .{ 0x80, 0xBF } } },
+ 0xF1...0xF3 => .{ .len = 4, .ranges = .{ .{ 0x80, 0xBF }, .{ 0x80, 0xBF }, .{ 0x80, 0xBF } } },
+ 0xF4 => .{ .len = 4, .ranges = .{ .{ 0x80, 0x8F }, .{ 0x80, 0xBF }, .{ 0x80, 0xBF } } },
+ else => unreachable,
+ };
+}
+
test "decode no escape" {
const testing = std.testing;
@@ -131,7 +195,7 @@ test "decode incomplete UTF-8" {
var output: [64]u32 = undefined;
- // 2-byte
+ // 2-byte truncated at end of buffer
{
const str = "hello\xc2";
try testing.expectEqual(DecodeResult{
@@ -140,16 +204,18 @@ test "decode incomplete UTF-8" {
}, utf8DecodeUntilControlSeq(str, &output));
}
- // 3-byte
+ // 3-byte: \xe0 expects A0-BF next, but \x00 is not in range.
+ // \xe0 is a maximal subpart of length 1 → FFFD, then \x00 is ASCII NUL.
{
const str = "hello\xe0\x00";
- try testing.expectEqual(DecodeResult{
- .consumed = 5,
- .decoded = 5,
- }, utf8DecodeUntilControlSeq(str, &output));
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ try testing.expectEqual(@as(usize, 7), result.consumed);
+ try testing.expectEqual(@as(usize, 7), result.decoded);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[5]);
+ try testing.expectEqual(@as(u32, 0x00), output[6]);
}
- // 4-byte
+ // 4-byte truncated at end of buffer (F0 90 is valid so far)
{
const str = "hello\xf0\x90";
try testing.expectEqual(DecodeResult{
@@ -178,19 +244,248 @@ test "decode invalid UTF-8" {
try testing.expectEqual(@as(u32, 0x01), output[6]);
}
-// This is testing our current behavior so that we know we have to handle
-// this case in terminal/stream.zig. If we change this behavior, we can
-// remove the special handling in terminal/stream.zig.
-test "decode invalid leading byte isn't consumed or replaced" {
+// Per the maximal subpart spec, bytes F5-FF are each replaced with FFFD.
+test "decode invalid leading byte is replaced" {
const testing = std.testing;
var output: [64]u32 = undefined;
{
const str = "hello\xFF";
- try testing.expectEqual(DecodeResult{
- .consumed = 5,
- .decoded = 5,
- }, utf8DecodeUntilControlSeq(str, &output));
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ try testing.expectEqual(@as(usize, 6), result.consumed);
+ try testing.expectEqual(@as(usize, 6), result.decoded);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[5]);
+ }
+}
+
+test "decode invalid continuation in 3-byte sequence" {
+ const testing = std.testing;
+
+ var output: [64]u32 = undefined;
+
+ // \xe2 expects two continuation bytes, \x28 is not one
+ {
+ const str = "hello\xe2\x28world";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ // "hello" + replacement + "(" + "world" = 12 codepoints
+ try testing.expectEqual(@as(usize, 12), result.decoded);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[5]);
+ try testing.expectEqual(@as(u32, '('), output[6]);
+ try testing.expectEqual(@as(u32, 'w'), output[7]);
+ }
+}
+
+test "decode invalid continuation in 4-byte sequence" {
+ const testing = std.testing;
+
+ var output: [64]u32 = undefined;
+
+ // \xf0\x90 is a valid prefix of a 4-byte sequence, but \x28 breaks it.
+ // Maximal subpart is F0 90 (length 2) → single FFFD, then '(' proceeds.
+ {
+ const str = "hello\xf0\x90\x28world";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ // "hello" + FFFD + "(" + "world" = 12 codepoints
+ try testing.expectEqual(@as(usize, 12), result.decoded);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[5]);
+ try testing.expectEqual(@as(u32, '('), output[6]);
+ try testing.expectEqual(@as(u32, 'w'), output[7]);
+ }
+}
+
+test "decode multiple consecutive invalid bytes" {
+ const testing = std.testing;
+
+ var output: [64]u32 = undefined;
+
+ // Each lone continuation byte is its own maximal subpart → one FFFD each.
+ {
+ const str = "a\x80\x80b";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ // "a" + FFFD + FFFD + "b" = 4 codepoints
+ try testing.expectEqual(@as(usize, 4), result.decoded);
+ try testing.expectEqual(@as(u32, 'a'), output[0]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[1]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[2]);
+ try testing.expectEqual(@as(u32, 'b'), output[3]);
+ }
+
+ // C0 is an invalid lead byte (< C2), each byte gets its own FFFD.
+ {
+ const str = "a\xc0\xc0b";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ // "a" + FFFD + FFFD + "b" = 4 codepoints
+ try testing.expectEqual(@as(usize, 4), result.decoded);
+ try testing.expectEqual(@as(u32, 'a'), output[0]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[1]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[2]);
+ try testing.expectEqual(@as(u32, 'b'), output[3]);
+ }
+}
+
+test "decode unexpected continuation byte as lead" {
+ const testing = std.testing;
+
+ var output: [64]u32 = undefined;
+
+ // 0x80 is a continuation byte appearing as a lead byte
+ {
+ const str = "a\x80b";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ // "a" + replacement + "b" = 3 codepoints
+ try testing.expectEqual(@as(usize, 3), result.decoded);
+ try testing.expectEqual(@as(u32, 'a'), output[0]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[1]);
+ try testing.expectEqual(@as(u32, 'b'), output[2]);
+ }
+}
+
+test "decode overlong 2-byte encoding" {
+ const testing = std.testing;
+
+ var output: [64]u32 = undefined;
+
+ // \xc0\xaf: C0 is invalid lead (< C2) → FFFD, AF is lone continuation → FFFD
+ // Per Table 3-8: C0 AF → FFFD FFFD
+ {
+ const str = "a\xc0\xafb";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ // "a" + FFFD + FFFD + "b" = 4 codepoints
+ try testing.expectEqual(@as(usize, 4), result.decoded);
+ try testing.expectEqual(@as(u32, 'a'), output[0]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[1]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[2]);
+ try testing.expectEqual(@as(u32, 'b'), output[3]);
+ }
+}
+
+test "decode surrogate half" {
+ const testing = std.testing;
+
+ var output: [64]u32 = undefined;
+
+ // \xed\xa0\x80 encodes U+D800 (a surrogate). Per Table 3-7, after ED
+ // the next byte must be 80-9F. A0 is out of range, so ED is a maximal
+ // subpart of length 1 → FFFD. Then A0 and 80 are lone continuations
+ // → FFFD each. Per Table 3-9: ED A0 80 → FFFD FFFD FFFD
+ {
+ const str = "a\xed\xa0\x80b";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ // "a" + FFFD + FFFD + FFFD + "b" = 5 codepoints
+ try testing.expectEqual(@as(usize, 5), result.decoded);
+ try testing.expectEqual(@as(u32, 'a'), output[0]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[1]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[2]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[3]);
+ try testing.expectEqual(@as(u32, 'b'), output[4]);
+ }
+}
+
+test "decode valid multibyte surrounded by invalid" {
+ const testing = std.testing;
+
+ var output: [64]u32 = undefined;
+
+ // \xc3\xa9 = é (U+00E9), surrounded by invalid continuation bytes
+ {
+ const str = "\x80\xc3\xa9\x80";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ // replacement + é + replacement = 3 codepoints
+ try testing.expectEqual(@as(usize, 3), result.decoded);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[0]);
+ try testing.expectEqual(@as(u32, 0x00E9), output[1]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[2]);
+ }
+}
+
+test "decode invalid byte before escape" {
+ const testing = std.testing;
+
+ var output: [64]u32 = undefined;
+
+ // Invalid byte followed by ESC - should replace then stop
+ {
+ const str = "hi\x80\x1b[0m";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ try testing.expectEqual(@as(usize, 3), result.consumed);
+ try testing.expectEqual(@as(usize, 3), result.decoded);
+ try testing.expectEqual(@as(u32, 'h'), output[0]);
+ try testing.expectEqual(@as(u32, 'i'), output[1]);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[2]);
+ }
+}
+
+// Unicode Table 3-8: U+FFFD for Non-Shortest Form Sequences
+// Bytes: C0 AF E0 80 BF F0 81 82 41
+// Output: FFFD FFFD FFFD FFFD FFFD FFFD FFFD FFFD 0041
+test "Table 3-8: non-shortest form sequences" {
+ const testing = std.testing;
+ var output: [64]u32 = undefined;
+
+ const str = "\xC0\xAF\xE0\x80\xBF\xF0\x81\x82\x41";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ try testing.expectEqual(@as(usize, 9), result.consumed);
+ try testing.expectEqual(@as(usize, 9), result.decoded);
+ for (0..8) |i| {
+ try testing.expectEqual(@as(u32, 0xFFFD), output[i]);
+ }
+ try testing.expectEqual(@as(u32, 0x41), output[8]);
+}
+
+// Unicode Table 3-9: U+FFFD for Ill-Formed Sequences for Surrogates
+// Bytes: ED A0 80 ED BF BF ED AF 41
+// Output: FFFD FFFD FFFD FFFD FFFD FFFD FFFD FFFD 0041
+test "Table 3-9: surrogate sequences" {
+ const testing = std.testing;
+ var output: [64]u32 = undefined;
+
+ const str = "\xED\xA0\x80\xED\xBF\xBF\xED\xAF\x41";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ try testing.expectEqual(@as(usize, 9), result.consumed);
+ try testing.expectEqual(@as(usize, 9), result.decoded);
+ for (0..8) |i| {
+ try testing.expectEqual(@as(u32, 0xFFFD), output[i]);
}
+ try testing.expectEqual(@as(u32, 0x41), output[8]);
+}
+
+// Unicode Table 3-10: U+FFFD for Other Ill-Formed Sequences
+// Bytes: F4 91 92 93 FF 41 80 BF 42
+// Output: FFFD FFFD FFFD FFFD FFFD 0041 FFFD FFFD 0042
+test "Table 3-10: other ill-formed sequences" {
+ const testing = std.testing;
+ var output: [64]u32 = undefined;
+
+ const str = "\xF4\x91\x92\x93\xFF\x41\x80\xBF\x42";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ try testing.expectEqual(@as(usize, 9), result.consumed);
+ try testing.expectEqual(@as(usize, 9), result.decoded);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[0]); // F4
+ try testing.expectEqual(@as(u32, 0xFFFD), output[1]); // 91
+ try testing.expectEqual(@as(u32, 0xFFFD), output[2]); // 92
+ try testing.expectEqual(@as(u32, 0xFFFD), output[3]); // 93
+ try testing.expectEqual(@as(u32, 0xFFFD), output[4]); // FF
+ try testing.expectEqual(@as(u32, 0x0041), output[5]); // 41
+ try testing.expectEqual(@as(u32, 0xFFFD), output[6]); // 80
+ try testing.expectEqual(@as(u32, 0xFFFD), output[7]); // BF
+ try testing.expectEqual(@as(u32, 0x0042), output[8]); // 42
+}
+
+// Unicode Table 3-11: U+FFFD for Truncated Sequences
+// Bytes: E1 80 E2 F0 91 92 F1 BF 41
+// Output: FFFD FFFD FFFD FFFD 0041
+test "Table 3-11: truncated sequences" {
+ const testing = std.testing;
+ var output: [64]u32 = undefined;
+
+ const str = "\xE1\x80\xE2\xF0\x91\x92\xF1\xBF\x41";
+ const result = utf8DecodeUntilControlSeq(str, &output);
+ try testing.expectEqual(@as(usize, 9), result.consumed);
+ try testing.expectEqual(@as(usize, 5), result.decoded);
+ try testing.expectEqual(@as(u32, 0xFFFD), output[0]); // E1 80 (truncated 3-byte)
+ try testing.expectEqual(@as(u32, 0xFFFD), output[1]); // E2 (truncated 3-byte, next byte F0 not continuation)
+ try testing.expectEqual(@as(u32, 0xFFFD), output[2]); // F0 91 92 (truncated 4-byte)
+ try testing.expectEqual(@as(u32, 0xFFFD), output[3]); // F1 BF (truncated 4-byte, next byte 41 not continuation)
+ try testing.expectEqual(@as(u32, 0x0041), output[4]); // 41
}