diff options
| author | Can Celik <ogulcancelik@gmail.com> | 2026-09-10 21:51:57 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-09-10 21:51:57 +0300 |
| commit | 425c86179791cd32e2d4ce0ae7f269940b4b0663 (patch) | |
| tree | e52c4383a6d69216cbe2a32638bf90841d7e2189 /vendor/libghostty-vt/src/terminal/stream.zig | |
| parent | 90e947a6e26e02f10ec7f1694c4eb5b78a86406b (diff) | |
chore: upgrade libghostty and preserve terminal behavior (#3906)
Diffstat (limited to 'vendor/libghostty-vt/src/terminal/stream.zig')
| -rw-r--r-- | vendor/libghostty-vt/src/terminal/stream.zig | 1566 |
1 files changed, 1479 insertions, 87 deletions
diff --git a/vendor/libghostty-vt/src/terminal/stream.zig b/vendor/libghostty-vt/src/terminal/stream.zig index efb06ecd..29bdde7d 100644 --- a/vendor/libghostty-vt/src/terminal/stream.zig +++ b/vendor/libghostty-vt/src/terminal/stream.zig @@ -16,6 +16,7 @@ const kitty = @import("kitty.zig"); const modes = @import("modes.zig"); const osc = @import("osc.zig"); const sgr = @import("sgr.zig"); +const continuationpkg = @import("stream_continuation.zig"); const UTF8Decoder = @import("UTF8Decoder.zig"); const MouseShape = @import("mouse.zig").Shape; @@ -109,7 +110,7 @@ pub const Action = union(Key) { dcs_put: u8, dcs_unhook, apc_start, - apc_end, + apc_end: ApcEnd, apc_put: u8, apc_put_slice: ApcPutSlice, end_hyperlink, @@ -127,6 +128,8 @@ pub const Action = union(Key) { kitty_color_report: kitty.color.OSC, color_operation: ColorOperation, semantic_prompt: SemanticPrompt, + kitty_clipboard: KittyClipboard, + kitty_dnd: KittyDnd, pub const Key = lib.Enum( lib.target, @@ -226,6 +229,8 @@ pub const Action = union(Key) { "kitty_color_report", "color_operation", "semantic_prompt", + "kitty_clipboard", + "kitty_dnd", }, ); @@ -235,7 +240,7 @@ pub const Action = union(Key) { @This(), // TODO: Before shipping an ABI-compatible libghostty, verify this. // This was just arbitrarily chosen for now. - [16]u64, + .{ .padding = [16]u64 }, ); pub const Tag = c_union.Tag; pub const Value = c_union.Value; @@ -290,6 +295,11 @@ pub const Action = union(Key) { } }; + pub const ApcEnd = extern struct { + /// False when CAN, SUB, or another aborting transition ended the APC. + terminated: bool, + }; + pub const InvokeCharset = lib.Struct(lib.target, struct { bank: charsets.ActiveSlot, charset: charsets.Slots, @@ -406,6 +416,7 @@ pub const Action = union(Key) { pub const ClipboardContents = struct { kind: u8, data: []const u8, + terminator: osc.Terminator, pub const C = extern struct { kind: u8, @@ -438,6 +449,10 @@ pub const Action = union(Key) { }; pub const SemanticPrompt = osc.Command.SemanticPrompt; + + pub const KittyClipboard = osc.Command.KittyClipboardProtocol; + + pub const KittyDnd = osc.Command.KittyDndProtocol; }; /// Returns a type that can process a stream of tty control characters. @@ -476,46 +491,177 @@ pub fn Stream(comptime H: type) type { handler: Handler, parser: Parser, utf8decoder: UTF8Decoder, + continuation: ?continuationpkg.Tracker, + + pub const Options = struct { + /// The handler initial value, must be set. + handler: Handler, + + /// Allocator to use. If this is not set then the stream + /// will be fully allocation free. There are some operations + /// that will be dropped in this case such as OSC 52 clipboard + /// ops. + allocator: ?Allocator = null, + + /// Maximum size in bytes of the continuation suffix. If this is + /// null or zero then continuation tracking is disabled. This is + /// only applied when `allocator` is non-null; without an allocator + /// continuation tracking is disabled. Feeding this continuation + /// suffix into an equivalent stream at ground reconstructs the + /// unfinished state without repeating committed terminal effects. + /// Continuation tracking is only supported by TerminalStream. + continuation_max_bytes: ?usize = null, + }; - /// Initialize an allocation-free stream. This will preallocate various - /// sizes as necessary and anything over that will be dropped. If you - /// want to support more dynamic behavior use initAlloc instead. + /// Initialize a stream. Without an allocator, operations that require + /// heap allocation are dropped. /// /// As a concrete example of something that requires heap allocation, /// consider OSC 52 (clipboard operations) which can be arbitrarily /// large. /// - /// If you want to limit allocation size, use an allocator with - /// a size limit with initAlloc. - /// /// This takes ownership of the handler and will call deinit /// when the stream is deinitialized. - pub fn init(h: Handler) Self { + pub fn init(options: Options) Self { + // Initialize the parser + var parser: Parser = .init(); + if (options.allocator) |alloc| parser.osc_parser.alloc = alloc; + + // Initialize the continuation tracker if one is requested. + var tracker: ?continuationpkg.Tracker = null; + if (options.allocator) |alloc| { + if (options.continuation_max_bytes) |max_bytes| { + if (max_bytes > 0) tracker = .init(alloc, max_bytes); + } + } + return .{ - .handler = h, - .parser = .init(), + .handler = options.handler, + .parser = parser, .utf8decoder = .{}, + .continuation = tracker, }; } - /// Initialize the stream that supports heap allocation as necessary. - pub fn initAlloc(alloc: Allocator, h: Handler) Self { - var self: Self = .init(h); - self.parser.osc_parser.alloc = alloc; - return self; - } - pub fn deinit(self: *Self) void { + if (self.continuation) |*tracker| tracker.deinit(); self.parser.deinit(); self.handler.deinit(); } + /// Write the current continuation suffix directly to a caller-owned + /// writer. The caller must pause and serialize access to this Stream. + pub fn writeContinuation( + self: *const Self, + writer: *std.Io.Writer, + ) (std.Io.Writer.Error || error{ + ContinuationDisabled, + ContinuationUnavailable, + })!void { + const tracker = self.continuation orelse + return error.ContinuationDisabled; + if (tracker.broken) + return error.ContinuationUnavailable; + try tracker.write(writer); + } + + /// True when no continuation suffix is needed to reproduce the + /// stream's current parsing state. + pub inline fn ground(self: *const Self) bool { + // Parser ground alone is not sufficient because the UTF-8 + // decoder may have some state. + return self.parser.state == .ground and self.utf8decoder.state == 0; + } + + /// Update the continuation suffix after one complete feed call. + /// Must only be called when tracking is enabled. + fn trackContinuation(self: *Self, input: []const u8) void { + const tracker = &self.continuation.?; + + // If we're in a ground state, we have no continuation suffix + // to track by definition. + if (self.ground()) { + tracker.reset(); + return; + } + + // Retain the part of this feed needed to replay the unfinished + // state. When the parser is grounded here, the feed must have + // ended inside a UTF-8 codepoint instead, because the ground + // check above covers both state machines. + tracker.append( + if (self.parser.state != .ground) .vt else .utf8, + input, + ); + } + /// Process a string of characters. pub inline fn nextSlice(self: *Self, input: []const u8) void { - // Disable SIMD optimizations if build requests it or if our - // manual debug mode is on. - if (comptime debug or !build_options.simd) { - for (input) |c| self.next(c); + self.nextSliceUntracked(input); + + // Continuation tracking is opt-in and this branch predicts + // perfectly, so disabled streams pay nothing else here. + if (self.continuation != null) self.trackContinuation(input); + } + + /// Process a string of characters, but only the shortest prefix + /// needed to reach the ground state. + /// + /// The ground state is when the stream isn't in the middle of any + /// type of sequence: UTF-8, ESC, CSI, OSC, etc. It is the stateless + /// point of the stream. + /// + /// If the stream is already at ground then this consumes nothing + /// and returns zero. A non-null return is the number of bytes consumed + /// before reaching ground, including the byte that reaches it. A null + /// return means the full slice was consumed without reaching ground. + /// + /// This is anywhere from 1% to 5% slower than nextSlice, depending + /// on the types of inputs provided (e.g. OSC vs APC, whether it + /// ever reaches ground, etc.). + pub inline fn nextSliceUntilGround( + self: *Self, + input: []const u8, + ) ?usize { + const consumed = self.nextSliceUntilGroundUntracked(input); + const consumed_len = consumed orelse input.len; + if (self.continuation != null and consumed_len > 0) { + self.trackContinuation(input[0..consumed_len]); + } + + return consumed; + } + + inline fn nextSliceUntilGroundUntracked( + self: *Self, + input: []const u8, + ) ?usize { + if (self.ground()) return 0; + + // Process UTF-8 if we're within that state. + var offset: usize = 0; + while (self.utf8decoder.state != 0) { + if (offset >= input.len) return null; + self.nextUtf8(input[offset]); + offset += 1; + } + + // Process non-UTF-8 + if (self.parser.state != .ground) { + offset += self.consumeUntilGround(input[offset..]); + } + + return if (self.ground()) offset else null; + } + + inline fn nextSliceUntracked(self: *Self, input: []const u8) void { + // Byte-at-a-time parsing in debug mode only. Even without + // SIMD support (build_options.simd == false, e.g. wasm), the + // batched path below is much faster than per-byte dispatch + // because it decodes UTF-8 in bulk (via scalar fallbacks) and + // hands printable runs to the handler as print_slice actions. + if (comptime debug) { + for (input) |c| self.nextUntracked(c); return; } @@ -572,15 +718,57 @@ pub fn Stream(comptime H: type) type { var i: usize = 0; while (i < cps.len) { const cp = cps[i]; - if (cp <= 0xF) { + // C0 and UTF-8-decoded C1 controls never enter + // printable runs. A codepoint is one of those + // exactly when it has no bit set outside 0x9F: + // bits 0-4 and bit 7 cover 0x00-0x1F and + // 0x80-0x9F and nothing else, so a single + // AND-test classifies both ranges. + if ((cp & ~@as(u32, 0x9F)) == 0) { @branchHint(.unlikely); - self.execute(@intCast(cp)); + if (cp <= 0x1F) { + // C0 controls execute rather than print. + self.execute(@intCast(cp)); + } else { + // C1 controls decoded from UTF-8 are ignored. + logUnsupportedOnce( + "ignoring UTF-8-decoded C1 controls, first: 0x{x}", + .{cp}, + 0x80, + ); + } i += 1; continue; } + // Find the end of the printable run. This is an + // early-exit search loop that LLVM won't + // auto-vectorize, and printable runs dominate real + // input, so scan several codepoints at a time + // manually (same idiom as the printSliceFill run + // scan). var end = i + 1; - while (end < cps.len and cps[end] > 0xF) end += 1; + scan: { + if (simd.lanes(u32)) |lanes| { + const V = @Vector(lanes, u32); + // Non-printable (C0 or decoded C1) is + // equivalent to "no bit outside 0x9F is + // set": one AND-test per lane. + const mask: V = @splat(~@as(u32, 0x9F)); + const zero: V = @splat(0); + while (end + lanes <= cps.len) { + const v: V = cps[end..][0..lanes].*; + const stop = (v & mask) == zero; + if (@reduce(.Or, stop)) { + const bits: std.meta.Int(.unsigned, lanes) = @bitCast(stop); + end += @ctz(bits); + break :scan; + } + end += lanes; + } + } + while (end < cps.len and (cps[end] & ~@as(u32, 0x9F)) != 0) end += 1; + } self.handler.vt(.print_slice, .{ .cps = cps[i..end] }); i = end; } @@ -665,8 +853,70 @@ pub fn Stream(comptime H: type) type { if (self.parser.state == .sos_pm_apc_string) { offset += self.consumeApcString(input[offset..]); if (offset >= input.len) return input.len; - // The next byte exits the string state; let - // nextNonUtf8 below handle it. + + // Fast-path normal string termination. This matches + // Parser.next's exit and entry actions while avoiding + // the generic action loop for every completed APC. + switch (input[offset]) { + std.ascii.control_code.esc => { + self.parser.clear(); + self.parser.state = .escape; + self.handler.vt(.apc_end, .{ .terminated = true }); + offset += 1; + continue; + }, + 0x9C => { + self.parser.state = .ground; + self.handler.vt(.apc_end, .{ .terminated = true }); + offset += 1; + continue; + }, + else => {}, + } + + // Aborting transitions need the scalar path so the + // handler can distinguish them from terminators. + } + + // Bulk-consume OSC string bytes into the OSC parser. + // OSC payloads (e.g. OSC 52 clipboard operations and + // the kitty clipboard protocol) can be megabytes of + // base64 data, so per-byte dispatch is far too slow. + // This can't be used for handlers with a vtRaw hook + // because the bytes never produce osc_put actions. + if (self.parser.state == .osc_string) { + offset += self.consumeOscString(input[offset..]); + if (offset >= input.len) return input.len; + + // Fast-path normal string termination. This matches + // Parser.next's exit and entry actions while avoiding + // the generic action loop for every completed OSC. + switch (input[offset]) { + // The sequence should be terminated by a full + // ST ("ESC \"); the "\" that should follow is + // dispatched as a normal escape sequence. + std.ascii.control_code.esc => { + if (self.parser.osc_parser.end( + std.ascii.control_code.esc, + )) |cmd| self.oscDispatch(cmd.*); + self.parser.clear(); + self.parser.state = .escape; + offset += 1; + continue; + }, + // BEL terminates the string directly. + std.ascii.control_code.bel => { + if (self.parser.osc_parser.end( + std.ascii.control_code.bel, + )) |cmd| self.oscDispatch(cmd.*); + self.parser.state = .ground; + offset += 1; + continue; + }, + // CAN/SUB abort and ignored C0 bytes go + // through the state machine below. + else => {}, + } } } @@ -822,10 +1072,52 @@ pub fn Stream(comptime H: type) type { return end; } + /// Bulk-consume OSC string bytes into the OSC parser. Returns + /// the number of bytes consumed. Stops at the first byte that + /// is not an osc_put byte in the parse table, leaving it for + /// the caller to process through the state machine. Every byte + /// >= 0x20 is an osc_put byte; bytes below either terminate or + /// abort the string (BEL, CAN, SUB, ESC) or are ignored. + /// + /// Must not be used by handlers with a vtRaw hook because the + /// consumed bytes never produce osc_put actions. + fn consumeOscString(self: *Self, input: []const u8) usize { + comptime assert(!@hasDecl(T, "vtRaw")); + assert(self.parser.state == .osc_string); + + var end: usize = 0; + if (comptime std.simd.suggestVectorLength(u8)) |vector_len| { + const ByteVector = @Vector(vector_len, u8); + while (end + vector_len <= input.len) { + const bytes: ByteVector = input[end..][0..vector_len].*; + const stop = bytes < @as(ByteVector, @splat(0x20)); + if (@reduce(.Or, stop)) break; + end += vector_len; + } + } + while (end < input.len) { + switch (input[end]) { + // Not osc_put bytes: BEL/CAN/SUB/ESC terminate or + // abort the state; other C0 bytes are ignored by it. + 0x00...0x1F => break, + // Everything else is an osc_put byte. + else => end += 1, + } + } + + if (end > 0) self.parser.osc_parser.nextSlice(input[0..end]); + return end; + } + /// Like nextSlice but takes one byte and is necessarily a scalar /// operation that can't use SIMD. Prefer nextSlice if you can and /// try to get multiple bytes at once. pub inline fn next(self: *Self, c: u8) void { + self.nextUntracked(c); + if (self.continuation != null) self.trackContinuation(&.{c}); + } + + inline fn nextUntracked(self: *Self, c: u8) void { // The scalar path can be responsible for decoding UTF-8. if (self.parser.state == .ground) { self.nextUtf8(c); @@ -871,20 +1163,36 @@ pub fn Stream(comptime H: type) type { // We need to increase the eval branch limit because a lot of // tests end up running almost completely at comptime due to // a chain of inline functions. - @setEvalBranchQuota(100_000); - - // C0 control - if (c <= 0xF) { + @setEvalBranchQuota(200_000); + + // C0 control or a C1 control decoded from UTF-8: exactly + // the codepoints with no bit set outside 0x9F (bits 0-4 + // and bit 7 cover 0x00-0x1F and 0x80-0x9F and nothing + // else), so the printable fast path stays a single + // AND-test. + if ((c & ~@as(u21, 0x9F)) == 0) { @branchHint(.unlikely); + + // ESC + if (c == 0x1B) { + self.parser.state = .escape; + self.parser.clear(); + return; + } + + // Ignore C1 that came via UTF-8 decoding, matching xterm. + if (c > 0x1F) { + logUnsupportedOnce( + "ignoring UTF-8-decoded C1 controls, first: 0x{x}", + .{c}, + 0x80, + ); + return; + } + self.execute(@intCast(c)); return; } - // ESC - if (c == 0x1B) { - self.parser.state = .escape; - self.parser.clear(); - return; - } self.print(@intCast(c)); } @@ -1003,7 +1311,9 @@ pub fn Stream(comptime H: type) type { .dcs_unhook => self.handler.vt(.dcs_unhook, {}), .apc_start => self.handler.vt(.apc_start, {}), .apc_put => |code| self.handler.vt(.apc_put, code), - .apc_end => self.handler.vt(.apc_end, {}), + .apc_end => self.handler.vt(.apc_end, .{ + .terminated = c == std.ascii.control_code.esc or c == 0x9C, + }), } } } @@ -1323,7 +1633,7 @@ pub fn Stream(comptime H: type) type { const mode_: ?csi.EraseDisplay = switch (input.params.len) { 0 => .below, - 1 => std.meta.intToEnum(csi.EraseDisplay, input.params[0]) catch null, + 1 => std.enums.fromInt(csi.EraseDisplay, input.params[0]), else => null, }; @@ -1652,7 +1962,7 @@ pub fn Stream(comptime H: type) type { 'g' => switch (input.intermediates.len) { 0 => { const mode: csi.TabClear = switch (input.params.len) { - 1 => std.meta.intToEnum(csi.TabClear, input.params[0]) catch { + 1 => std.enums.fromInt(csi.TabClear, input.params[0]) orelse { log.warn("invalid tab clear mode: {}", .{input.params[0]}); return; }, @@ -2309,6 +2619,7 @@ pub fn Stream(comptime H: type) type { self.handler.vt(.clipboard_contents, .{ .kind = clip.kind, .data = clip.data, + .terminator = clip.terminator, }); }, @@ -2364,6 +2675,14 @@ pub fn Stream(comptime H: type) type { self.handler.vt(.progress_report, v); }, + .kitty_clipboard_protocol => |v| { + self.handler.vt(.kitty_clipboard, v); + }, + + .kitty_dnd_protocol => |v| { + self.handler.vt(.kitty_dnd, v); + }, + .conemu_sleep, .conemu_show_message_box, .conemu_change_tab_title, @@ -2374,8 +2693,7 @@ pub fn Stream(comptime H: type) type { .conemu_output_environment_variable, .conemu_run_process, .kitty_text_sizing, - .kitty_clipboard_protocol, - .kitty_dnd_protocol, + .kitty_desktop_notification, .context_signal, => { log.debug("unimplemented OSC callback: {}", .{cmd}); @@ -2780,7 +3098,7 @@ test "stream: print" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.next('x'); try testing.expectEqual(@as(u21, 'x'), s.handler.c.?); } @@ -2802,7 +3120,7 @@ test "simd: print invalid utf-8" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice(&.{0xFF}); try testing.expectEqual(@as(u21, 0xFFFD), s.handler.c.?); } @@ -2824,7 +3142,7 @@ test "simd: complete incomplete utf-8" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice(&.{0xE0}); // 3 byte try testing.expect(s.handler.c == null); s.nextSlice(&.{0xA0}); // still incomplete @@ -2833,6 +3151,166 @@ test "simd: complete incomplete utf-8" { try testing.expectEqual(@as(u21, 0x800), s.handler.c.?); } +test "stream: ground state C0 controls are executed, not printed" { + const H = struct { + buf: [128]u21 = undefined, + len: usize = 0, + + pub fn vt( + self: *@This(), + comptime action: Action.Tag, + value: Action.Value(action), + ) void { + switch (action) { + .print => { + self.buf[self.len] = value.cp; + self.len += 1; + }, + .print_slice => for (value.cps) |cp| { + self.buf[self.len] = @intCast(cp); + self.len += 1; + }, + else => {}, + } + } + }; + + // Every C0 control except ESC must never produce a print action + // in the ground state. ESC is excluded because it begins an + // escape sequence rather than executing. + for (0..0x20) |c| { + if (c == 0x1B) continue; + + // Scalar path. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + s.next('A'); + s.next(@intCast(c)); + s.next('B'); + try testing.expectEqual(@as(usize, 2), s.handler.len); + try testing.expectEqual(@as(u21, 'A'), s.handler.buf[0]); + try testing.expectEqual(@as(u21, 'B'), s.handler.buf[1]); + } + + // Batched path. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + s.nextSlice(&.{ 'A', 'B', @intCast(c), 'C', 'D' }); + try testing.expectEqual(@as(usize, 4), s.handler.len); + for ("ABCD", 0..) |expected, i| { + try testing.expectEqual(@as(u21, expected), s.handler.buf[i]); + } + } + + // Batched path with a run long enough to exercise the + // vectorized printable-run scan on either side of the control. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + var input: [65]u8 = @splat('A'); + input[32] = @intCast(c); + s.nextSlice(&input); + try testing.expectEqual(@as(usize, 64), s.handler.len); + for (s.handler.buf[0..s.handler.len]) |cp| { + try testing.expectEqual(@as(u21, 'A'), cp); + } + } + } +} + +test "stream: ground state UTF-8-decoded C1 controls are ignored" { + const H = struct { + buf: [128]u21 = undefined, + len: usize = 0, + + pub fn vt( + self: *@This(), + comptime action: Action.Tag, + value: Action.Value(action), + ) void { + switch (action) { + .print => { + self.buf[self.len] = value.cp; + self.len += 1; + }, + .print_slice => for (value.cps) |cp| { + self.buf[self.len] = @intCast(cp); + self.len += 1; + }, + else => {}, + } + } + }; + + // Every C1 control that arrives as well-formed UTF-8 (a two byte + // 0xC2-lead sequence) must be dropped: not printed and not + // interpreted as a control (e.g. U+009B must not start a CSI). + for (0x80..0xA0) |c| { + const enc: [2]u8 = .{ 0xC2, @intCast(c) }; + + // Scalar path. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + s.next('A'); + s.next(enc[0]); + s.next(enc[1]); + s.next('B'); + try testing.expectEqual(@as(usize, 2), s.handler.len); + try testing.expectEqual(@as(u21, 'A'), s.handler.buf[0]); + try testing.expectEqual(@as(u21, 'B'), s.handler.buf[1]); + } + + // Batched path. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + s.nextSlice(&.{ 'A', 'B', enc[0], enc[1], 'C', 'D' }); + try testing.expectEqual(@as(usize, 4), s.handler.len); + for ("ABCD", 0..) |expected, i| { + try testing.expectEqual(@as(u21, expected), s.handler.buf[i]); + } + } + + // Batched path with a run long enough to exercise the + // vectorized printable-run scan on either side of the control. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + var input: [66]u8 = @splat('A'); + input[32] = enc[0]; + input[33] = enc[1]; + s.nextSlice(&input); + try testing.expectEqual(@as(usize, 64), s.handler.len); + for (s.handler.buf[0..s.handler.len]) |cp| { + try testing.expectEqual(@as(u21, 'A'), cp); + } + } + } + + // U+00A0 (NBSP), just past the C1 range, must still print. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + s.nextSlice(&.{ 0xC2, 0xA0 }); + try testing.expectEqual(@as(usize, 1), s.handler.len); + try testing.expectEqual(@as(u21, 0xA0), s.handler.buf[0]); + } + + // A codepoint whose continuation byte falls in the C1 range must + // still print: "Ü" is 0xC3 0x9C. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + s.nextSlice("Ü"); + try testing.expectEqual(@as(usize, 1), s.handler.len); + try testing.expectEqual(@as(u21, 0xDC), s.handler.buf[0]); + } + + // A raw C1 byte is ill-formed UTF-8, not a decoded C1: it must + // still produce a U+FFFD replacement. + { + var s: Stream(H) = .init(.{ .handler = .{} }); + s.nextSlice(&.{0x9B}); + try testing.expectEqual(@as(usize, 1), s.handler.len); + try testing.expectEqual(@as(u21, 0xFFFD), s.handler.buf[0]); + } +} + test "stream: cursor right (CUF)" { const H = struct { amount: u16 = 0, @@ -2849,7 +3327,7 @@ test "stream: cursor right (CUF)" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[C"); try testing.expectEqual(@as(u16, 1), s.handler.amount); @@ -2882,7 +3360,7 @@ test "stream: dec set mode (SM) and reset mode (RM)" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[?6h"); try testing.expectEqual(@as(modes.Mode, .origin), s.handler.mode); @@ -2911,7 +3389,7 @@ test "stream: ansi set mode (SM) and reset mode (RM)" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[4h"); try testing.expectEqual(@as(modes.Mode, .insert), s.handler.mode.?); @@ -2942,7 +3420,7 @@ test "stream: ansi set mode (SM) and reset mode (RM) with unknown value" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[6h"); try testing.expect(s.handler.mode == null); @@ -2968,7 +3446,7 @@ test "stream: restore mode" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); for ("\x1B[?42r") |c| s.next(c); try testing.expect(!s.handler.called); } @@ -2990,7 +3468,7 @@ test "stream: pop kitty keyboard with no params defaults to 1" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); for ("\x1B[<u") |c| s.next(c); try testing.expectEqual(@as(u16, 1), s.handler.n); } @@ -3015,7 +3493,7 @@ test "stream: DECSCA" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); { for ("\x1B[\"q") |c| s.next(c); try testing.expectEqual(ansi.ProtectedMode.off, s.handler.v.?); @@ -3071,7 +3549,7 @@ test "stream: DECED, DECSED" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); { for ("\x1B[?J") |c| s.next(c); try testing.expectEqual(csi.EraseDisplay.below, s.handler.mode.?); @@ -3164,7 +3642,7 @@ test "stream: DECEL, DECSEL" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); { for ("\x1B[?K") |c| s.next(c); try testing.expectEqual(csi.EraseLine.right, s.handler.mode.?); @@ -3230,7 +3708,7 @@ test "stream: DECSCUSR" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[ q"); try testing.expect(s.handler.style.? == .default); @@ -3258,7 +3736,7 @@ test "stream: DECSCUSR without space" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[q"); try testing.expect(s.handler.style == null); @@ -3282,7 +3760,7 @@ test "stream: XTSHIFTESCAPE" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[>2s"); try testing.expect(s.handler.escape == null); @@ -3318,18 +3796,203 @@ test "stream: change window title with invalid utf-8" { }; { - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b]2;abc\x1b\\"); try testing.expect(s.handler.seen); } { - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b]2;abc\xc0\x1b\\"); try testing.expect(!s.handler.seen); } } +test "stream: osc 52 large payload in chunks" { + const alloc = testing.allocator; + + const H = struct { + const Self = @This(); + data: std.ArrayListUnmanaged(u8) = .empty, + kind: u8 = 0, + terminator: ?osc.Terminator = null, + count: usize = 0, + + pub fn vt( + self: *Self, + comptime action: Action.Tag, + value: Action.Value(action), + ) void { + switch (action) { + .clipboard_contents => { + self.count += 1; + self.kind = value.kind; + self.terminator = value.terminator; + self.data.appendSlice( + testing.allocator, + value.data, + ) catch @panic("OOM"); + }, + else => {}, + } + } + }; + + // A payload that far exceeds the fixed buffer so the allocating + // capture is used, fed in chunk sizes that don't align with the + // sequence so the bulk path sees every kind of boundary. + const prefix = "\x1b]52;c;"; + const payload_len = 150_000; + var input: std.ArrayListUnmanaged(u8) = .empty; + defer input.deinit(alloc); + try input.appendSlice(alloc, prefix); + for (0..payload_len) |i| try input.append( + alloc, + 'A' + @as(u8, @intCast(i % 26)), + ); + try input.appendSlice(alloc, "\x1b\\"); + + var s: Stream(H) = .init(.{ .handler = .{}, .allocator = alloc }); + defer s.parser.deinit(); + defer s.handler.data.deinit(alloc); + + var i: usize = 0; + while (i < input.items.len) { + const end = @min(i + 4093, input.items.len); + s.nextSlice(input.items[i..end]); + i = end; + } + + try testing.expectEqual(@as(usize, 1), s.handler.count); + try testing.expectEqual(@as(u8, 'c'), s.handler.kind); + try testing.expectEqual(osc.Terminator.st, s.handler.terminator.?); + try testing.expectEqualSlices( + u8, + input.items[prefix.len .. prefix.len + payload_len], + s.handler.data.items, + ); +} + +test "stream: osc bulk path matches per-byte path" { + const alloc = testing.allocator; + + // Records every dispatch relevant to OSC processing in a + // normalized text form so streams fed different ways can be + // compared byte-for-byte. + const H = struct { + const Self = @This(); + alloc: Allocator, + journal: std.ArrayListUnmanaged(u8) = .empty, + + fn record(self: *Self, comptime fmt: []const u8, args: anytype) void { + const s = std.fmt.allocPrint( + self.alloc, + fmt, + args, + ) catch @panic("OOM"); + defer self.alloc.free(s); + self.journal.appendSlice(self.alloc, s) catch @panic("OOM"); + } + + pub fn vt( + self: *Self, + comptime action: Action.Tag, + value: Action.Value(action), + ) void { + switch (action) { + .clipboard_contents => self.record( + "clip kind={c} term={s} data={s}\n", + .{ value.kind, @tagName(value.terminator), value.data }, + ), + .kitty_clipboard => self.record( + "kitty meta={s} payload={?s}\n", + .{ value.metadata, value.payload }, + ), + .window_title => self.record( + "title {s}\n", + .{value.title}, + ), + // Normalize prints to per-codepoint so the per-byte + // and slice paths journal identically. + .print => self.record("print {u}\n", .{value.cp}), + .print_slice => for (value.cps) |cp| self.record( + "print {u}\n", + .{@as(u21, @intCast(cp))}, + ), + else => {}, + } + } + }; + + const cases = [_][]const u8{ + "\x1b]52;c;aGVsbG8=\x1b\\", + "\x1b]52;c;aGVsbG8=\x07", + "\x1b]52;;aGVsbG8=\x07", + // Ignored C0 byte embedded in the payload. + "\x1b]52;c;aGVs\x01bG8=\x07", + // CAN and SUB aborts. + "\x1b]52;c;aGVsbG8=\x18", + "\x1b]52;c;aGVsbG8=\x1a", + // C1 byte embedded in the payload is data. + "\x1b]0;ab\x9ccd\x07", + // Terminated with trailing printable text. + "\x1b]0;a title\x07x", + // Back-to-back sequences. + "\x1b]2;another title\x1b\\\x1b]0;t2\x07", + "\x1b]5522;type=write;aGVsbG8=\x1b\\", + // Invalid OSC number. + "\x1b]999;junk\x07", + // Exceeds the fixed buffer: allocating capture. + "\x1b]52;c;" ++ "y" ** 3000 ++ "\x1b\\", + // Exceeds the fixed buffer: overflow, no dispatch. + "\x1b]0;" ++ "x" ** 3000 ++ "\x07", + }; + + for (cases) |case| { + // Reference: byte-at-a-time. + var ref: Stream(H) = .init(.{ + .handler = .{ .alloc = alloc }, + .allocator = alloc, + }); + defer ref.parser.deinit(); + defer ref.handler.journal.deinit(alloc); + for (case) |c| ref.next(c); + + // The whole slice at once. + { + var s: Stream(H) = .init(.{ + .handler = .{ .alloc = alloc }, + .allocator = alloc, + }); + defer s.parser.deinit(); + defer s.handler.journal.deinit(alloc); + s.nextSlice(case); + try testing.expectEqualSlices( + u8, + ref.handler.journal.items, + s.handler.journal.items, + ); + } + + // Split into two slices at every possible boundary. + for (0..case.len + 1) |split| { + var s: Stream(H) = .init(.{ + .handler = .{ .alloc = alloc }, + .allocator = alloc, + }); + defer s.parser.deinit(); + defer s.handler.journal.deinit(alloc); + s.nextSlice(case[0..split]); + s.nextSlice(case[split..]); + try testing.expectEqualSlices( + u8, + ref.handler.journal.items, + s.handler.journal.items, + ); + } + } +} + test "stream: insert characters" { const H = struct { const Self = @This(); @@ -3348,7 +4011,7 @@ test "stream: insert characters" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); for ("\x1B[42@") |c| s.next(c); try testing.expect(s.handler.called); @@ -3374,7 +4037,7 @@ test "stream: insert characters explicit zero clamps to 1" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); for ("\x1B[0@") |c| s.next(c); try testing.expectEqual(@as(usize, 1), s.handler.value.?); } @@ -3398,7 +4061,7 @@ test "stream: SCOSC" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); for ("\x1B[s") |c| s.next(c); try testing.expect(s.handler.called); } @@ -3421,7 +4084,7 @@ test "stream: SCORC" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); for ("\x1B[u") |c| s.next(c); try testing.expect(s.handler.called); } @@ -3442,7 +4105,7 @@ test "stream: too many csi params" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1C"); } @@ -3459,7 +4122,7 @@ test "stream: csi param too long" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1B[1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111C"); } @@ -3479,7 +4142,7 @@ test "stream: send report with CSI t" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[14t"); try testing.expectEqual(csi.SizeReportStyle.csi_14_t, s.handler.style); @@ -3513,7 +4176,7 @@ test "stream: invalid CSI t" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[19t"); try testing.expectEqual(null, s.handler.style); @@ -3535,7 +4198,7 @@ test "stream: CSI t push title" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[22;0t"); try testing.expectEqual(@as(u16, 0), s.handler.index.?); @@ -3557,7 +4220,7 @@ test "stream: CSI t push title with explicit window" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[22;2t"); try testing.expectEqual(@as(u16, 0), s.handler.index.?); @@ -3579,7 +4242,7 @@ test "stream: CSI t push title with explicit icon" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[22;1t"); try testing.expectEqual(null, s.handler.index); @@ -3601,7 +4264,7 @@ test "stream: CSI t push title with index" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[22;0;5t"); try testing.expectEqual(@as(u16, 5), s.handler.index.?); @@ -3623,7 +4286,7 @@ test "stream: CSI t pop title" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[23;0t"); try testing.expectEqual(@as(u16, 0), s.handler.index.?); @@ -3645,7 +4308,7 @@ test "stream: CSI t pop title with explicit window" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[23;2t"); try testing.expectEqual(@as(u16, 0), s.handler.index.?); @@ -3667,7 +4330,7 @@ test "stream: CSI t pop title with explicit icon" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[23;1t"); try testing.expectEqual(null, s.handler.index); @@ -3689,7 +4352,7 @@ test "stream: CSI t pop title with index" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[23;0;5t"); try testing.expectEqual(@as(u16, 5), s.handler.index.?); @@ -3709,7 +4372,7 @@ test "stream CSI W clear tab stops" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[2W"); try testing.expectEqual(Action.Key.tab_clear_current, s.handler.action.?); @@ -3732,7 +4395,7 @@ test "stream CSI W tab set" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[W"); try testing.expectEqual(Action.Key.tab_set, s.handler.action.?); @@ -3764,7 +4427,7 @@ test "stream CSI ? W reset tab stops" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); s.nextSlice("\x1b[?2W"); try testing.expect(s.handler.action == null); @@ -3798,7 +4461,7 @@ test "stream: SGR with 17+ parameters for underline color" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); // Kakoune-style SGR with underline color as 17th parameter // This tests the fix where param 17 was being dropped @@ -3826,7 +4489,7 @@ test "stream: tab clear with overflowing param" { } }; - var s: Stream(H) = .init(.{}); + var s: Stream(H) = .init(.{ .handler = .{} }); // This is the exact input from the fuzz crash (minus the mode byte): // CSI with a huge numeric param that saturates to 65535, followed by 'g'. s.nextSlice("\x1b[388888888888888888888888888888888888g\x1b[0m"); @@ -3866,7 +4529,7 @@ const ApcTestHandler = struct { }; test "stream: apc bulk slice" { - var s: Stream(ApcTestHandler) = .init(.{}); + var s: Stream(ApcTestHandler) = .init(.{ .handler = .{} }); s.nextSlice("\x1b_Gf=24,s=10,v=20;aGVsbG8=\x1b\\"); try testing.expectEqual(@as(usize, 1), s.handler.started); @@ -3883,8 +4546,20 @@ test "stream: apc bulk slice" { } } +test "stream: apc bulk slice C1 ST" { + var s: Stream(ApcTestHandler) = .init(.{ .handler = .{} }); + s.nextSlice("\x1b_Gpayload\x9c"); + + try testing.expectEqual(@as(usize, 1), s.handler.started); + try testing.expectEqual(@as(usize, 1), s.handler.ended); + try testing.expectEqualStrings( + "Gpayload", + s.handler.buf[0..s.handler.len], + ); +} + test "stream: apc bulk slice split across inputs" { - var s: Stream(ApcTestHandler) = .init(.{}); + var s: Stream(ApcTestHandler) = .init(.{ .handler = .{} }); s.nextSlice("\x1b_Gf=24,s=10"); s.nextSlice(",v=20;aGVs"); s.nextSlice("bG8=\x1b\\"); @@ -3898,7 +4573,7 @@ test "stream: apc bulk slice split across inputs" { } test "stream: apc bulk slice keeps C0 bytes as data" { - var s: Stream(ApcTestHandler) = .init(.{}); + var s: Stream(ApcTestHandler) = .init(.{ .handler = .{} }); // BEL does not terminate an APC string; it is payload data. s.nextSlice("\x1b_Gx\x07y\x1b\\"); @@ -3907,7 +4582,7 @@ test "stream: apc bulk slice keeps C0 bytes as data" { } test "stream: apc aborted by CAN" { - var s: Stream(ApcTestHandler) = .init(.{}); + var s: Stream(ApcTestHandler) = .init(.{ .handler = .{} }); // CAN (0x18) aborts the APC string via the anywhere => ground // transition. Exiting the sos_pm_apc_string state emits apc_end, // and the trailing bytes are printed, not treated as APC data. @@ -3922,7 +4597,7 @@ test "stream: apc aborted by CAN" { } test "stream: apc scalar path matches" { - var s: Stream(ApcTestHandler) = .init(.{}); + var s: Stream(ApcTestHandler) = .init(.{ .handler = .{} }); for ("\x1b_Gf=24;aGVsbG8=\x1b\\") |c| s.next(c); try testing.expectEqual(@as(usize, 1), s.handler.started); @@ -3945,9 +4620,9 @@ test "stream: apc vector boundaries match scalar path" { input[4 + position] = '\\'; const bytes = input[0 .. 5 + position]; - var bulk: Stream(ApcTestHandler) = .init(.{}); + var bulk: Stream(ApcTestHandler) = .init(.{ .handler = .{} }); bulk.nextSlice(bytes); - var scalar: Stream(ApcTestHandler) = .init(.{}); + var scalar: Stream(ApcTestHandler) = .init(.{ .handler = .{} }); for (bytes) |byte| scalar.next(byte); try testing.expectEqual(scalar.handler.started, bulk.handler.started); @@ -3958,3 +4633,720 @@ test "stream: apc vector boundaries match scalar path" { ); }; } + +const ContinuationTestHandler = struct { + committed: usize = 0, + apc_active: bool = false, + apc_buf: [256]u8 = undefined, + apc_len: usize = 0, + dcs_active: bool = false, + + pub fn deinit(_: *@This()) void {} + + pub fn vt( + self: *@This(), + comptime action: Action.Tag, + value: Action.Value(action), + ) void { + switch (action) { + .apc_start => self.apc_active = true, + .apc_put => { + self.apc_buf[self.apc_len] = value; + self.apc_len += 1; + }, + .apc_put_slice => { + @memcpy( + self.apc_buf[self.apc_len..][0..value.bytes.len], + value.bytes, + ); + self.apc_len += value.bytes.len; + }, + .dcs_hook => self.dcs_active = true, + .dcs_put => {}, + .apc_end => { + self.apc_active = false; + self.apc_len = 0; + self.committed += 1; + }, + .dcs_unhook => { + self.dcs_active = false; + self.committed += 1; + }, + .print => self.committed += 1, + .print_slice => self.committed += value.cps.len, + .print_repeat => self.committed += value, + else => self.committed += 1, + } + } +}; + +const ContinuationNullHandler = struct { + pub fn deinit(_: *@This()) void {} + + pub fn vt( + _: *@This(), + comptime _: Action.Tag, + _: anytype, + ) void {} +}; + +test "stream: nextSliceUntilGround stops at the earliest boundary" { + const S = Stream(ContinuationTestHandler); + var stream: S = .init(.{ .handler = .{} }); + defer stream.deinit(); + + stream.nextSlice("\x1b[31"); + try testing.expect(!stream.ground()); + stream.handler.committed = 0; + + const input = "mABC\x1b["; + const consumed = stream.nextSliceUntilGround(input).?; + try testing.expectEqual(@as(usize, 1), consumed); + try testing.expect(stream.ground()); + try testing.expectEqual(@as(usize, 1), stream.handler.committed); + + // The suffix was not inspected by the handler and can be processed after + // the caller performs work at the boundary. + stream.nextSlice(input[consumed..]); + try testing.expect(!stream.ground()); + try testing.expectEqual(Parser.State.csi_entry, stream.parser.state); + try testing.expectEqual(@as(usize, 4), stream.handler.committed); +} + +test "stream: nextSliceUntilGround consumes all input without a boundary" { + const S = Stream(ContinuationNullHandler); + var stream: S = .init(.{ .handler = .{} }); + defer stream.deinit(); + + try testing.expectEqual( + @as(?usize, 0), + stream.nextSliceUntilGround("unprocessed"), + ); + try testing.expect(stream.ground()); + + stream.nextSlice("\x1b["); + try testing.expectEqual( + @as(?usize, null), + stream.nextSliceUntilGround("123"), + ); + try testing.expect(!stream.ground()); + + // A boundary on the final byte is distinguishable from exhausting the + // input while still pending. + try testing.expectEqual( + @as(?usize, 1), + stream.nextSliceUntilGround("m"), + ); + try testing.expect(stream.ground()); +} + +test "stream: nextSliceUntilGround handles UTF-8 boundaries" { + const S = Stream(ContinuationTestHandler); + + // A completed codepoint is committed synchronously, and the printable + // suffix remains untouched. + var valid: S = .init(.{ .handler = .{} }); + defer valid.deinit(); + valid.nextSlice(&.{0xF0}); + const valid_input = [_]u8{ 0x9F, 0x98, 0x84, 'X' }; + try testing.expectEqual( + @as(?usize, 3), + valid.nextSliceUntilGround(&valid_input), + ); + try testing.expect(valid.ground()); + try testing.expectEqual(@as(usize, 1), valid.handler.committed); + + // A malformed continuation emits the replacement codepoint and retries + // the same byte. Ground is observed after that complete byte operation. + var malformed: S = .init(.{ .handler = .{} }); + defer malformed.deinit(); + malformed.nextSlice(&.{ 0xE0, 0xA0 }); + try testing.expectEqual( + @as(?usize, 1), + malformed.nextSliceUntilGround("A!"), + ); + try testing.expect(malformed.ground()); + try testing.expectEqual(@as(usize, 2), malformed.handler.committed); + + // If the retried byte is ESC, the stream has begun VT state at the end of + // that byte and must continue to the following ground boundary. + var retry_escape: S = .init(.{ .handler = .{} }); + defer retry_escape.deinit(); + retry_escape.nextSlice(&.{ 0xE0, 0xA0 }); + try testing.expectEqual( + @as(?usize, 3), + retry_escape.nextSliceUntilGround("\x1b[mX"), + ); + try testing.expect(retry_escape.ground()); +} + +test "stream: nextSliceUntilGround handles aborts and bulk strings" { + const S = Stream(ContinuationTestHandler); + + var aborted: S = .init(.{ .handler = .{} }); + defer aborted.deinit(); + aborted.nextSlice("\x1b[123"); + try testing.expectEqual( + @as(?usize, 1), + aborted.nextSliceUntilGround(&.{ 0x18, 'X' }), + ); + try testing.expect(aborted.ground()); + + var apc: S = .init(.{ .handler = .{} }); + defer apc.deinit(); + apc.nextSlice("\x1b_Gseed"); + var input: [131]u8 = undefined; + @memset(input[0..128], 'a'); + input[128..130].* = "\x1b\\".*; + input[130] = 'X'; + try testing.expectEqual( + @as(?usize, 130), + apc.nextSliceUntilGround(&input), + ); + try testing.expect(apc.ground()); +} + +test "stream: nextSliceUntilGround tracks only the consumed prefix" { + const S = Stream(ContinuationNullHandler); + var stream: S = .init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 64, + }); + defer stream.deinit(); + + stream.nextSlice("\x1b[31"); + try testing.expectEqual( + @as(?usize, 1), + stream.nextSliceUntilGround("mX\x1b["), + ); + + var buf: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buf); + try stream.writeContinuation(&writer); + try testing.expectEqual(@as(usize, 0), writer.buffered().len); + + stream.nextSlice("\x1b["); + try testing.expectEqual( + @as(?usize, null), + stream.nextSliceUntilGround("123"), + ); + var pending_writer: std.Io.Writer = .fixed(&buf); + try stream.writeContinuation(&pending_writer); + try testing.expectEqualStrings("\x1b[123", pending_writer.buffered()); +} + +test "stream: continuation lifecycle" { + const S = Stream(ContinuationTestHandler); + + var disabled: S = .init(.{ .handler = .{} }); + defer disabled.deinit(); + var disabled_buf: [1]u8 = undefined; + var disabled_writer: std.Io.Writer = .fixed(&disabled_buf); + try testing.expectError( + error.ContinuationDisabled, + disabled.writeContinuation(&disabled_writer), + ); + + var zero_capacity: S = .init(.{ + .handler = .{}, + .continuation_max_bytes = 0, + }); + defer zero_capacity.deinit(); + var zero_capacity_buf: [1]u8 = undefined; + var zero_capacity_writer: std.Io.Writer = .fixed(&zero_capacity_buf); + try testing.expectError( + error.ContinuationDisabled, + zero_capacity.writeContinuation(&zero_capacity_writer), + ); + var no_allocator: S = .init(.{ + .handler = .{}, + .continuation_max_bytes = 64, + }); + defer no_allocator.deinit(); + var no_allocator_buf: [1]u8 = undefined; + var no_allocator_writer: std.Io.Writer = .fixed(&no_allocator_buf); + try testing.expectError( + error.ContinuationDisabled, + no_allocator.writeContinuation(&no_allocator_writer), + ); + + var tracked = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 64, + }); + defer tracked.deinit(); + + tracked.nextSlice("complete input"); + var empty_buf: [1]u8 = undefined; + var empty_writer: std.Io.Writer = .fixed(&empty_buf); + try tracked.writeContinuation(&empty_writer); + try testing.expectEqual(@as(usize, 0), empty_writer.end); + + tracked.nextSlice("\x1b["); + var short_buf: [1]u8 = undefined; + var short_writer: std.Io.Writer = .fixed(&short_buf); + try testing.expectError( + error.WriteFailed, + tracked.writeContinuation(&short_writer), + ); + + var failing = testing.FailingAllocator.init(testing.allocator, .{ + .fail_index = 0, + }); + var failing_stream = Stream(ContinuationNullHandler).init(.{ + .allocator = failing.allocator(), + .handler = .{}, + .continuation_max_bytes = 64, + }); + defer failing_stream.deinit(); + failing_stream.nextSlice("\x1b["); + var unavailable_buf: [1]u8 = undefined; + var unavailable_writer: std.Io.Writer = .fixed(&unavailable_buf); + try testing.expectError( + error.ContinuationUnavailable, + failing_stream.writeContinuation(&unavailable_writer), + ); +} + +test "stream: continuation suffixes are replay safe" { + const Case = struct { + input: []const u8, + expected: []const u8, + }; + const cases = [_]Case{ + .{ .input = "text\x1b", .expected = "\x1b" }, + .{ .input = "text\x1b[12;", .expected = "\x1b[12;" }, + .{ .input = "text\x1b[1\x07;2", .expected = "\x1b[1;2" }, + .{ .input = "text\x1b]2;hello", .expected = "\x1b]2;hello" }, + .{ .input = "text\x1b_Gabc", .expected = "\x1b_Gabc" }, + .{ .input = "text\x1bP+qabc", .expected = "\x1bP+qabc" }, + .{ .input = "text\xE0\xA0\xF0", .expected = "\xF0" }, + .{ .input = "text\x1b[12\x1b", .expected = "\x1b" }, + .{ + .input = "text\x1b[12\x9D2;title", + .expected = "\x1b[12\x9D2;title", + }, + }; + + const S = Stream(ContinuationTestHandler); + for (cases) |case| { + var stream = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer stream.deinit(); + stream.nextSlice(case.input); + + var buf: [1024]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buf); + try stream.writeContinuation(&writer); + try testing.expectEqualStrings(case.expected, writer.buffered()); + } +} + +test "stream: continuation reconstructs every unfinished VT state" { + const Case = struct { + input: []const u8, + state: Parser.State, + }; + const cases = [_]Case{ + .{ .input = "\x1b", .state = .escape }, + .{ .input = "\x1b(", .state = .escape_intermediate }, + .{ .input = "\x1b[", .state = .csi_entry }, + .{ .input = "\x1b[1", .state = .csi_param }, + .{ .input = "\x1b[1$", .state = .csi_intermediate }, + .{ .input = "\x1b[:", .state = .csi_ignore }, + .{ .input = "\x1bP", .state = .dcs_entry }, + .{ .input = "\x1bP1", .state = .dcs_param }, + .{ .input = "\x1bP1$", .state = .dcs_intermediate }, + .{ .input = "\x1bP1q", .state = .dcs_passthrough }, + .{ .input = "\x1bP:", .state = .dcs_ignore }, + .{ .input = "\x1b]2;title", .state = .osc_string }, + .{ .input = "\x1b_Gpayload", .state = .sos_pm_apc_string }, + }; + const S = Stream(ContinuationTestHandler); + + for (cases) |case| { + var source = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer source.deinit(); + source.nextSlice(case.input); + try testing.expectEqual(case.state, source.parser.state); + + var buf: [1024]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buf); + try source.writeContinuation(&writer); + + var restored = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer restored.deinit(); + restored.nextSlice(writer.buffered()); + try testing.expectEqual(@as(usize, 0), restored.handler.committed); + try testing.expectEqual(source.parser.state, restored.parser.state); + try testing.expectEqual(source.utf8decoder.state, restored.utf8decoder.state); + } + + // Parser ground is still unfinished while the UTF-8 decoder is waiting + // for the remaining bytes of a codepoint. + var utf8 = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 4, + }); + defer utf8.deinit(); + utf8.next(0xF0); + try testing.expectEqual(Parser.State.ground, utf8.parser.state); + try testing.expect(utf8.utf8decoder.state != 0); + var utf8_buf: [4]u8 = undefined; + var utf8_writer: std.Io.Writer = .fixed(&utf8_buf); + try utf8.writeContinuation(&utf8_writer); + try testing.expectEqualSlices(u8, &.{0xF0}, utf8_writer.buffered()); +} + +test "stream: continuation is chunking-independent and idempotent" { + const S = Stream(ContinuationTestHandler); + const input = "committed\x1b[1\x07;2"; + + var bulk = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer bulk.deinit(); + bulk.nextSlice(input); + + var scalar = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer scalar.deinit(); + for (input) |c| scalar.next(c); + + var bulk_buf: [1024]u8 = undefined; + var bulk_writer: std.Io.Writer = .fixed(&bulk_buf); + try bulk.writeContinuation(&bulk_writer); + var scalar_buf: [1024]u8 = undefined; + var scalar_writer: std.Io.Writer = .fixed(&scalar_buf); + try scalar.writeContinuation(&scalar_writer); + try testing.expectEqualStrings( + bulk_writer.buffered(), + scalar_writer.buffered(), + ); + + var restored = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer restored.deinit(); + restored.nextSlice(bulk_writer.buffered()); + try testing.expectEqual(@as(usize, 0), restored.handler.committed); + + var restored_buf: [1024]u8 = undefined; + var restored_writer: std.Io.Writer = .fixed(&restored_buf); + try restored.writeContinuation(&restored_writer); + try testing.expectEqualStrings( + bulk_writer.buffered(), + restored_writer.buffered(), + ); + + bulk.handler.committed = 0; + restored.handler.committed = 0; + bulk.nextSlice("mZ"); + restored.next('m'); + restored.next('Z'); + try testing.expectEqual(bulk.handler.committed, restored.handler.committed); +} + +test "stream: continuation rebuilds APC handler input" { + const S = Stream(ContinuationTestHandler); + var source = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer source.deinit(); + source.nextSlice("committed\x1b_Gabc"); + + var continuation: [1024]u8 = undefined; + var writer: std.Io.Writer = .fixed(&continuation); + try source.writeContinuation(&writer); + + var restored = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer restored.deinit(); + restored.nextSlice(writer.buffered()); + try testing.expectEqual(@as(usize, 0), restored.handler.committed); + try testing.expect(restored.handler.apc_active); + try testing.expectEqualStrings( + source.handler.apc_buf[0..source.handler.apc_len], + restored.handler.apc_buf[0..restored.handler.apc_len], + ); + + source.handler.committed = 0; + restored.handler.committed = 0; + source.nextSlice("\x1b\\"); + restored.nextSlice("\x1b\\"); + try testing.expectEqual(source.handler.committed, restored.handler.committed); + try testing.expect(!source.handler.apc_active); + try testing.expect(!restored.handler.apc_active); +} + +test "stream: continuation cap and recovery" { + const S = Stream(ContinuationTestHandler); + + // The raw feed exceeds the cap, but only the unfinished three-byte + // CSI suffix is retained. + var seeded = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 4, + }); + defer seeded.deinit(); + seeded.nextSlice("committed text\x1b[1"); + var seeded_buf: [4]u8 = undefined; + var seeded_writer: std.Io.Writer = .fixed(&seeded_buf); + try seeded.writeContinuation(&seeded_writer); + try testing.expectEqualStrings("\x1b[1", seeded_writer.buffered()); + + var exceeded = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 4, + }); + defer exceeded.deinit(); + exceeded.nextSlice("\x1b[123"); + var unavailable_buf: [1]u8 = undefined; + var unavailable_writer: std.Io.Writer = .fixed(&unavailable_buf); + try testing.expectError( + error.ContinuationUnavailable, + exceeded.writeContinuation(&unavailable_writer), + ); + + // Completing the CSI reaches ground and recovers without rebuilding the + // Stream. A later unfinished sequence is tracked normally. + exceeded.nextSlice("mtext\x1b["); + var recovered_buf: [4]u8 = undefined; + var recovered_writer: std.Io.Writer = .fixed(&recovered_buf); + try exceeded.writeContinuation(&recovered_writer); + try testing.expectEqualStrings("\x1b[", recovered_writer.buffered()); + + // A fresh ESC seed also recovers broken tracking even when the stream + // never reaches ground: the ESC abandons the previous unfinished + // state and everything after it is retained. + exceeded.nextSlice("\x1b[123"); + var reexceeded_buf: [1]u8 = undefined; + var reexceeded_writer: std.Io.Writer = .fixed(&reexceeded_buf); + try testing.expectError( + error.ContinuationUnavailable, + exceeded.writeContinuation(&reexceeded_writer), + ); + exceeded.nextSlice("\x1b]0;"); + var seed_buf: [4]u8 = undefined; + var seed_writer: std.Io.Writer = .fixed(&seed_buf); + try exceeded.writeContinuation(&seed_writer); + try testing.expectEqualStrings("\x1b]0;", seed_writer.buffered()); +} + +test "stream: continuation spans multiple bulk feeds" { + const S = Stream(ContinuationTestHandler); + + // An unfinished APC grows across feeds that contain no new seed. + var apc = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer apc.deinit(); + apc.nextSlice("text\x1b_Gab"); + apc.nextSlice("cd"); + apc.nextSlice("ef"); + var apc_buf: [16]u8 = undefined; + var apc_writer: std.Io.Writer = .fixed(&apc_buf); + try apc.writeContinuation(&apc_writer); + try testing.expectEqualStrings("\x1b_Gabcdef", apc_writer.buffered()); + + // An incomplete UTF-8 sequence grows across feeds of its + // continuation bytes. + var utf8 = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 1024, + }); + defer utf8.deinit(); + utf8.nextSlice("text\xF0"); + utf8.nextSlice("\x9F"); + var utf8_buf: [4]u8 = undefined; + var utf8_writer: std.Io.Writer = .fixed(&utf8_buf); + try utf8.writeContinuation(&utf8_writer); + try testing.expectEqualSlices(u8, "\xF0\x9F", utf8_writer.buffered()); + + // A later feed with its own seed drops everything retained earlier. + utf8.nextSlice("\x98\x84 done \x1b[38;5"); + var seed_buf: [8]u8 = undefined; + var seed_writer: std.Io.Writer = .fixed(&seed_buf); + try utf8.writeContinuation(&seed_writer); + try testing.expectEqualStrings("\x1b[38;5", seed_writer.buffered()); +} + +test "stream: continuation exact cap and large unfinished string" { + const S = Stream(ContinuationNullHandler); + + var exact = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 5, + }); + defer exact.deinit(); + exact.nextSlice("\x1b[123"); + var exact_buf: [5]u8 = undefined; + var exact_writer: std.Io.Writer = .fixed(&exact_buf); + try exact.writeContinuation(&exact_writer); + try testing.expectEqualStrings("\x1b[123", exact_writer.buffered()); + + const payload_len = 12 * 1024; + const input = try testing.allocator.alloc(u8, payload_len); + defer testing.allocator.free(input); + input[0..3].* = "\x1b_G".*; + @memset(input[3..], 'A'); + + var large = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = payload_len, + }); + defer large.deinit(); + large.nextSlice(input); + const large_buf = try testing.allocator.alloc(u8, payload_len); + defer testing.allocator.free(large_buf); + var large_writer: std.Io.Writer = .fixed(large_buf); + try large.writeContinuation(&large_writer); + try testing.expectEqualSlices(u8, input, large_writer.buffered()); +} + +test "stream: continuation allocation failure recovers" { + var failing = testing.FailingAllocator.init(testing.allocator, .{}); + const alloc = failing.allocator(); + const S = Stream(ContinuationNullHandler); + var stream = S.init(.{ + .allocator = alloc, + .handler = .{}, + .continuation_max_bytes = 16 * 1024, + }); + defer stream.deinit(); + + var input = try testing.allocator.alloc(u8, 12 * 1024); + defer testing.allocator.free(input); + input[0..2].* = "\x1b[".*; + @memset(input[2..], '1'); + + failing.fail_index = failing.alloc_index; + stream.nextSlice(input); + var unavailable_buf: [1]u8 = undefined; + var unavailable_writer: std.Io.Writer = .fixed(&unavailable_buf); + try testing.expectError( + error.ContinuationUnavailable, + stream.writeContinuation(&unavailable_writer), + ); + + failing.fail_index = std.math.maxInt(usize); + stream.next('m'); + var recovered_buf: [1]u8 = undefined; + var recovered_writer: std.Io.Writer = .fixed(&recovered_buf); + try stream.writeContinuation(&recovered_writer); + try testing.expectEqual(@as(usize, 0), recovered_writer.end); +} + +test "stream: continuation every-byte cuts preserve future behavior" { + const corpora = [_][]const u8{ + "plain \xF0\x9F\x98\x84 utf8", + "bad \xE0\xA0\xF0\x9F\x98\x84 utf8", + "\x1b[1\x07;2mstyled\x1b[0m", + "\x1b]2;window title\x1b\\text", + "\x1bP$qm\x1b\\text", + "\x1b_Ga=q;payload\x1b\\text", + "\x1b_25a1;s\x1b\\text", + "\x1b]2;first\x1b\\\x1b_Gsecond", + "\x1b[12\x9D2;title\x1b\\text", + "\x1b[12\x18text\x1b[1\x1Atext", + }; + const S = Stream(ContinuationTestHandler); + + for (corpora) |corpus| for (0..corpus.len + 1) |cut| { + var source = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 64 * 1024, + }); + defer source.deinit(); + source.nextSlice(corpus[0..cut]); + + var continuation: [64 * 1024]u8 = undefined; + var continuation_writer: std.Io.Writer = .fixed(&continuation); + try source.writeContinuation(&continuation_writer); + + var restored = S.init(.{ + .allocator = testing.allocator, + .handler = .{}, + .continuation_max_bytes = 64 * 1024, + }); + defer restored.deinit(); + restored.nextSlice(continuation_writer.buffered()); + try testing.expectEqual(@as(usize, 0), restored.handler.committed); + try testing.expectEqual(source.handler.apc_active, restored.handler.apc_active); + try testing.expectEqual(source.handler.dcs_active, restored.handler.dcs_active); + if (source.handler.apc_active) { + try testing.expectEqualStrings( + source.handler.apc_buf[0..source.handler.apc_len], + restored.handler.apc_buf[0..restored.handler.apc_len], + ); + } + + var reexport: [64 * 1024]u8 = undefined; + var reexport_writer: std.Io.Writer = .fixed(&reexport); + try restored.writeContinuation(&reexport_writer); + try testing.expectEqualStrings( + continuation_writer.buffered(), + reexport_writer.buffered(), + ); + + source.handler.committed = 0; + restored.handler.committed = 0; + source.nextSlice(corpus[cut..]); + var offset = cut; + var partition = cut +% corpus.len +% 1; + while (offset < corpus.len) { + partition = partition *% 1664525 +% 1013904223; + const len = @min(1 + partition % 7, corpus.len - offset); + restored.nextSlice(corpus[offset..][0..len]); + offset += len; + } + try testing.expectEqual(source.handler.committed, restored.handler.committed); + try testing.expectEqual(source.handler.apc_active, restored.handler.apc_active); + try testing.expectEqual(source.handler.dcs_active, restored.handler.dcs_active); + + var source_final: [64]u8 = undefined; + var source_final_writer: std.Io.Writer = .fixed(&source_final); + try source.writeContinuation(&source_final_writer); + var restored_final: [64]u8 = undefined; + var restored_final_writer: std.Io.Writer = .fixed(&restored_final); + try restored.writeContinuation(&restored_final_writer); + try testing.expectEqualStrings( + source_final_writer.buffered(), + restored_final_writer.buffered(), + ); + }; +} |
