feat: finish ghostty pane backend integration

1 files changed, 245 insertions, 0 deletions

diff --git a/vendor/libghostty-vt/include/ghostty/vt/key/encoder.h b/vendor/libghostty-vt/include/ghostty/vt/key/encoder.h
new file mode 100644
index 00000000..9d8282ce
--- /dev/null
+++ b/vendor/libghostty-vt/include/ghostty/vt/key/encoder.h
@@ -0,0 +1,245 @@
+/**
+ * @file encoder.h
+ *
+ * Key event encoding to terminal escape sequences.
+ */
+
+#ifndef GHOSTTY_VT_KEY_ENCODER_H
+#define GHOSTTY_VT_KEY_ENCODER_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <ghostty/vt/types.h>
+#include <ghostty/vt/allocator.h>
+#include <ghostty/vt/terminal.h>
+#include <ghostty/vt/key/event.h>
+
+/**
+ * Opaque handle to a key encoder instance.
+ *
+ * This handle represents a key encoder that converts key events into terminal
+ * escape sequences.
+ *
+ * @ingroup key
+ */
+typedef struct GhosttyKeyEncoderImpl *GhosttyKeyEncoder;
+
+/**
+ * Kitty keyboard protocol flags.
+ *
+ * Bitflags representing the various modes of the Kitty keyboard protocol.
+ * These can be combined using bitwise OR operations. Valid values all
+ * start with `GHOSTTY_KITTY_KEY_`.
+ *
+ * @ingroup key
+ */
+typedef uint8_t GhosttyKittyKeyFlags;
+
+/** Kitty keyboard protocol disabled (all flags off) */
+#define GHOSTTY_KITTY_KEY_DISABLED 0
+
+/** Disambiguate escape codes */
+#define GHOSTTY_KITTY_KEY_DISAMBIGUATE (1 << 0)
+
+/** Report key press and release events */
+#define GHOSTTY_KITTY_KEY_REPORT_EVENTS (1 << 1)
+
+/** Report alternate key codes */
+#define GHOSTTY_KITTY_KEY_REPORT_ALTERNATES (1 << 2)
+
+/** Report all key events including those normally handled by the terminal */
+#define GHOSTTY_KITTY_KEY_REPORT_ALL (1 << 3)
+
+/** Report associated text with key events */
+#define GHOSTTY_KITTY_KEY_REPORT_ASSOCIATED (1 << 4)
+
+/** All Kitty keyboard protocol flags enabled */
+#define GHOSTTY_KITTY_KEY_ALL (GHOSTTY_KITTY_KEY_DISAMBIGUATE | GHOSTTY_KITTY_KEY_REPORT_EVENTS | GHOSTTY_KITTY_KEY_REPORT_ALTERNATES | GHOSTTY_KITTY_KEY_REPORT_ALL | GHOSTTY_KITTY_KEY_REPORT_ASSOCIATED)
+
+/**
+ * macOS option key behavior.
+ *
+ * Determines whether the "option" key on macOS is treated as "alt" or not.
+ * See the Ghostty `macos-option-as-alt` configuration option for more details.
+ *
+ * @ingroup key
+ */
+typedef enum {
+    /** Option key is not treated as alt */
+    GHOSTTY_OPTION_AS_ALT_FALSE = 0,
+    /** Option key is treated as alt */
+    GHOSTTY_OPTION_AS_ALT_TRUE = 1,
+    /** Only left option key is treated as alt */
+    GHOSTTY_OPTION_AS_ALT_LEFT = 2,
+    /** Only right option key is treated as alt */
+    GHOSTTY_OPTION_AS_ALT_RIGHT = 3,
+} GhosttyOptionAsAlt;
+
+/**
+ * Key encoder option identifiers.
+ *
+ * These values are used with ghostty_key_encoder_setopt() to configure
+ * the behavior of the key encoder.
+ *
+ * @ingroup key
+ */
+typedef enum {
+    /** Terminal DEC mode 1: cursor key application mode (value: bool) */
+    GHOSTTY_KEY_ENCODER_OPT_CURSOR_KEY_APPLICATION = 0,
+    
+    /** Terminal DEC mode 66: keypad key application mode (value: bool) */
+    GHOSTTY_KEY_ENCODER_OPT_KEYPAD_KEY_APPLICATION = 1,
+    
+    /** Terminal DEC mode 1035: ignore keypad with numlock (value: bool) */
+    GHOSTTY_KEY_ENCODER_OPT_IGNORE_KEYPAD_WITH_NUMLOCK = 2,
+    
+    /** Terminal DEC mode 1036: alt sends escape prefix (value: bool) */
+    GHOSTTY_KEY_ENCODER_OPT_ALT_ESC_PREFIX = 3,
+    
+    /** xterm modifyOtherKeys mode 2 (value: bool) */
+    GHOSTTY_KEY_ENCODER_OPT_MODIFY_OTHER_KEYS_STATE_2 = 4,
+    
+    /** Kitty keyboard protocol flags (value: GhosttyKittyKeyFlags bitmask) */
+    GHOSTTY_KEY_ENCODER_OPT_KITTY_FLAGS = 5,
+    
+    /** macOS option-as-alt setting (value: GhosttyOptionAsAlt) */
+    GHOSTTY_KEY_ENCODER_OPT_MACOS_OPTION_AS_ALT = 6,
+} GhosttyKeyEncoderOption;
+
+/**
+ * Create a new key encoder instance.
+ *
+ * Creates a new key encoder with default options. The encoder can be configured
+ * using ghostty_key_encoder_setopt() and must be freed using
+ * ghostty_key_encoder_free() when no longer needed.
+ *
+ * @param allocator Pointer to the allocator to use for memory management, or NULL to use the default allocator
+ * @param encoder Pointer to store the created encoder handle
+ * @return GHOSTTY_SUCCESS on success, or an error code on failure
+ *
+ * @ingroup key
+ */
+GHOSTTY_API GhosttyResult ghostty_key_encoder_new(const GhosttyAllocator *allocator, GhosttyKeyEncoder *encoder);
+
+/**
+ * Free a key encoder instance.
+ *
+ * Releases all resources associated with the key encoder. After this call,
+ * the encoder handle becomes invalid and must not be used.
+ *
+ * @param encoder The encoder handle to free (may be NULL)
+ *
+ * @ingroup key
+ */
+GHOSTTY_API void ghostty_key_encoder_free(GhosttyKeyEncoder encoder);
+
+/**
+ * Set an option on the key encoder.
+ *
+ * Configures the behavior of the key encoder. Options control various aspects
+ * of encoding such as terminal modes (cursor key application mode, keypad mode),
+ * protocol selection (Kitty keyboard protocol flags), and platform-specific
+ * behaviors (macOS option-as-alt).
+ *
+ * If you are using a terminal instance, you can set the key encoding
+ * options based on the active terminal state (e.g. legacy vs Kitty mode
+ * and associated flags) with ghostty_key_encoder_setopt_from_terminal().
+ *
+ * A null pointer value does nothing. It does not reset the value to the
+ * default. The setopt call will do nothing.
+ *
+ * @param encoder The encoder handle, must not be NULL
+ * @param option The option to set
+ * @param value Pointer to the value to set (type depends on the option)
+ *
+ * @ingroup key
+ */
+GHOSTTY_API void ghostty_key_encoder_setopt(GhosttyKeyEncoder encoder, GhosttyKeyEncoderOption option, const void *value);
+
+/**
+ * Set encoder options from a terminal's current state.
+ *
+ * Reads the terminal's current modes and flags and applies them to the
+ * encoder's options. This sets cursor key application mode, keypad mode,
+ * alt escape prefix, modifyOtherKeys state, and Kitty keyboard protocol
+ * flags from the terminal state.
+ *
+ * Note that the `macos_option_as_alt` option cannot be determined from
+ * terminal state and is reset to `GHOSTTY_OPTION_AS_ALT_FALSE` by this
+ * call. Use ghostty_key_encoder_setopt() to set it afterward if needed.
+ *
+ * @param encoder The encoder handle, must not be NULL
+ * @param terminal The terminal handle, must not be NULL
+ *
+ * @ingroup key
+ */
+GHOSTTY_API void ghostty_key_encoder_setopt_from_terminal(GhosttyKeyEncoder encoder, GhosttyTerminal terminal);
+
+/**
+ * Encode a key event into a terminal escape sequence.
+ *
+ * Converts a key event into the appropriate terminal escape sequence based on
+ * the encoder's current options. The sequence is written to the provided buffer.
+ *
+ * Not all key events produce output. For example, unmodified modifier keys
+ * typically don't generate escape sequences. Check the out_len parameter to
+ * determine if any data was written.
+ *
+ * If the output buffer is too small, this function returns GHOSTTY_OUT_OF_SPACE
+ * and out_len will contain the required buffer size. The caller can then
+ * allocate a larger buffer and call the function again.
+ *
+ * @param encoder The encoder handle, must not be NULL
+ * @param event The key event to encode, must not be NULL
+ * @param out_buf Buffer to write the encoded sequence to
+ * @param out_buf_size Size of the output buffer in bytes
+ * @param out_len Pointer to store the number of bytes written (may be NULL)
+ * @return GHOSTTY_SUCCESS on success, GHOSTTY_OUT_OF_SPACE if buffer too small, or other error code
+ *
+ * ## Example: Calculate required buffer size
+ *
+ * @code{.c}
+ * // Query the required size with a NULL buffer (always returns OUT_OF_SPACE)
+ * size_t required = 0;
+ * GhosttyResult result = ghostty_key_encoder_encode(encoder, event, NULL, 0, &required);
+ * assert(result == GHOSTTY_OUT_OF_SPACE);
+ * 
+ * // Allocate buffer of required size
+ * char *buf = malloc(required);
+ * 
+ * // Encode with properly sized buffer
+ * size_t written = 0;
+ * result = ghostty_key_encoder_encode(encoder, event, buf, required, &written);
+ * assert(result == GHOSTTY_SUCCESS);
+ * 
+ * // Use the encoded sequence...
+ * 
+ * free(buf);
+ * @endcode
+ *
+ * ## Example: Direct encoding with static buffer
+ *
+ * @code{.c}
+ * // Most escape sequences are short, so a static buffer often suffices
+ * char buf[128];
+ * size_t written = 0;
+ * GhosttyResult result = ghostty_key_encoder_encode(encoder, event, buf, sizeof(buf), &written);
+ * 
+ * if (result == GHOSTTY_SUCCESS) {
+ *   // Write the encoded sequence to the terminal
+ *   write(pty_fd, buf, written);
+ * } else if (result == GHOSTTY_OUT_OF_SPACE) {
+ *   // Buffer too small, written contains required size
+ *   char *dynamic_buf = malloc(written);
+ *   result = ghostty_key_encoder_encode(encoder, event, dynamic_buf, written, &written);
+ *   assert(result == GHOSTTY_SUCCESS);
+ *   write(pty_fd, dynamic_buf, written);
+ *   free(dynamic_buf);
+ * }
+ * @endcode
+ *
+ * @ingroup key
+ */
+GHOSTTY_API GhosttyResult ghostty_key_encoder_encode(GhosttyKeyEncoder encoder, GhosttyKeyEvent event, char *out_buf, size_t out_buf_size, size_t *out_len);
+
+#endif /* GHOSTTY_VT_KEY_ENCODER_H */