fix(docs): publish channel-aware documentation indexes

3 files changed, 221 insertions, 23 deletions

diff --git a/website/scripts/check-built-docs.mjs b/website/scripts/check-built-docs.mjs
index a838187a..5bda738e 100644
--- a/website/scripts/check-built-docs.mjs
+++ b/website/scripts/check-built-docs.mjs
@@ -72,13 +72,47 @@ if (nonCanonicalDocsUrl.test(sitemap)) {
 }
 
 const llmsIndex = await readFile(resolve(distDir, 'llms.txt'), 'utf8');
+const llmsPreview = await readFile(resolve(distDir, 'llms-preview.txt'), 'utf8');
 const llmsSmall = await readFile(resolve(distDir, 'llms-small.txt'), 'utf8');
 const llmsFull = await readFile(resolve(distDir, 'llms-full.txt'), 'utf8');
 const headers = await readFile(resolve(distDir, '_headers'), 'utf8');
+const currentDocs = versions.versions.find((entry) => entry.version === versions.current);
+if (!currentDocs?.tag || !currentDocs.source) {
+  throw new Error(`current docs version ${versions.current} is missing its tag or source`);
+}
+const stableRawRoot = `https://raw.githubusercontent.com/herdrdev/herdr/${currentDocs.tag}`;
+const stableRawBase = `${stableRawRoot}/${currentDocs.source}`;
+const previewRawBase = `https://raw.githubusercontent.com/herdrdev/herdr/${versions.preview.commit}/docs/next/website/src/content/docs`;
+assertIncludes(llmsIndex, `Current stable release: ${versions.current}.`);
+assertIncludes(llmsIndex, `${stableRawBase}/quick-start.mdx`);
+assertIncludes(
+  llmsIndex,
+  `${stableRawRoot}/${currentDocs.source.replace(/\/content\/docs$/, '/data/config-reference.json')}`,
+);
 assertIncludes(llmsIndex, 'https://herdr.dev/llms-small.txt');
 assertIncludes(llmsIndex, 'https://herdr.dev/llms-full.txt');
 assertIncludes(llmsIndex, 'https://herdr.dev/agent-guide.md');
-for (const path of ['/llms.txt', '/llms-small.txt', '/llms-full.txt']) {
+assertIncludes(llmsIndex, 'https://herdr.dev/llms-preview.txt');
+assertIncludes(llmsPreview, `Active preview build: ${versions.preview.build_id}`);
+assertIncludes(llmsPreview, `${previewRawBase}/quick-start.mdx`);
+assertIncludes(llmsPreview, 'https://herdr.dev/llms.txt');
+const stablePageLinks = llmsIndex
+  .split('\n')
+  .filter((line) => line.includes(`](${stableRawBase}/`));
+if (stablePageLinks.length !== versions.scopes.stable.locales.root.length) {
+  throw new Error(
+    `llms.txt lists ${stablePageLinks.length} stable pages, expected ${versions.scopes.stable.locales.root.length}`,
+  );
+}
+const previewPageLinks = llmsPreview
+  .split('\n')
+  .filter((line) => line.includes(`](${previewRawBase}/`));
+if (previewPageLinks.length !== versions.scopes.preview.locales.root.length) {
+  throw new Error(
+    `llms-preview.txt lists ${previewPageLinks.length} preview pages, expected ${versions.scopes.preview.locales.root.length}`,
+  );
+}
+for (const path of ['/llms.txt', '/llms-preview.txt', '/llms-small.txt', '/llms-full.txt']) {
   assertIncludes(headers, `${path}\n  Content-Type: text/markdown; charset=utf-8`);
 }
 for (const [name, content] of [
diff --git a/website/scripts/patch-starlight-llms.mjs b/website/scripts/patch-starlight-llms.mjs
index 16d3ba9b..3582b6f6 100644
--- a/website/scripts/patch-starlight-llms.mjs
+++ b/website/scripts/patch-starlight-llms.mjs
@@ -1,24 +1,54 @@
 import { readFile, writeFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 
-const path = resolve('node_modules/starlight-llms-txt/llms-full.txt.ts');
-const original = await readFile(path, 'utf8');
-const importLine = "import { starlightLllmsTxtContext } from 'virtual:starlight-llms-txt/context';";
-const excludeLine = '\t\texclude: starlightLllmsTxtContext.exclude,';
-
-if (original.includes(importLine) && original.includes(excludeLine)) process.exit(0);
-
-const withImport = original.replace(
-  "import { generateLlmsTxt } from './generator';",
-  `${importLine}\nimport { generateLlmsTxt } from './generator';`,
-);
-const patched = withImport.replace(
-  '\t\tdescription: `This is the full developer documentation for ${getSiteTitle()}`,',
-  '\t\tdescription: `This is the full developer documentation for ${getSiteTitle()}`,\n' + excludeLine,
-);
-
-if (patched === original || !patched.includes(importLine) || !patched.includes(excludeLine)) {
-  throw new Error('starlight-llms-txt changed: update the stable-only llms-full compatibility patch');
+await patchFullBundleExclusions();
+await disablePluginIndexRoute();
+
+async function patchFullBundleExclusions() {
+  const path = resolve('node_modules/starlight-llms-txt/llms-full.txt.ts');
+  const original = await readFile(path, 'utf8');
+  const importLine =
+    "import { starlightLllmsTxtContext } from 'virtual:starlight-llms-txt/context';";
+  const excludeLine = '\t\texclude: starlightLllmsTxtContext.exclude,';
+
+  if (original.includes(importLine) && original.includes(excludeLine)) return;
+
+  const withImport = original.replace(
+    "import { generateLlmsTxt } from './generator';",
+    `${importLine}\nimport { generateLlmsTxt } from './generator';`,
+  );
+  const patched = withImport.replace(
+    '\t\tdescription: `This is the full developer documentation for ${getSiteTitle()}`,',
+    '\t\tdescription: `This is the full developer documentation for ${getSiteTitle()}`,\n' +
+      excludeLine,
+  );
+
+  if (patched === original || !patched.includes(importLine) || !patched.includes(excludeLine)) {
+    throw new Error(
+      'starlight-llms-txt changed: update the stable-only llms-full compatibility patch',
+    );
+  }
+
+  await writeFile(path, patched, 'utf8');
 }
 
-await writeFile(path, patched, 'utf8');
+async function disablePluginIndexRoute() {
+  const path = resolve('node_modules/starlight-llms-txt/index.ts');
+  const original = await readFile(path, 'utf8');
+  const marker =
+    '\t\t\t\t\t\t\t// Herdr publishes channel-aware raw-source indexes from website/public.';
+  if (original.includes(marker)) return;
+
+  const route = `\t\t\t\t\t\t\tinjectRoute({
+\t\t\t\t\t\t\t\tentrypoint: new URL('./llms.txt.ts', import.meta.url),
+\t\t\t\t\t\t\t\tpattern: '/llms.txt',
+\t\t\t\t\t\t\t});`;
+  const patched = original.replace(route, marker);
+  if (patched === original || !patched.includes(marker)) {
+    throw new Error(
+      'starlight-llms-txt changed: update the custom llms.txt index compatibility patch',
+    );
+  }
+
+  await writeFile(path, patched, 'utf8');
+}
diff --git a/website/scripts/prepare-docs.mjs b/website/scripts/prepare-docs.mjs
index 074ef36d..55a463bb 100644
--- a/website/scripts/prepare-docs.mjs
+++ b/website/scripts/prepare-docs.mjs
@@ -53,8 +53,12 @@ if (process.argv[2] === '--rewrite-preview-doc-fixture') {
   if (unsupported.length > 0) {
     throw new Error('usage: node website/scripts/prepare-docs.mjs [--docs-only] [--draft]');
   }
-  if (!process.argv.includes('--docs-only')) await preparePublicAssets();
-  await prepareDocs({ draft: process.argv.includes('--draft') });
+  const docsOnly = process.argv.includes('--docs-only');
+  if (!docsOnly) await preparePublicAssets();
+  await prepareDocs({
+    draft: process.argv.includes('--draft'),
+    publishAgentIndexes: !docsOnly,
+  });
 }
 
 async function preparePublicAssets() {
@@ -86,7 +90,7 @@ async function preparePublicAssets() {
   }
 }
 
-async function prepareDocs({ draft }) {
+async function prepareDocs({ draft, publishAgentIndexes }) {
   const manifest = JSON.parse(await readFile(versionsManifestPath, 'utf8'));
   if (
     manifest.schema_version !== 1 ||
@@ -195,6 +199,136 @@ async function prepareDocs({ draft }) {
     `${JSON.stringify(configReferences, null, 2)}\n`,
     'utf8',
   );
+
+  if (publishAgentIndexes) {
+    if (typeof currentEntry.tag !== 'string' || typeof currentEntry.source !== 'string') {
+      throw new Error(`current docs version ${manifest.current} is missing its tag or source`);
+    }
+    const stableSourceDir = resolve(versionsDir, manifest.current, 'website/src/content/docs');
+    const previewRef = draft ? 'master' : previewManifest.commit;
+    const previewBuild = draft ? 'draft' : previewManifest.build_id;
+    await Promise.all([
+      writeAgentDocsIndex({
+        output: resolve(publicDir, 'llms.txt'),
+        heading: 'Herdr stable documentation index',
+        summary: `Current stable release: ${manifest.current}.`,
+        sourceDir: stableSourceDir,
+        ref: currentEntry.tag,
+        repositoryPath: currentEntry.source,
+        extraSections: [
+          '## Optional bundles',
+          '- [Abridged documentation](https://herdr.dev/llms-small.txt)',
+          '- [Complete documentation](https://herdr.dev/llms-full.txt)',
+          '',
+          '## Other resources',
+          '- [Agent guide](https://herdr.dev/agent-guide.md): help a human understand, set up, or troubleshoot Herdr',
+          '- [Preview documentation](https://herdr.dev/llms-preview.txt): use only for the preview channel',
+        ],
+      }),
+      writeAgentDocsIndex({
+        output: resolve(publicDir, 'llms-preview.txt'),
+        heading: 'Herdr preview documentation index',
+        summary: `Active preview build: ${previewBuild} (${previewRef.slice(0, 12)}).`,
+        sourceDir: previewDocsSourceDir,
+        ref: previewRef,
+        repositoryPath: 'docs/next/website/src/content/docs',
+        extraSections: [
+          '## Other resources',
+          '- [Stable documentation](https://herdr.dev/llms.txt): use unless the human is running the preview channel',
+          '- [Agent guide](https://herdr.dev/agent-guide.md): help a human understand, set up, or troubleshoot Herdr',
+        ],
+      }),
+    ]);
+  }
+}
+
+async function writeAgentDocsIndex({
+  output,
+  heading,
+  summary,
+  sourceDir,
+  ref,
+  repositoryPath,
+  extraSections,
+}) {
+  const pages = await collectAgentDocs(sourceDir);
+  const rawRoot = `https://raw.githubusercontent.com/herdrdev/herdr/${ref}`;
+  const rawBase = `${rawRoot}/${repositoryPath}`;
+  const configDataPath = repositoryPath.replace(
+    /\/content\/docs$/,
+    '/data/config-reference.json',
+  );
+  if (configDataPath === repositoryPath) {
+    throw new Error(`cannot derive config reference path from ${repositoryPath}`);
+  }
+
+  const lines = [
+    `# ${heading}`,
+    '',
+    `> ${summary}`,
+    '',
+    'Open only the pages relevant to the current task. Links return raw MDX from the exact documented source revision. For another topic, return to this index instead of following `/docs/` links inside a page, which serve human-facing HTML.',
+    '',
+    '## Documentation',
+    '',
+    ...pages.map(
+      ({ title, description, path }) =>
+        `- [${title}](${rawBase}/${path})${description ? `: ${description}` : ''}`,
+    ),
+    '',
+    '## Generated reference data',
+    '',
+    `- [Config reference data](${rawRoot}/${configDataPath}): every canonical config.toml key, type, default, and description`,
+    '',
+    ...extraSections,
+    '',
+  ];
+  await writeFile(output, lines.join('\n'), 'utf8');
+}
+
+async function collectAgentDocs(sourceDir) {
+  const pages = [];
+
+  async function walk(directory, prefix = '') {
+    for (const entry of await readdir(directory, { withFileTypes: true })) {
+      if (!prefix && entry.isDirectory() && ['ja', 'zh-cn'].includes(entry.name)) continue;
+      const path = join(directory, entry.name);
+      const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        await walk(path, relativePath);
+        continue;
+      }
+      if (!entry.isFile() || !['.md', '.mdx'].includes(extname(entry.name).toLowerCase())) {
+        continue;
+      }
+      const content = await readFile(path, 'utf8');
+      const title = frontmatterValue(content, 'title');
+      if (!title) throw new Error(`${path} is missing a frontmatter title`);
+      pages.push({
+        title,
+        description: frontmatterValue(content, 'description'),
+        path: relativePath,
+      });
+    }
+  }
+
+  await walk(sourceDir);
+  pages.sort((a, b) => {
+    if (a.path === 'index.mdx' || a.path === 'index.md') return -1;
+    if (b.path === 'index.mdx' || b.path === 'index.md') return 1;
+    return a.title.localeCompare(b.title, 'en');
+  });
+  return pages;
+}
+
+function frontmatterValue(content, key) {
+  const frontmatter = content.match(/^---\n([\s\S]*?)\n---(?:\n|$)/)?.[1];
+  const value = frontmatter?.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'))?.[1]?.trim();
+  if (!value) return undefined;
+  const quote = value[0];
+  return (quote === '"' || quote === "'") && value.endsWith(quote)
+    ? value.slice(1, -1)
+    : value;
 }
 
 async function copyPreparedDocs(sourceDir, destinationDir, rewrite, pathPrefix = '') {