diff options
| author | Ogulcan Celik <ogulcancelik@gmail.com> | 2026-09-01 14:48:45 +0300 |
|---|---|---|
| committer | Ogulcan Celik <ogulcancelik@gmail.com> | 2026-09-01 14:48:45 +0300 |
| commit | 8a6d697308162874a236c84aac0b0f8c7ac01f83 (patch) | |
| tree | ed7dc7e8c6f50d66bde721fae78576aede929f66 /website/scripts | |
| parent | b54a78aa510b87d1e3b56c619b72eaeb59e4a2a1 (diff) | |
refactor: separate website presentation from runtime
Diffstat (limited to 'website/scripts')
| -rw-r--r-- | website/scripts/check-built-docs.mjs | 178 | ||||
| -rw-r--r-- | website/scripts/docs-preview.integration.test.ts | 90 | ||||
| -rw-r--r-- | website/scripts/docs-preview.mjs | 75 | ||||
| -rw-r--r-- | website/scripts/docs-snapshot.mjs | 116 | ||||
| -rw-r--r-- | website/scripts/docs-versions.integration.test.ts | 191 | ||||
| -rw-r--r-- | website/scripts/docs-versions.mjs | 286 | ||||
| -rw-r--r-- | website/scripts/docs-versions.test.ts | 33 | ||||
| -rw-r--r-- | website/scripts/patch-starlight-llms.mjs | 54 | ||||
| -rw-r--r-- | website/scripts/prepare-docs.mjs | 479 |
9 files changed, 0 insertions, 1502 deletions
diff --git a/website/scripts/check-built-docs.mjs b/website/scripts/check-built-docs.mjs deleted file mode 100644 index f6a2f199..00000000 --- a/website/scripts/check-built-docs.mjs +++ /dev/null @@ -1,178 +0,0 @@ -import { access, readFile, readdir, stat } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const websiteDir = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -const distDir = resolve(websiteDir, 'dist'); -const nonCanonicalDocsUrl = /https:\/\/herdr\.dev\/(?:ja\/|zh-cn\/)?docs\/(?:preview|\d+\.\d+\.\d+)(?:\/|<)/; -const versions = JSON.parse( - await readFile(resolve(websiteDir, 'src/data/docs-versions.json'), 'utf8'), -); - -for (const entry of versions.versions) { - const scope = versions.scopes[entry.version]; - if (!scope) throw new Error(`missing generated scope for ${entry.version}`); - for (const [locale, pages] of Object.entries(scope.locales)) { - const localePrefix = locale === 'root' ? '' : `${locale}/`; - for (const page of pages) { - const output = resolve( - distDir, - localePrefix, - 'docs', - entry.version, - page, - 'index.html', - ); - await access(output); - } - } -} - -const stable = await readFile(resolve(distDir, 'docs/index.html'), 'utf8'); -const preview = await readFile(resolve(distDir, 'docs/preview/index.html'), 'utf8'); -const archived = await readFile( - resolve(distDir, 'docs', versions.current, 'index.html'), - 'utf8', -); - -assertIncludes(stable, 'data-pagefind-filter="version[content]" content="stable"'); -if (stable.includes('name="robots" content="noindex')) { - throw new Error('stable docs must remain indexable'); -} -assertIncludes(preview, 'data-pagefind-filter="version[content]" content="preview"'); -assertIncludes(preview, `Preview build <code dir="auto">${versions.preview.build_id}</code>`); -if (versions.preview.commit !== 'master') { - assertIncludes(preview, versions.preview.commit.slice(0, 12)); -} -assertIncludes(preview, 'name="robots" content="noindex, nofollow"'); -assertIncludes(archived, `data-pagefind-filter="version[content]" content="${versions.current}"`); -assertIncludes(archived, 'name="robots" content="noindex, nofollow"'); -if (archived.includes(`This page documents Herdr ${versions.current}`)) { - throw new Error('the current documentation version must not be labeled as outdated'); -} -const versionSelect = stable.match(/<select[^>]*aria-label="Documentation version"[^>]*>([\s\S]*?)<\/select>/)?.[1]; -if (!versionSelect) throw new Error('stable docs are missing the version selector'); -if (versionSelect.includes(`value="/docs/${versions.current}/"`)) { - throw new Error('the current release is duplicated in the version selector'); -} - -const previous = versions.versions.find((entry) => entry.version !== versions.current); -if (previous) { - const previousArchive = await readFile( - resolve(distDir, 'docs', previous.version, 'index.html'), - 'utf8', - ); - assertIncludes(previousArchive, `This page documents Herdr ${previous.version}`); -} - -const sitemap = await readFile(resolve(distDir, 'sitemap-0.xml'), 'utf8'); -assertIncludes(sitemap, 'https://herdr.dev/docs/'); -if (nonCanonicalDocsUrl.test(sitemap)) { - throw new Error('preview or versioned documentation URLs must not appear in the sitemap'); -} - -const llmsIndex = await readFile(resolve(distDir, 'llms.txt'), 'utf8'); -const llmsPreview = await readFile(resolve(distDir, 'llms-preview.txt'), 'utf8'); -const llmsSmall = await readFile(resolve(distDir, 'llms-small.txt'), 'utf8'); -const llmsFull = await readFile(resolve(distDir, 'llms-full.txt'), 'utf8'); -const headers = await readFile(resolve(distDir, '_headers'), 'utf8'); -const currentDocs = versions.versions.find((entry) => entry.version === versions.current); -if (!currentDocs?.tag || !currentDocs.source) { - throw new Error(`current docs version ${versions.current} is missing its tag or source`); -} -const stableRawRoot = `https://raw.githubusercontent.com/herdrdev/herdr/${currentDocs.tag}`; -const stableRawBase = `${stableRawRoot}/${currentDocs.source}`; -const stableConfigReferenceUrl = `${stableRawRoot}/${currentDocs.source.replace(/\/content\/docs$/, '/data/config-reference.json')}`; -const previewRawRoot = `https://raw.githubusercontent.com/herdrdev/herdr/${versions.preview.commit}`; -const previewRawBase = `${previewRawRoot}/docs/next/website/src/content/docs`; -const previewConfigReferenceUrl = `${previewRawRoot}/docs/next/website/src/data/config-reference.json`; -assertIncludes(llmsIndex, `Current stable release: ${versions.current}.`); -assertIncludes(llmsIndex, `${stableRawBase}/quick-start.mdx`); -assertIncludes(llmsIndex, `- [Config reference](${stableConfigReferenceUrl})`); -assertIncludes(llmsIndex, '## Using the config reference'); -assertIncludes(llmsIndex, "jq --arg key 'ui.sidebar_width'"); -if (llmsIndex.includes(`${stableRawBase}/config-reference.mdx`)) { - throw new Error('llms.txt must link to config reference data instead of the unrendered MDX component'); -} -assertIncludes(llmsIndex, 'https://herdr.dev/llms-small.txt'); -assertIncludes(llmsIndex, 'https://herdr.dev/llms-full.txt'); -assertIncludes(llmsIndex, 'https://herdr.dev/agent-guide.md'); -assertIncludes(llmsIndex, 'https://herdr.dev/llms-preview.txt'); -assertIncludes(llmsPreview, `Active preview build: ${versions.preview.build_id}`); -assertIncludes(llmsPreview, `${previewRawBase}/quick-start.mdx`); -assertIncludes(llmsPreview, `- [Config reference](${previewConfigReferenceUrl})`); -if (llmsPreview.includes(`${previewRawBase}/config-reference.mdx`)) { - throw new Error('llms-preview.txt must link to config reference data instead of the unrendered MDX component'); -} -assertIncludes(llmsPreview, 'https://herdr.dev/llms.txt'); -const stablePageLinks = llmsIndex - .split('\n') - .filter( - (line) => - line.includes(`](${stableRawBase}/`) || line.includes(`](${stableConfigReferenceUrl})`), - ); -if (stablePageLinks.length !== versions.scopes.stable.locales.root.length) { - throw new Error( - `llms.txt lists ${stablePageLinks.length} stable pages, expected ${versions.scopes.stable.locales.root.length}`, - ); -} -const previewPageLinks = llmsPreview - .split('\n') - .filter( - (line) => - line.includes(`](${previewRawBase}/`) || line.includes(`](${previewConfigReferenceUrl})`), - ); -if (previewPageLinks.length !== versions.scopes.preview.locales.root.length) { - throw new Error( - `llms-preview.txt lists ${previewPageLinks.length} preview pages, expected ${versions.scopes.preview.locales.root.length}`, - ); -} -for (const path of ['/llms.txt', '/llms-preview.txt', '/llms-small.txt', '/llms-full.txt']) { - assertIncludes(headers, `${path}\n Content-Type: text/markdown; charset=utf-8`); -} -for (const [name, content] of [ - ['llms-small.txt', llmsSmall], - ['llms-full.txt', llmsFull], -]) { - assertIncludes(content, '# Herdr documentation'); - assertIncludes(content, '# Troubleshooting'); - assertIncludes(content, '# Socket API'); - if (content.match(/^# Herdr documentation$/gm)?.length !== 1) { - throw new Error(`${name} must contain exactly one stable documentation set`); - } - if (/\]\(\/docs\/(?:preview|\d+\.\d+\.\d+)(?:\/|\))/.test(content)) { - throw new Error(`${name} must not link to preview or versioned documentation`); - } -} - -const build = await inspectFiles(distDir); -if (build.count > 20_000) { - throw new Error(`website build has ${build.count} files, exceeding the Cloudflare Pages free-plan limit`); -} -if (build.largest.bytes > 25 * 1024 * 1024) { - throw new Error(`website asset ${build.largest.path} is ${build.largest.bytes} bytes, exceeding Cloudflare Pages' 25 MiB limit`); -} -process.stdout.write(`validated ${versions.versions.length} documentation versions in ${build.count} website files\n`); - -function assertIncludes(content, expected) { - if (!content.includes(expected)) throw new Error(`built documentation is missing ${expected}`); -} - -async function inspectFiles(directory, root = directory) { - const result = { count: 0, largest: { path: '', bytes: 0 } }; - for (const entry of await readdir(directory, { withFileTypes: true })) { - const path = join(directory, entry.name); - if (entry.isDirectory()) { - const nested = await inspectFiles(path, root); - result.count += nested.count; - if (nested.largest.bytes > result.largest.bytes) result.largest = nested.largest; - } else if (entry.isFile()) { - result.count += 1; - const { size } = await stat(path); - if (size > result.largest.bytes) { - result.largest = { path: path.slice(root.length + 1), bytes: size }; - } - } - } - return result; -} diff --git a/website/scripts/docs-preview.integration.test.ts b/website/scripts/docs-preview.integration.test.ts deleted file mode 100644 index f850689c..00000000 --- a/website/scripts/docs-preview.integration.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import { execFileSync } from 'node:child_process'; -import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; -import { tmpdir } from 'node:os'; - -const script = resolve(import.meta.dir, 'docs-preview.mjs'); -const temporaryDirectories: string[] = []; - -afterEach(async () => { - await Promise.all( - temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true })), - ); -}); - -describe('preview documentation snapshots', () => { - test('snapshots and validates the selected commit exactly', async () => { - const root = await fixture(); - const commit = git(root, ['rev-parse', 'HEAD']).trim(); - await write(root, 'docs/preview/website/stale.mdx', 'stale\n'); - await writeManifest(root, commit); - - runScript(root, ['snapshot', commit]); - runScript(root, ['check']); - - expect(await read(root, 'docs/preview/website/src/content/docs/index.mdx')).toBe( - 'selected preview\n', - ); - await expect(read(root, 'docs/preview/website/stale.mdx')).rejects.toThrow(); - }); - - test('rejects snapshot drift', async () => { - const root = await fixture(); - const commit = git(root, ['rev-parse', 'HEAD']).trim(); - await writeManifest(root, commit); - runScript(root, ['snapshot', commit]); - await write(root, 'docs/preview/website/src/content/docs/index.mdx', 'changed\n'); - - expect(() => runScript(root, ['check'])).toThrow(); - }); -}); - -async function fixture() { - const root = await mkdtemp(join(tmpdir(), 'herdr-preview-docs-')); - temporaryDirectories.push(root); - await write(root, 'docs/next/website/src/content/docs/index.mdx', 'selected preview\n'); - await write(root, 'docs/next/website/src/data/config-reference.json', '{"preview":true}\n'); - git(root, ['init', '-q']); - git(root, ['config', 'user.email', 'test@example.com']); - git(root, ['config', 'user.name', 'Test']); - git(root, ['add', '.']); - git(root, ['commit', '-qm', 'preview fixture']); - return root; -} - -async function writeManifest(root: string, commit: string) { - await write( - root, - 'website/preview.json', - `${JSON.stringify({ - schema_version: 1, - channel: 'preview', - build_id: 'test-build', - commit, - builds: { 'test-build': { commit, tag: 'preview-test-build' } }, - })}\n`, - ); -} - -async function write(root: string, path: string, content: string) { - const destination = resolve(root, path); - await mkdir(resolve(destination, '..'), { recursive: true }); - await writeFile(destination, content, 'utf8'); -} - -async function read(root: string, path: string) { - return readFile(resolve(root, path), 'utf8'); -} - -function git(root: string, args: string[]) { - return execFileSync('git', args, { cwd: root, encoding: 'utf8', stdio: 'pipe' }); -} - -function runScript(root: string, args: string[]) { - execFileSync('node', [script, ...args], { - cwd: root, - env: { ...process.env, HERDR_DOCS_REPO_ROOT: root }, - stdio: 'pipe', - }); -} diff --git a/website/scripts/docs-preview.mjs b/website/scripts/docs-preview.mjs deleted file mode 100644 index c0de1fcd..00000000 --- a/website/scripts/docs-preview.mjs +++ /dev/null @@ -1,75 +0,0 @@ -import { readFile } from 'node:fs/promises'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { - compareGitTree, - createGit, - extractGitTree, - gitPathExists, - resolveCommit, -} from './docs-snapshot.mjs'; - -const websiteDir = dirname(fileURLToPath(import.meta.url)); -const repoRoot = process.env.HERDR_DOCS_REPO_ROOT - ? resolve(process.env.HERDR_DOCS_REPO_ROOT) - : resolve(websiteDir, '../..'); -const git = createGit(repoRoot); -const sourceRoot = 'docs/next/website'; -const snapshotRoot = resolve(repoRoot, 'docs/preview/website'); -const manifestPath = resolve(repoRoot, 'website/preview.json'); -const fullShaPattern = /^[0-9a-f]{40}$/; - -export async function snapshotPreview(ref) { - const commit = resolveCommit(git, ref); - if (!gitPathExists(git, commit, `${sourceRoot}/src/content/docs`)) { - throw new Error(`${commit} does not contain staged website documentation`); - } - if (!gitPathExists(git, commit, `${sourceRoot}/src/data/config-reference.json`)) { - throw new Error(`${commit} does not contain the staged config reference`); - } - await extractGitTree(git, commit, sourceRoot, snapshotRoot); - process.stdout.write(`snapshotted preview documentation from ${commit}\n`); -} - -export async function checkPreview() { - const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); - if (manifest.schema_version !== 1 || manifest.channel !== 'preview') { - throw new Error(`${manifestPath} has an unsupported schema`); - } - if (typeof manifest.commit !== 'string' || !fullShaPattern.test(manifest.commit)) { - throw new Error(`${manifestPath} must contain a full preview commit SHA`); - } - const build = manifest.builds?.[manifest.build_id]; - if (!build || build.commit !== manifest.commit) { - throw new Error(`${manifestPath} active build does not match its preview commit`); - } - const expectedTag = `preview-${manifest.build_id}`; - if (build.tag !== expectedTag) { - throw new Error(`${manifestPath} active build tag does not match ${expectedTag}`); - } - if (!gitPathExists(git, manifest.commit, `${sourceRoot}/src/content/docs`)) { - throw new Error(`${manifest.commit} does not contain staged website documentation`); - } - if (!gitPathExists(git, manifest.commit, `${sourceRoot}/src/data/config-reference.json`)) { - throw new Error(`${manifest.commit} does not contain the staged config reference`); - } - await compareGitTree(git, manifest.commit, sourceRoot, snapshotRoot); - process.stdout.write(`validated preview documentation snapshot ${manifest.commit}\n`); -} - -async function main() { - const [command, value] = process.argv.slice(2); - if (command === 'snapshot' && value) { - await snapshotPreview(value); - return; - } - if (command === 'check' && !value) { - await checkPreview(); - return; - } - throw new Error('usage: node website/scripts/docs-preview.mjs check | snapshot <commit>'); -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - await main(); -} diff --git a/website/scripts/docs-snapshot.mjs b/website/scripts/docs-snapshot.mjs deleted file mode 100644 index e9eaf92c..00000000 --- a/website/scripts/docs-snapshot.mjs +++ /dev/null @@ -1,116 +0,0 @@ -import { execFileSync } from 'node:child_process'; -import { mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; -import { dirname, relative, resolve } from 'node:path'; - -export function createGit(repoRoot) { - return function git(args, options = {}) { - return execFileSync('git', args, { - cwd: repoRoot, - encoding: options.binary ? undefined : 'utf8', - maxBuffer: 64 * 1024 * 1024, - stdio: ['ignore', 'pipe', 'pipe'], - }); - }; -} - -export function resolveCommit(git, ref) { - return git(['rev-parse', `${ref}^{commit}`]).trim(); -} - -export function gitPathExists(git, ref, path) { - try { - git(['cat-file', '-e', `${ref}:${path}`]); - return true; - } catch { - return false; - } -} - -export function gitTreesEqual(git, ref, left, right) { - try { - git(['diff', '--quiet', `${ref}:${left}`, `${ref}:${right}`]); - return true; - } catch { - return false; - } -} - -export function listGitEntries(git, ref, root) { - const output = git(['ls-tree', '-r', '-z', ref, '--', root]); - return output - .split('\0') - .filter(Boolean) - .map((line) => { - const separator = line.indexOf('\t'); - const [mode, type, object] = line.slice(0, separator).split(' '); - return { mode, type, object, path: line.slice(separator + 1) }; - }); -} - -export async function extractGitTree(git, ref, sourceRoot, destinationRoot) { - const entries = listGitEntries(git, ref, sourceRoot); - if (entries.length === 0) throw new Error(`${ref}:${sourceRoot} contains no files`); - - await rm(destinationRoot, { recursive: true, force: true }); - for (const entry of entries) { - assertOrdinaryFile(entry, ref); - const relativePath = relative(sourceRoot, entry.path); - assertRelativePath(relativePath, entry.path); - const destination = resolve(destinationRoot, relativePath); - await mkdir(dirname(destination), { recursive: true }); - await writeFile(destination, git(['cat-file', 'blob', entry.object], { binary: true })); - } -} - -export async function compareGitTree(git, ref, sourceRoot, snapshotRoot) { - const expectedEntries = listGitEntries(git, ref, sourceRoot); - if (expectedEntries.length === 0) throw new Error(`${ref}:${sourceRoot} contains no files`); - - const expected = new Map(); - for (const entry of expectedEntries) { - assertOrdinaryFile(entry, ref); - const relativePath = relative(sourceRoot, entry.path); - assertRelativePath(relativePath, entry.path); - expected.set(relativePath, entry); - } - const actualPaths = await listDocumentationPaths(snapshotRoot); - const expectedPaths = [...expected.keys()].sort(); - if (JSON.stringify(actualPaths) !== JSON.stringify(expectedPaths)) { - throw new Error(`${snapshotRoot} file list differs from ${ref}:${sourceRoot}`); - } - - for (const relativePath of expectedPaths) { - const entry = expected.get(relativePath); - const snapshotPath = resolve(snapshotRoot, relativePath); - const expectedContent = git(['cat-file', 'blob', entry.object], { binary: true }); - if (!(await readFile(snapshotPath)).equals(expectedContent)) { - throw new Error(`${relativePath} differs from ${ref}:${entry.path}`); - } - } -} - -export async function listDocumentationPaths(root) { - const paths = []; - async function walk(directory) { - for (const entry of await readdir(directory, { withFileTypes: true })) { - const path = resolve(directory, entry.name); - if (entry.isDirectory()) await walk(path); - else if (entry.isFile()) paths.push(relative(root, path)); - else throw new Error(`${path} is not an ordinary documentation file`); - } - } - await walk(root); - return paths.sort(); -} - -function assertOrdinaryFile(entry, ref) { - if (entry.type !== 'blob' || entry.mode !== '100644') { - throw new Error(`${ref}:${entry.path} is not an ordinary documentation file`); - } -} - -function assertRelativePath(relativePath, originalPath) { - if (!relativePath || relativePath === '..' || relativePath.startsWith(`..${process.platform === 'win32' ? '\\' : '/'}`)) { - throw new Error(`unsafe snapshot path ${originalPath}`); - } -} diff --git a/website/scripts/docs-versions.integration.test.ts b/website/scripts/docs-versions.integration.test.ts deleted file mode 100644 index 2eba45cb..00000000 --- a/website/scripts/docs-versions.integration.test.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import { execFileSync } from 'node:child_process'; -import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; -import { tmpdir } from 'node:os'; - -const script = resolve(import.meta.dir, 'docs-versions.mjs'); -const prepareScript = resolve(import.meta.dir, 'prepare-docs.mjs'); -const temporaryDirectories: string[] = []; - -afterEach(async () => { - await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true }))); -}); - -describe('documentation release publishing', () => { - test('snapshots tagged next docs and switches stable to generated snapshots', async () => { - const root = await mkdtemp(join(tmpdir(), 'herdr-docs-')); - temporaryDirectories.push(root); - await write(root, 'website/src/content/docs/index.mdx', 'stable docs\n'); - await write(root, 'website/src/data/config-reference.json', '{"stable":true}\n'); - await write(root, 'website/latest.json', '{"version":"0.9.0"}\n'); - await write(root, 'README.md', 'stable readme\n'); - await write(root, 'README.zh-CN.md', 'stable readme zh-cn\n'); - const nextDocs = '---\ntitle: Documentation\n---\n\nnext docs\n'; - await write(root, 'docs/next/website/src/content/docs/index.mdx', nextDocs); - await write(root, 'docs/next/website/src/data/config-reference.json', '{"next":true}\n'); - await write(root, 'docs/next/README.md', 'next readme\n'); - await write(root, 'docs/next/README.zh-CN.md', 'next readme zh-cn\n'); - - git(root, ['init', '-q']); - git(root, ['config', 'user.email', 'test@example.com']); - git(root, ['config', 'user.name', 'Test']); - git(root, ['add', '.']); - git(root, ['commit', '-qm', 'release fixture']); - git(root, ['tag', 'v0.9.0']); - git(root, ['tag', 'v1.0.0']); - - runScript(root, ['publish', 'v1.0.0']); - - await expect(read(root, 'website/src/content/docs/index.mdx')).rejects.toThrow(); - await expect(read(root, 'website/src/data/config-reference.json')).rejects.toThrow(); - expect(await read(root, 'README.md')).toBe('next readme\n'); - expect(await read(root, 'README.zh-CN.md')).toBe('next readme zh-cn\n'); - expect(await read(root, 'docs/versions/1.0.0/website/src/content/docs/index.mdx')).toBe(nextDocs); - - const manifest = JSON.parse(await read(root, 'docs/versions/manifest.json')); - expect(manifest.current).toBe('1.0.0'); - expect(manifest.stable_source).toBe('snapshot'); - expect(manifest.versions[0]).toMatchObject({ - version: '1.0.0', - tag: 'v1.0.0', - commit: expect.stringMatching(/^[0-9a-f]{40}$/), - source: 'docs/next/website/src/content/docs', - }); - - const previewCommit = git(root, ['rev-parse', 'HEAD']).trim(); - await write(root, 'docs/preview/website/src/content/docs/index.mdx', 'preview docs\n'); - await write( - root, - 'docs/preview/website/src/data/config-reference.json', - '{"preview":true}\n', - ); - await write( - root, - 'website/preview.json', - `${JSON.stringify({ build_id: 'preview-test', commit: previewCommit })}\n`, - ); - runPrepare(root); - const stableDocs = await read(root, 'website/src/content/docs/index.mdx'); - expect(stableDocs).toContain('next docs'); - expect(stableDocs).toContain( - 'editUrl: https://github.com/herdrdev/herdr/blob/master/docs/versions/1.0.0/website/src/content/docs/index.mdx', - ); - const versionedDocs = await read( - root, - 'website/src/content/docs/_versions/1.0.0/index.mdx', - ); - expect(versionedDocs).toContain('next docs'); - expect(versionedDocs).toContain( - 'editUrl: https://github.com/herdrdev/herdr/blob/master/docs/versions/1.0.0/website/src/content/docs/index.mdx', - ); - expect(await read(root, 'website/src/content/docs/preview/index.mdx')).toContain( - 'Preview build `preview-test`', - ); - expect(await read(root, 'website/src/data/config-reference.json')).toBe('{"next":true}\n'); - - await write(root, 'website/latest.json', '{"version":"1.0.0"}\n'); - runScript(root, ['check']); - - const correctedDocs = '---\ntitle: Documentation\n---\n\ncorrected docs\n'; - await write( - root, - 'docs/versions/1.0.0/website/src/content/docs/index.mdx', - correctedDocs, - ); - await write( - root, - 'docs/versions/1.0.0/website/src/data/config-reference.json', - '{"corrected":true}\n', - ); - runScript(root, ['check']); - - await write(root, 'README.md', 'post-release correction\n'); - runScript(root, ['publish', 'v1.0.0']); - expect(await read(root, 'README.md')).toBe('post-release correction\n'); - - runPrepare(root); - expect(await read(root, 'website/src/content/docs/index.mdx')).toContain('corrected docs'); - expect(await read(root, 'website/src/content/docs/_versions/1.0.0/index.mdx')).toContain( - 'corrected docs', - ); - expect(await read(root, 'website/src/data/config-reference.json')).toBe( - '{"corrected":true}\n', - ); - const versionReferences = JSON.parse( - await read(root, 'website/src/data/config-reference-versions.json'), - ); - expect(versionReferences['1.0.0']).toEqual({ corrected: true }); - - runScript(root, ['publish', 'v0.9.0']); - const archivedManifest = JSON.parse(await read(root, 'docs/versions/manifest.json')); - expect(archivedManifest.current).toBe('1.0.0'); - expect(archivedManifest.versions.map(({ version }) => version)).toEqual(['1.0.0', '0.9.0']); - expect(await read(root, 'README.md')).toBe('post-release correction\n'); - runScript(root, ['check']); - - const correctedArchivedDocs = '---\ntitle: Documentation\n---\n\ncorrected archived docs\n'; - await write( - root, - 'docs/versions/0.9.0/website/src/content/docs/index.mdx', - correctedArchivedDocs, - ); - runScript(root, ['publish', 'v0.9.0']); - expect(await read(root, 'docs/versions/0.9.0/website/src/content/docs/index.mdx')).toBe( - correctedArchivedDocs, - ); - - const archivedDocsRoot = resolve( - root, - 'docs/versions/0.9.0/website/src/content/docs', - ); - await rm(archivedDocsRoot, { recursive: true, force: true }); - expect(() => runScript(root, ['check'])).toThrow(); - await write( - root, - 'docs/versions/0.9.0/website/src/content/docs/index.mdx', - correctedArchivedDocs, - ); - const linkedPage = resolve(archivedDocsRoot, 'linked.mdx'); - await symlink('index.mdx', linkedPage); - expect(() => runScript(root, ['check'])).toThrow(); - await rm(linkedPage); - runScript(root, ['check']); - - delete archivedManifest.versions[0].commit; - await write(root, 'docs/versions/manifest.json', `${JSON.stringify(archivedManifest)}\n`); - expect(() => runScript(root, ['check'])).toThrow(); - // This test runs a dozen git and node subprocesses; slow CI builders can - // exceed the 5s default timeout. - }, 30_000); -}); - -async function write(root: string, path: string, content: string) { - const destination = resolve(root, path); - await mkdir(resolve(destination, '..'), { recursive: true }); - await writeFile(destination, content, 'utf8'); -} - -async function read(root: string, path: string) { - return readFile(resolve(root, path), 'utf8'); -} - -function git(root: string, args: string[]) { - return execFileSync('git', args, { cwd: root, encoding: 'utf8', stdio: 'pipe' }); -} - -function runScript(root: string, args: string[]) { - execFileSync('node', [script, ...args], { - cwd: root, - env: { ...process.env, HERDR_DOCS_REPO_ROOT: root }, - stdio: 'pipe', - }); -} - -function runPrepare(root: string) { - execFileSync('node', [prepareScript, '--docs-only'], { - cwd: root, - env: { ...process.env, HERDR_DOCS_REPO_ROOT: root }, - stdio: 'pipe', - }); -} diff --git a/website/scripts/docs-versions.mjs b/website/scripts/docs-versions.mjs deleted file mode 100644 index 0923d683..00000000 --- a/website/scripts/docs-versions.mjs +++ /dev/null @@ -1,286 +0,0 @@ -import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { - createGit, - extractGitTree, - gitPathExists, - gitTreesEqual, - listDocumentationPaths, - resolveCommit, -} from './docs-snapshot.mjs'; - -const websiteDir = dirname(fileURLToPath(import.meta.url)); -const repoRoot = process.env.HERDR_DOCS_REPO_ROOT - ? resolve(process.env.HERDR_DOCS_REPO_ROOT) - : resolve(websiteDir, '../..'); -const versionsDir = resolve(repoRoot, 'docs/versions'); -const manifestPath = resolve(versionsDir, 'manifest.json'); -const stableDocsDir = resolve(repoRoot, 'website/src/content/docs'); -const stableReferencePath = resolve(repoRoot, 'website/src/data/config-reference.json'); -const git = createGit(repoRoot); - -const VERSION_PATTERN = /^v?(\d+\.\d+\.\d+)$/; - -export function normalizeVersion(value) { - const match = VERSION_PATTERN.exec(value); - if (!match) throw new Error(`version must look like 0.7.5 or v0.7.5, got ${value}`); - return match[1]; -} - -export function sortVersionsNewestFirst(versions) { - return [...versions].sort((left, right) => compareVersions(right.version, left.version)); -} - -function compareVersions(left, right) { - const a = left.split('.').map(Number); - const b = right.split('.').map(Number); - for (let index = 0; index < 3; index += 1) { - if (a[index] !== b[index]) return a[index] - b[index]; - } - return 0; -} - -async function readManifest() { - try { - return JSON.parse(await readFile(manifestPath, 'utf8')); - } catch (error) { - if (error.code === 'ENOENT') { - return { schema_version: 1, stable_source: 'legacy', current: null, versions: [] }; - } - throw error; - } -} - -async function writeManifest(manifest) { - manifest.versions = sortVersionsNewestFirst(manifest.versions).map( - ({ version, tag, commit, source }) => ({ - version, - tag, - ...(commit ? { commit } : {}), - source, - }), - ); - await mkdir(dirname(manifestPath), { recursive: true }); - await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); -} - -function releaseMetadata(tag) { - const version = normalizeVersion(tag); - const normalizedTag = `v${version}`; - return { version, tag: normalizedTag, commit: resolveCommit(git, normalizedTag) }; -} - -async function snapshotTag(tag, sourceRoot) { - const metadata = { ...releaseMetadata(tag), source: sourceRoot }; - const destination = resolve(versionsDir, metadata.version); - await rm(destination, { recursive: true, force: true }); - await extractGitTree(git, tag, sourceRoot, resolve(destination, 'website/src/content/docs')); - - const referenceSource = sourceRoot.includes('docs/next/') - ? 'docs/next/website/src/data/config-reference.json' - : 'website/src/data/config-reference.json'; - if (gitPathExists(git, tag, referenceSource)) { - const referenceDestination = resolve(destination, 'website/src/data/config-reference.json'); - await mkdir(dirname(referenceDestination), { recursive: true }); - await writeFile(referenceDestination, git(['show', `${tag}:${referenceSource}`], { binary: true })); - } - - return metadata; -} - -export async function backfillVersions() { - const tags = git(['tag', '--list', 'v*', '--sort=v:refname']).split('\n').filter(Boolean); - const manifest = await readManifest(); - const entries = new Map(manifest.versions.map((entry) => [entry.version, entry])); - - for (const tag of tags) { - const version = normalizeVersion(tag); - if (entries.has(version)) continue; - const nextRoot = 'docs/next/website/src/content/docs'; - const legacyRoot = 'website/src/content/docs'; - const hasNext = gitPathExists(git, tag, nextRoot); - const hasLegacy = gitPathExists(git, tag, legacyRoot); - if (!hasNext && !hasLegacy) continue; - const sourceRoot = - hasNext && (!hasLegacy || !gitTreesEqual(git, tag, legacyRoot, nextRoot)) - ? nextRoot - : legacyRoot; - const metadata = await snapshotTag(tag, sourceRoot); - entries.set(metadata.version, metadata); - process.stdout.write(`snapshotted ${tag}\n`); - } - - const current = JSON.parse(await readFile(resolve(repoRoot, 'website/latest.json'), 'utf8')).version; - await writeManifest({ - schema_version: 1, - stable_source: manifest.stable_source ?? 'legacy', - current: normalizeVersion(current), - versions: [...entries.values()], - }); -} - -export async function checkVersions() { - const manifest = await readManifest(); - if ( - manifest.schema_version !== 1 || - typeof manifest.current !== 'string' || - !['legacy', 'snapshot'].includes(manifest.stable_source ?? 'legacy') - ) { - throw new Error(`${manifestPath} has an unsupported schema`); - } - - const latest = JSON.parse(await readFile(resolve(repoRoot, 'website/latest.json'), 'utf8')).version; - if (manifest.current !== normalizeVersion(latest)) { - throw new Error(`docs current version ${manifest.current} does not match website latest ${latest}`); - } - - const expectedOrder = sortVersionsNewestFirst(manifest.versions).map(({ version }) => version); - if (JSON.stringify(manifest.versions.map(({ version }) => version)) !== JSON.stringify(expectedOrder)) { - throw new Error('documentation versions are not sorted newest first'); - } - - const seen = new Set(); - for (const entry of manifest.versions) { - if (seen.has(entry.version)) throw new Error(`duplicate docs version ${entry.version}`); - seen.add(entry.version); - if (normalizeVersion(entry.tag) !== entry.version) { - throw new Error(`docs version ${entry.version} has mismatched tag ${entry.tag}`); - } - const requiresCommit = - entry.source === 'docs/next/website/src/content/docs' || - ((manifest.stable_source ?? 'legacy') === 'snapshot' && entry.version === manifest.current); - if (requiresCommit && !entry.commit) { - throw new Error(`docs version ${entry.version} is missing commit provenance`); - } - if (entry.commit) { - if (!/^[0-9a-f]{40}$/.test(entry.commit)) { - throw new Error(`docs version ${entry.version} has invalid commit ${entry.commit}`); - } - const taggedCommit = resolveCommit(git, entry.tag); - if (taggedCommit !== entry.commit) { - throw new Error( - `docs version ${entry.version} tag ${entry.tag} moved from ${entry.commit} to ${taggedCommit}`, - ); - } - } - if (!['website/src/content/docs', 'docs/next/website/src/content/docs'].includes(entry.source)) { - throw new Error(`docs version ${entry.version} has unsupported source ${entry.source}`); - } - - const versionRoot = resolve(versionsDir, entry.version, 'website'); - const documentationPaths = await listDocumentationPaths( - resolve(versionRoot, 'src/content/docs'), - ); - if (!documentationPaths.includes('index.md') && !documentationPaths.includes('index.mdx')) { - throw new Error(`docs version ${entry.version} has no root index`); - } - - try { - JSON.parse(await readFile(resolve(versionRoot, 'src/data/config-reference.json'), 'utf8')); - } catch (error) { - if (error.code !== 'ENOENT') throw error; - } - } - - if (!seen.has(manifest.current)) { - throw new Error(`current docs version ${manifest.current} has no snapshot`); - } - process.stdout.write(`validated ${manifest.versions.length} documentation versions\n`); -} - -export async function publishVersion(tag) { - const release = releaseMetadata(tag); - if (!gitPathExists(git, tag, 'docs/next/website/src/content/docs')) { - throw new Error(`${tag} does not contain staged website documentation`); - } - - const manifest = await readManifest(); - const existing = manifest.versions.find((entry) => entry.version === release.version); - if (manifest.current === release.version) { - if ((manifest.stable_source ?? 'legacy') === 'legacy') { - process.stdout.write(`documentation ${release.tag} is already published with the legacy stable source\n`); - return; - } - if (!existing?.commit || existing.commit !== release.commit) { - throw new Error(`published documentation ${release.version} has mismatched commit provenance`); - } - process.stdout.write(`documentation ${release.tag} is already published\n`); - return; - } - if (manifest.current && compareVersions(release.version, manifest.current) < 0) { - if (existing) { - if (existing.tag !== release.tag) { - throw new Error(`version ${release.version} is already associated with ${existing.tag}`); - } - if (existing.commit && existing.commit !== release.commit) { - throw new Error(`version ${release.version} has mismatched commit provenance`); - } - process.stdout.write(`documentation ${release.tag} is already archived\n`); - return; - } - const archived = await snapshotTag(tag, 'docs/next/website/src/content/docs'); - const archivedEntries = new Map(manifest.versions.map((entry) => [entry.version, entry])); - archivedEntries.set(archived.version, archived); - await writeManifest({ - ...manifest, - versions: [...archivedEntries.values()], - }); - process.stdout.write( - `archived documentation snapshot ${release.tag} without changing current ${manifest.current}\n`, - ); - return; - } - if (existing) { - throw new Error(`version ${release.version} already has published documentation`); - } - - const metadata = await snapshotTag(tag, 'docs/next/website/src/content/docs'); - await rm(stableDocsDir, { recursive: true, force: true }); - await rm(stableReferencePath, { force: true }); - - for (const readme of ['README.md', 'README.zh-CN.md']) { - const nextReadme = `docs/next/${readme}`; - if (gitPathExists(git, tag, nextReadme)) { - await writeFile(resolve(repoRoot, readme), git(['show', `${tag}:${nextReadme}`], { binary: true })); - } - } - - const entries = new Map(manifest.versions.map((entry) => [entry.version, entry])); - entries.set(metadata.version, metadata); - await writeManifest({ - schema_version: 1, - stable_source: 'snapshot', - current: metadata.version, - versions: [...entries.values()], - }); - process.stdout.write(`published documentation snapshot ${metadata.tag}\n`); -} - -async function main() { - const [command, value] = process.argv.slice(2); - if (command === 'backfill' && !value) { - await backfillVersions(); - return; - } - if (command === 'publish' && value) { - await publishVersion(value); - return; - } - if (command === 'check' && !value) { - await checkVersions(); - return; - } - if (command === 'current' && !value) { - const manifest = await readManifest(); - process.stdout.write(`${manifest.current ?? ''}\n`); - return; - } - throw new Error( - 'usage: node website/scripts/docs-versions.mjs backfill | check | current | publish <tag>', - ); -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - await main(); -} diff --git a/website/scripts/docs-versions.test.ts b/website/scripts/docs-versions.test.ts deleted file mode 100644 index dc450b37..00000000 --- a/website/scripts/docs-versions.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { describe, expect, test } from 'bun:test'; -import { normalizeVersion, sortVersionsNewestFirst } from './docs-versions.mjs'; - -describe('normalizeVersion', () => { - test.each([ - ['0.7.5', '0.7.5'], - ['v0.7.5', '0.7.5'], - ])('normalizes %s', (input, expected) => { - expect(normalizeVersion(input)).toBe(expected); - }); - - test('rejects non-release refs', () => { - expect(() => normalizeVersion('preview-123')).toThrow(); - }); -}); - -describe('sortVersionsNewestFirst', () => { - test('sorts semantic versions numerically', () => { - const versions = [ - { version: '0.6.10' }, - { version: '0.7.2' }, - { version: '0.7.10' }, - { version: '0.5.12' }, - ]; - - expect(sortVersionsNewestFirst(versions).map(({ version }) => version)).toEqual([ - '0.7.10', - '0.7.2', - '0.6.10', - '0.5.12', - ]); - }); -}); diff --git a/website/scripts/patch-starlight-llms.mjs b/website/scripts/patch-starlight-llms.mjs deleted file mode 100644 index 3582b6f6..00000000 --- a/website/scripts/patch-starlight-llms.mjs +++ /dev/null @@ -1,54 +0,0 @@ -import { readFile, writeFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; - -await patchFullBundleExclusions(); -await disablePluginIndexRoute(); - -async function patchFullBundleExclusions() { - const path = resolve('node_modules/starlight-llms-txt/llms-full.txt.ts'); - const original = await readFile(path, 'utf8'); - const importLine = - "import { starlightLllmsTxtContext } from 'virtual:starlight-llms-txt/context';"; - const excludeLine = '\t\texclude: starlightLllmsTxtContext.exclude,'; - - if (original.includes(importLine) && original.includes(excludeLine)) return; - - const withImport = original.replace( - "import { generateLlmsTxt } from './generator';", - `${importLine}\nimport { generateLlmsTxt } from './generator';`, - ); - const patched = withImport.replace( - '\t\tdescription: `This is the full developer documentation for ${getSiteTitle()}`,', - '\t\tdescription: `This is the full developer documentation for ${getSiteTitle()}`,\n' + - excludeLine, - ); - - if (patched === original || !patched.includes(importLine) || !patched.includes(excludeLine)) { - throw new Error( - 'starlight-llms-txt changed: update the stable-only llms-full compatibility patch', - ); - } - - await writeFile(path, patched, 'utf8'); -} - -async function disablePluginIndexRoute() { - const path = resolve('node_modules/starlight-llms-txt/index.ts'); - const original = await readFile(path, 'utf8'); - const marker = - '\t\t\t\t\t\t\t// Herdr publishes channel-aware raw-source indexes from website/public.'; - if (original.includes(marker)) return; - - const route = `\t\t\t\t\t\t\tinjectRoute({ -\t\t\t\t\t\t\t\tentrypoint: new URL('./llms.txt.ts', import.meta.url), -\t\t\t\t\t\t\t\tpattern: '/llms.txt', -\t\t\t\t\t\t\t});`; - const patched = original.replace(route, marker); - if (patched === original || !patched.includes(marker)) { - throw new Error( - 'starlight-llms-txt changed: update the custom llms.txt index compatibility patch', - ); - } - - await writeFile(path, patched, 'utf8'); -} diff --git a/website/scripts/prepare-docs.mjs b/website/scripts/prepare-docs.mjs deleted file mode 100644 index 1fc8bc56..00000000 --- a/website/scripts/prepare-docs.mjs +++ /dev/null @@ -1,479 +0,0 @@ -import { cp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; -import { dirname, extname, join, relative, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import process from 'node:process'; - -const websiteDir = dirname(fileURLToPath(import.meta.url)); -const repoRoot = process.env.HERDR_DOCS_REPO_ROOT - ? resolve(process.env.HERDR_DOCS_REPO_ROOT) - : resolve(websiteDir, '../..'); -const publicDir = resolve(repoRoot, 'website/public'); -const stableDocsDir = resolve(repoRoot, 'website/src/content/docs'); -const previewDocsDir = resolve(stableDocsDir, 'preview'); -const generatedVersionsDocsDir = resolve(stableDocsDir, '_versions'); -const versionsDir = resolve(repoRoot, 'docs/versions'); -const versionsManifestPath = resolve(versionsDir, 'manifest.json'); -const previewManifestPath = resolve(repoRoot, 'website/preview.json'); -const generatedVersionsDataPath = resolve(repoRoot, 'website/src/data/docs-versions.json'); -const stableConfigReferenceDestination = resolve( - repoRoot, - 'website/src/data/config-reference.json', -); -const previewConfigReferenceDestination = resolve( - repoRoot, - 'website/src/data/config-reference-preview.json', -); -const generatedVersionReferencesPath = resolve( - repoRoot, - 'website/src/data/config-reference-versions.json', -); - -if (process.argv[2] === '--rewrite-preview-doc-fixture') { - const chunks = []; - for await (const chunk of process.stdin) chunks.push(chunk); - process.stdout.write( - rewritePreviewDocContent(Buffer.concat(chunks).toString('utf8'), '', { - buildId: '2026-07-29-44b3adb12552', - commit: '44b3adb125524ea9a55739eee3776f922f2115ad', - }), - ); -} else if (process.argv[2] === '--rewrite-version-doc-fixture') { - const chunks = []; - for await (const chunk of process.stdin) chunks.push(chunk); - process.stdout.write( - rewriteVersionDocContent(Buffer.concat(chunks).toString('utf8'), { - version: process.argv[3] ?? '0.7.5', - tag: `v${process.argv[3] ?? '0.7.5'}`, - relativePath: 'index.mdx', - }), - ); -} else { - const supported = new Set(['--docs-only', '--draft']); - const unsupported = process.argv.slice(2).filter((argument) => !supported.has(argument)); - if (unsupported.length > 0) { - throw new Error('usage: node website/scripts/prepare-docs.mjs [--docs-only] [--draft]'); - } - const docsOnly = process.argv.includes('--docs-only'); - if (!docsOnly) await preparePublicAssets(); - await prepareDocs({ - draft: process.argv.includes('--draft'), - publishAgentIndexes: !docsOnly, - }); -} - -async function preparePublicAssets() { - await rm(publicDir, { recursive: true, force: true }); - await mkdir(publicDir, { recursive: true }); - - for (const file of [ - 'install.sh', - 'install.ps1', - 'install.cmd', - 'agent-guide.md', - 'latest.json', - 'preview.json', - 'robots.txt', - 'favicon.ico', - '_headers', - '_redirects', - ]) { - const source = resolve(repoRoot, 'website', file); - try { - await cp(source, resolve(publicDir, file)); - } catch (error) { - if (file !== 'preview.json' || error.code !== 'ENOENT') throw error; - } - } - - for (const directory of ['assets', 'css', 'agent-detection']) { - await cp(resolve(repoRoot, 'website', directory), resolve(publicDir, directory), { - recursive: true, - }); - } -} - -async function prepareDocs({ draft, publishAgentIndexes }) { - const manifest = JSON.parse(await readFile(versionsManifestPath, 'utf8')); - if ( - manifest.schema_version !== 1 || - typeof manifest.current !== 'string' || - !['legacy', 'snapshot'].includes(manifest.stable_source ?? 'legacy') - ) { - throw new Error(`${versionsManifestPath} has an unsupported schema`); - } - const currentEntry = manifest.versions.find((entry) => entry.version === manifest.current); - if (!currentEntry) throw new Error(`current docs version ${manifest.current} has no snapshot`); - - const previewManifest = JSON.parse(await readFile(previewManifestPath, 'utf8')); - if (!/^[0-9a-f]{40}$/.test(previewManifest.commit ?? '')) { - throw new Error(`${previewManifestPath} must contain a full preview commit SHA`); - } - const previewWebsiteRoot = resolve(repoRoot, draft ? 'docs/next/website' : 'docs/preview/website'); - const previewDocsSourceDir = resolve(previewWebsiteRoot, 'src/content/docs'); - const previewConfigReferenceSource = resolve( - previewWebsiteRoot, - 'src/data/config-reference.json', - ); - - if ((manifest.stable_source ?? 'legacy') === 'snapshot') { - await rm(stableDocsDir, { recursive: true, force: true }); - const currentSnapshotRoot = resolve(versionsDir, manifest.current, 'website'); - await copyPreparedDocs( - resolve(currentSnapshotRoot, 'src/content/docs'), - stableDocsDir, - (content, relativePath) => - rewriteStableDocContent(content, { - version: currentEntry.version, - tag: currentEntry.tag, - relativePath, - }), - ); - const currentReference = resolve(currentSnapshotRoot, 'src/data/config-reference.json'); - try { - await cp(currentReference, stableConfigReferenceDestination); - } catch (error) { - if (error.code !== 'ENOENT') throw error; - await rm(stableConfigReferenceDestination, { force: true }); - } - } else { - await rm(previewDocsDir, { recursive: true, force: true }); - await rm(generatedVersionsDocsDir, { recursive: true, force: true }); - } - - await copyPreparedDocs(previewDocsSourceDir, previewDocsDir, (content, relativePath) => - rewritePreviewDocContent(content, relativePath, { - buildId: draft ? 'draft' : previewManifest.build_id, - commit: draft ? 'master' : previewManifest.commit, - }), - ); - await cp(previewConfigReferenceSource, previewConfigReferenceDestination); - - const scopes = { - stable: await collectDocsScope( - (manifest.stable_source ?? 'legacy') === 'snapshot' - ? resolve(versionsDir, manifest.current, 'website/src/content/docs') - : stableDocsDir, - new Set(['preview', '_versions']), - ), - preview: await collectDocsScope(previewDocsSourceDir), - }; - const configReferences = {}; - - for (const entry of manifest.versions) { - const version = entry.version; - const snapshotDocsRoot = resolve(versionsDir, version, 'website/src/content/docs'); - const destinationRoot = resolve(generatedVersionsDocsDir, version); - await copyPreparedDocs(snapshotDocsRoot, destinationRoot, (content, relativePath) => - rewriteVersionDocContent(content, { - version, - tag: entry.tag, - relativePath, - }), - ); - scopes[version] = await collectDocsScope(snapshotDocsRoot); - - const referencePath = resolve( - versionsDir, - version, - 'website/src/data/config-reference.json', - ); - try { - configReferences[version] = JSON.parse(await readFile(referencePath, 'utf8')); - } catch (error) { - if (error.code !== 'ENOENT') throw error; - } - } - - await writeFile( - generatedVersionsDataPath, - `${JSON.stringify({ - ...manifest, - preview: { - build_id: draft ? 'draft' : previewManifest.build_id, - commit: draft ? 'master' : previewManifest.commit, - }, - scopes, - }, null, 2)}\n`, - 'utf8', - ); - await writeFile( - generatedVersionReferencesPath, - `${JSON.stringify(configReferences, null, 2)}\n`, - 'utf8', - ); - - if (publishAgentIndexes) { - if (typeof currentEntry.tag !== 'string' || typeof currentEntry.source !== 'string') { - throw new Error(`current docs version ${manifest.current} is missing its tag or source`); - } - const stableSourceDir = resolve(versionsDir, manifest.current, 'website/src/content/docs'); - const previewRef = draft ? 'master' : previewManifest.commit; - const previewBuild = draft ? 'draft' : previewManifest.build_id; - await Promise.all([ - writeAgentDocsIndex({ - output: resolve(publicDir, 'llms.txt'), - heading: 'Herdr stable documentation index', - summary: `Current stable release: ${manifest.current}.`, - sourceDir: stableSourceDir, - ref: currentEntry.tag, - repositoryPath: currentEntry.source, - extraSections: [ - '## Optional bundles', - '- [Abridged documentation](https://herdr.dev/llms-small.txt)', - '- [Complete documentation](https://herdr.dev/llms-full.txt)', - '', - '## Other resources', - '- [Agent guide](https://herdr.dev/agent-guide.md): help a human understand, set up, or troubleshoot Herdr', - '- [Preview documentation](https://herdr.dev/llms-preview.txt): use only for the preview channel', - ], - }), - writeAgentDocsIndex({ - output: resolve(publicDir, 'llms-preview.txt'), - heading: 'Herdr preview documentation index', - summary: `Active preview build: ${previewBuild} (${previewRef.slice(0, 12)}).`, - sourceDir: previewDocsSourceDir, - ref: previewRef, - repositoryPath: 'docs/next/website/src/content/docs', - extraSections: [ - '## Other resources', - '- [Stable documentation](https://herdr.dev/llms.txt): use unless the human is running the preview channel', - '- [Agent guide](https://herdr.dev/agent-guide.md): help a human understand, set up, or troubleshoot Herdr', - ], - }), - ]); - } -} - -async function writeAgentDocsIndex({ - output, - heading, - summary, - sourceDir, - ref, - repositoryPath, - extraSections, -}) { - const pages = await collectAgentDocs(sourceDir); - const rawRoot = `https://raw.githubusercontent.com/herdrdev/herdr/${ref}`; - const rawBase = `${rawRoot}/${repositoryPath}`; - const configDataPath = repositoryPath.replace( - /\/content\/docs$/, - '/data/config-reference.json', - ); - if (configDataPath === repositoryPath) { - throw new Error(`cannot derive config reference path from ${repositoryPath}`); - } - const configDataUrl = `${rawRoot}/${configDataPath}`; - - const lines = [ - `# ${heading}`, - '', - `> ${summary}`, - '', - 'Open only the pages relevant to the current task. Links return raw source files from the exact documented revision. For another topic, return to this index instead of following `/docs/` links inside a page, which serve human-facing HTML.', - '', - '## Documentation', - '', - ...pages.map(({ title, description, path }) => { - const url = /^config-reference\.mdx?$/.test(path) ? configDataUrl : `${rawBase}/${path}`; - return `- [${title}](${url})${description ? `: ${description}` : ''}`; - }), - '', - '## Using the config reference', - '', - 'The Config reference link above is a large structured JSON file. When command tools are available, fetch and filter it by exact key instead of loading the whole file:', - '', - '```sh', - `curl -fsSL '${configDataUrl}' | jq --arg key 'ui.sidebar_width' \\`, - " '.sections[].keys[] | select(.key == $key)'", - '```', - '', - ...extraSections, - '', - ]; - await writeFile(output, lines.join('\n'), 'utf8'); -} - -async function collectAgentDocs(sourceDir) { - const pages = []; - - async function walk(directory, prefix = '') { - for (const entry of await readdir(directory, { withFileTypes: true })) { - if (!prefix && entry.isDirectory() && ['ja', 'zh-cn'].includes(entry.name)) continue; - const path = join(directory, entry.name); - const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - await walk(path, relativePath); - continue; - } - if (!entry.isFile() || !['.md', '.mdx'].includes(extname(entry.name).toLowerCase())) { - continue; - } - const content = await readFile(path, 'utf8'); - const title = frontmatterValue(content, 'title'); - if (!title) throw new Error(`${path} is missing a frontmatter title`); - pages.push({ - title, - description: frontmatterValue(content, 'description'), - path: relativePath, - }); - } - } - - await walk(sourceDir); - pages.sort((a, b) => { - if (a.path === 'index.mdx' || a.path === 'index.md') return -1; - if (b.path === 'index.mdx' || b.path === 'index.md') return 1; - return a.title.localeCompare(b.title, 'en'); - }); - return pages; -} - -function frontmatterValue(content, key) { - const frontmatter = content.match(/^---\n([\s\S]*?)\n---(?:\n|$)/)?.[1]; - const value = frontmatter?.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'))?.[1]?.trim(); - if (!value) return undefined; - const quote = value[0]; - return (quote === '"' || quote === "'") && value.endsWith(quote) - ? value.slice(1, -1) - : value; -} - -async function copyPreparedDocs(sourceDir, destinationDir, rewrite, pathPrefix = '') { - await mkdir(destinationDir, { recursive: true }); - for (const entry of await readdir(sourceDir, { withFileTypes: true })) { - const source = join(sourceDir, entry.name); - const destination = join(destinationDir, entry.name); - const relativePath = pathPrefix ? `${pathPrefix}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - await copyPreparedDocs(source, destination, rewrite, relativePath); - continue; - } - if (!entry.isFile()) continue; - - if (!['.md', '.mdx'].includes(extname(entry.name).toLowerCase())) { - await cp(source, destination); - continue; - } - const content = await readFile(source, 'utf8'); - await writeFile(destination, rewrite(content, relativePath), 'utf8'); - } -} - -async function collectDocsScope(sourceDir, excludedDirectories = new Set()) { - const locales = { root: [], ja: [], 'zh-cn': [] }; - - async function walk(directory, prefix = '') { - for (const entry of await readdir(directory, { withFileTypes: true })) { - if (entry.isDirectory() && excludedDirectories.has(entry.name)) continue; - const path = join(directory, entry.name); - const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - await walk(path, relativePath); - continue; - } - if (!entry.isFile() || !['.md', '.mdx'].includes(extname(entry.name))) continue; - - const segments = relativePath.replace(/\.(md|mdx)$/i, '').split('/'); - const locale = segments[0] === 'ja' || segments[0] === 'zh-cn' ? segments.shift() : 'root'; - const page = segments.join('/').replace(/(^|\/)index$/, '').replace(/\/$/, ''); - locales[locale].push(page); - } - } - - await walk(sourceDir); - for (const pages of Object.values(locales)) pages.sort(); - for (const locale of ['ja', 'zh-cn']) { - if (locales[locale].length === 0) locales[locale] = [...locales.root]; - } - return { locales }; -} - -export function rewritePreviewDocContent( - content, - relativePath = '', - { buildId, commit } = { - buildId: 'preview', - commit: 'master', - }, -) { - const taggedContent = rewriteRepositoryLinks( - content.replaceAll('/docs/', '/docs/preview/'), - commit, - ); - const rewritten = rewriteRelativeDocPaths(taggedContent, 1); - const withSourceLink = setGeneratedEditUrl( - rewritten, - `https://github.com/herdrdev/herdr/blob/${commit}/docs/next/website/src/content/docs/${relativePath}`, - ); - return insertPreviewNotice(withSourceLink, relativePath, { buildId, commit }); -} - -export function rewriteStableDocContent(content, { version, tag, relativePath }) { - const taggedContent = rewriteRepositoryLinks(content, tag); - return setGeneratedEditUrl(taggedContent, versionedDocSourceUrl(version, relativePath)); -} - -export function rewriteVersionDocContent(content, { version, tag, relativePath }) { - const taggedContent = rewriteRepositoryLinks( - content.replaceAll('/docs/', `/docs/${version}/`), - tag, - ); - const rewritten = rewriteRelativeDocPaths(taggedContent, 2); - return setGeneratedEditUrl(rewritten, versionedDocSourceUrl(version, relativePath)); -} - -function versionedDocSourceUrl(version, relativePath) { - return `https://github.com/herdrdev/herdr/blob/master/docs/versions/${version}/website/src/content/docs/${relativePath}`; -} - -function rewriteRepositoryLinks(content, ref) { - return content - .replaceAll( - 'https://github.com/herdrdev/herdr/blob/master/', - `https://github.com/herdrdev/herdr/blob/${ref}/`, - ) - .replaceAll( - 'https://raw.githubusercontent.com/herdrdev/herdr/master/', - `https://raw.githubusercontent.com/herdrdev/herdr/${ref}/`, - ); -} - -function rewriteRelativeDocPaths(content, extraDepth) { - const parents = '../'.repeat(extraDepth); - return content - .replace(/((?:\.\.\/)+)(?=public\/)/g, `$1${parents}`) - .replace(/^(import .*from\s+['"])(?=(?:\.\.\/)+components\/)/gm, `$1${parents}`); -} - -function setGeneratedEditUrl(content, editUrl) { - if (!content.startsWith('---\n') || /^editUrl:/m.test(content)) return content; - return content.replace(/^---\n/, `---\neditUrl: ${editUrl}\n`); -} - -function insertPreviewNotice(content, relativePath, { buildId, commit }) { - const source = commit === 'master' ? '`master` draft' : `[${commit.slice(0, 12)}](https://github.com/herdrdev/herdr/commit/${commit})`; - const notice = [ - `> Preview build \`${buildId}\`, published from ${source}. Stable docs remain at [/docs/](/docs/).`, - '', - '', - ].join('\n'); - const indexPrefix = - relativePath === 'index.mdx' - ? content.replace('title: Herdr documentation', 'title: Herdr preview documentation') - : content; - const frontmatter = indexPrefix.match(/^---\n[\s\S]*?\n---\n/); - if (!frontmatter) { - return insertNoticeAfterImports(indexPrefix, notice); - } - const body = indexPrefix.slice(frontmatter[0].length); - return `${frontmatter[0]}\n${insertNoticeAfterImports(body, notice)}`; -} - -function insertNoticeAfterImports(content, notice) { - const imports = content.match(/^(\s*import .+?;\n)+\s*/); - if (!imports) { - return `${notice}${content}`; - } - return `${imports[0]}${notice}${content.slice(imports[0].length)}`; -} |
