From 92edd91f4702e4bcd58b9355093a8dcbf0e7eda6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 21:36:58 +0000 Subject: [PATCH 1/5] Initial plan From 00ee844cccc513326cf9e40c7f6f657db334570e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 21:54:06 +0000 Subject: [PATCH 2/5] Add llms.txt generation feature with documentation and changeset Co-authored-by: mikebarkmin <2592379+mikebarkmin@users.noreply.github.com> --- .changeset/llms-txt-generation.md | 6 + packages/hyperbook/build.ts | 110 ++++++++++++++++++ packages/types/src/index.ts | 1 + .../vscode/schemas/hyperbook.schema.json | 3 + website/de/book/configuration/book.md | 1 + website/en/book/configuration/book.md | 1 + 6 files changed, 122 insertions(+) create mode 100644 .changeset/llms-txt-generation.md diff --git a/.changeset/llms-txt-generation.md b/.changeset/llms-txt-generation.md new file mode 100644 index 000000000..fbe4eeea2 --- /dev/null +++ b/.changeset/llms-txt-generation.md @@ -0,0 +1,6 @@ +--- +"hyperbook": minor +"@hyperbook/types": minor +--- + +Add llms.txt file generation feature. When the `llms` property is set to `true` in hyperbook.json, a `llms.txt` file will be generated during build that combines all markdown files in order. The file includes the book name and version in the header. Pages and sections with `hide: true` are automatically excluded from the generated file. diff --git a/packages/hyperbook/build.ts b/packages/hyperbook/build.ts index bfd24b991..d1988857b 100644 --- a/packages/hyperbook/build.ts +++ b/packages/hyperbook/build.ts @@ -19,6 +19,104 @@ import packageJson from "./package.json"; export const ASSETS_FOLDER = "__hyperbook_assets"; +/** + * Generates an llms.txt file by combining all markdown files in order + */ +async function generateLlmsTxt( + root: string, + rootOut: string, + hyperbookJson: any, + pagesAndSections: Pick, + version: string, +): Promise { + const lines: string[] = []; + + // Add header with book name and version + lines.push(`${hyperbookJson.name} - Version ${version}`); + lines.push(""); // Empty line after header + + // Helper function to recursively process sections and pages + const processSection = async ( + section: any, + level: number = 0, + ): Promise => { + // Skip if hidden + if (section.hide) { + return; + } + + // Add section header if it has content + if (section.href && !section.isEmpty) { + const files = await vfile.listForFolder(root, "book"); + const file = files.find((f) => f.path.href === section.href); + if (file) { + // Add section name as a header + lines.push(`# ${section.name}`); + lines.push(""); + + // Get the markdown content without frontmatter + const content = file.markdown.content.trim(); + if (content) { + lines.push(content); + lines.push(""); // Empty line after content + } + } + } + + // Process nested pages + if (section.pages) { + for (const page of section.pages) { + await processPage(page); + } + } + + // Process nested sections + if (section.sections) { + for (const subsection of section.sections) { + await processSection(subsection, level + 1); + } + } + }; + + const processPage = async (page: any): Promise => { + // Skip if hidden or empty + if (page.hide || page.isEmpty) { + return; + } + + if (page.href) { + const files = await vfile.listForFolder(root, "book"); + const file = files.find((f) => f.path.href === page.href); + if (file) { + // Add page name as a header + lines.push(`# ${page.name}`); + lines.push(""); + + // Get the markdown content without frontmatter + const content = file.markdown.content.trim(); + if (content) { + lines.push(content); + lines.push(""); // Empty line after content + } + } + } + }; + + // Process root-level pages first + for (const page of pagesAndSections.pages) { + await processPage(page); + } + + // Process sections + for (const section of pagesAndSections.sections) { + await processSection(section); + } + + // Write the llms.txt file + const llmsTxtContent = lines.join("\n"); + await fs.writeFile(path.join(rootOut, "llms.txt"), llmsTxtContent); +} + export async function runBuildProject( project: Hyperproject, rootProject: Hyperproject, @@ -598,5 +696,17 @@ const SEARCH_DOCUMENTS = ${JSON.stringify(documents)}; ), ); + // Generate llms.txt if enabled + if (hyperbookJson.llms) { + console.log(`${chalk.blue(`[${prefix}]`)} Generating llms.txt`); + await generateLlmsTxt( + root, + rootOut, + hyperbookJson, + pagesAndSections, + packageJson.version, + ); + } + console.log(`${chalk.green(`[${prefix}]`)} Build success: ${rootOut}`); } diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 93395f646..172d05886 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -93,6 +93,7 @@ export type HyperbookJson = { search?: boolean; qrcode?: boolean; toc?: boolean; + llms?: boolean; author?: { name?: string; url?: string; diff --git a/platforms/vscode/schemas/hyperbook.schema.json b/platforms/vscode/schemas/hyperbook.schema.json index f9384d637..d397c7a69 100644 --- a/platforms/vscode/schemas/hyperbook.schema.json +++ b/platforms/vscode/schemas/hyperbook.schema.json @@ -280,6 +280,9 @@ }, "type": "array" }, + "llms": { + "type": "boolean" + }, "logo": { "type": "string" }, diff --git a/website/de/book/configuration/book.md b/website/de/book/configuration/book.md index d59cc6992..891d78676 100644 --- a/website/de/book/configuration/book.md +++ b/website/de/book/configuration/book.md @@ -38,6 +38,7 @@ von Optionen, die du definieren kannst. Optionen mit einem "\*" müssen gesetzt | allowDangerousHtml | Erlaube HTML im Hyperbook. Dies kann zu Inkompatibilität in zukünftigen Versionen führen. | | qrcode | Zeigt ein Icon, um einen QR-Code zur aktuellen Seite anzuzeigen. | | toc | Zeige ein Inhaltsverzeichnis. Diese ist standardmäßig aktiviert für Seiten und deaktiviert für Begriffe im Glossar. | +| llms | Wenn auf true gesetzt, wird eine llms.txt-Datei generiert, die alle Markdown-Dateien in Reihenfolge kombiniert. Die Datei enthält den Buchnamen und die Version im Header-Format. | | trailingSlash | Exportiert alle Datei in eigene Verzeichnisse und erzeugt nur index.html Dateien. | | importExport | Ermöglicht das Importieren und Exportieren des Zustands des Hyperbooks als Datei. Schaltflächen zum Importieren und Exportieren befinden sich am unteren Rand der Seite. | diff --git a/website/en/book/configuration/book.md b/website/en/book/configuration/book.md index b72a9ba83..14a4e4699 100644 --- a/website/en/book/configuration/book.md +++ b/website/en/book/configuration/book.md @@ -37,6 +37,7 @@ can and part wise must set (indicated by a \*). | allowDangerousHtml | Allow HTML. This can lead to incompatibilities in future versions. | | qrcode | Shows an icon, which opens a qr code to the current page. | | toc | Show or hide a table of content for the page. This is on for pages and off for glossary entries by default | +| llms | When set to true, generates an llms.txt file that combines all markdown files in order. The file includes the book name and version in a header format. | | trailingSlash | Outputs all files into ther own folders and produces only index.html files. | | importExport | Allows to import and export the state of the Hyperbook as a file. Buttons for importing and exporting will be at the bottom of the page. | From c4901118354a33ec1e6439583095216ed1c90f7b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 22:00:12 +0000 Subject: [PATCH 3/5] Address code review comments - improve type safety and optimize file system operations Co-authored-by: mikebarkmin <2592379+mikebarkmin@users.noreply.github.com> --- packages/hyperbook/build.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/hyperbook/build.ts b/packages/hyperbook/build.ts index d1988857b..bffb10c9b 100644 --- a/packages/hyperbook/build.ts +++ b/packages/hyperbook/build.ts @@ -11,6 +11,9 @@ import { Link, Hyperproject, HyperbookContext, + HyperbookJson, + HyperbookPage, + HyperbookSection, Navigation, } from "@hyperbook/types"; import lunr from "lunr"; @@ -25,7 +28,7 @@ export const ASSETS_FOLDER = "__hyperbook_assets"; async function generateLlmsTxt( root: string, rootOut: string, - hyperbookJson: any, + hyperbookJson: HyperbookJson, pagesAndSections: Pick, version: string, ): Promise { @@ -35,9 +38,12 @@ async function generateLlmsTxt( lines.push(`${hyperbookJson.name} - Version ${version}`); lines.push(""); // Empty line after header + // Get all book files once to avoid repeated file system operations + const allFiles = await vfile.listForFolder(root, "book"); + // Helper function to recursively process sections and pages const processSection = async ( - section: any, + section: HyperbookSection, level: number = 0, ): Promise => { // Skip if hidden @@ -47,8 +53,7 @@ async function generateLlmsTxt( // Add section header if it has content if (section.href && !section.isEmpty) { - const files = await vfile.listForFolder(root, "book"); - const file = files.find((f) => f.path.href === section.href); + const file = allFiles.find((f) => f.path.href === section.href); if (file) { // Add section name as a header lines.push(`# ${section.name}`); @@ -78,15 +83,14 @@ async function generateLlmsTxt( } }; - const processPage = async (page: any): Promise => { + const processPage = async (page: HyperbookPage): Promise => { // Skip if hidden or empty if (page.hide || page.isEmpty) { return; } if (page.href) { - const files = await vfile.listForFolder(root, "book"); - const file = files.find((f) => f.path.href === page.href); + const file = allFiles.find((f) => f.path.href === page.href); if (file) { // Add page name as a header lines.push(`# ${page.name}`); From b4d8a1e45e6bede7107b029b9051fff112f415b9 Mon Sep 17 00:00:00 2001 From: Mike Barkmin Date: Tue, 4 Nov 2025 23:02:33 +0100 Subject: [PATCH 4/5] activate llms --- website/de/hyperbook.json | 1 + website/en/hyperbook.json | 1 + 2 files changed, 2 insertions(+) diff --git a/website/de/hyperbook.json b/website/de/hyperbook.json index eae6d5a51..0da130f09 100644 --- a/website/de/hyperbook.json +++ b/website/de/hyperbook.json @@ -2,6 +2,7 @@ "name": "Hyperbook Dokumenation", "qrcode": true, "search": true, + "llms": true, "importExport": true, "description": "Dokumentation für Hyperbook erstellt mit Hyperbook", "author": { diff --git a/website/en/hyperbook.json b/website/en/hyperbook.json index 0e675e2d0..14b585c91 100644 --- a/website/en/hyperbook.json +++ b/website/en/hyperbook.json @@ -1,6 +1,7 @@ { "name": "Hyperbook Documentation", "qrcode": true, + "llms": true, "importExport": true, "search": true, "description": "Documentation for Hyperbook created with Hyperbook", From 16385192e67144c57cce1bdbaade0271241864d9 Mon Sep 17 00:00:00 2001 From: Mike Barkmin Date: Tue, 4 Nov 2025 23:26:57 +0100 Subject: [PATCH 5/5] add more instructions to llms.txt --- packages/hyperbook/build.ts | 81 ++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/packages/hyperbook/build.ts b/packages/hyperbook/build.ts index bffb10c9b..6fd3315c4 100644 --- a/packages/hyperbook/build.ts +++ b/packages/hyperbook/build.ts @@ -33,14 +33,14 @@ async function generateLlmsTxt( version: string, ): Promise { const lines: string[] = []; - + // Add header with book name and version lines.push(`${hyperbookJson.name} - Version ${version}`); lines.push(""); // Empty line after header - + // Get all book files once to avoid repeated file system operations const allFiles = await vfile.listForFolder(root, "book"); - + // Helper function to recursively process sections and pages const processSection = async ( section: HyperbookSection, @@ -50,7 +50,7 @@ async function generateLlmsTxt( if (section.hide) { return; } - + // Add section header if it has content if (section.href && !section.isEmpty) { const file = allFiles.find((f) => f.path.href === section.href); @@ -58,7 +58,7 @@ async function generateLlmsTxt( // Add section name as a header lines.push(`# ${section.name}`); lines.push(""); - + // Get the markdown content without frontmatter const content = file.markdown.content.trim(); if (content) { @@ -67,14 +67,14 @@ async function generateLlmsTxt( } } } - + // Process nested pages if (section.pages) { for (const page of section.pages) { await processPage(page); } } - + // Process nested sections if (section.sections) { for (const subsection of section.sections) { @@ -82,20 +82,20 @@ async function generateLlmsTxt( } } }; - + const processPage = async (page: HyperbookPage): Promise => { // Skip if hidden or empty if (page.hide || page.isEmpty) { return; } - + if (page.href) { const file = allFiles.find((f) => f.path.href === page.href); if (file) { // Add page name as a header lines.push(`# ${page.name}`); lines.push(""); - + // Get the markdown content without frontmatter const content = file.markdown.content.trim(); if (content) { @@ -105,18 +105,52 @@ async function generateLlmsTxt( } } }; - + // Process root-level pages first for (const page of pagesAndSections.pages) { await processPage(page); } - + // Process sections for (const section of pagesAndSections.sections) { await processSection(section); } - + // Write the llms.txt file + lines.push(` +When you are writing a hyperbook syntax you must use markdown plus the documented elements, also keep track of colons. Colons act like parentheses in programming languages. Every opening colon must have a closing colon. For example, in the syntax :bold text: the first colon opens the bold formatting and the second colon closes it. If there is a missing colon, it can lead to formatting errors or unexpected behavior in the rendered output. Always ensure that colons are properly paired to maintain the intended structure and appearance of your hyperbook content. + +Single colons are inline elements. + +Example: :t[Test] + +Double colons are block elements. + +Example: ::p5{src="sketch.js"} + +Triple colons are special elements that can contain other elements inside them. + +Example: + +:::::alert{info} + +::::tabs + +:::tab{title="JavaScript"} + +Hi + +::: + + +:::: + +::::: + +When you want to nest elements you need to increase the number of colons by one for each level of nesting. The outer level should have the most colons. + +Also you need to use unique ids when the element supports it. +`); const llmsTxtContent = lines.join("\n"); await fs.writeFile(path.join(rootOut, "llms.txt"), llmsTxtContent); } @@ -492,17 +526,17 @@ async function runBuild( if (!faviconExists && hyperbookJson.logo) { console.log(`${chalk.blue(`[${prefix}]`)} Generating favicons from logo.`); - + // Only generate if logo is a local file (not a URL) if (!hyperbookJson.logo.includes("://")) { let logoPath: string | null = null; - + // Resolve logo path by checking multiple locations if (hyperbookJson.logo.startsWith("/")) { // Absolute path starting with / - check book folder, then public folder const bookPath = path.join(root, "book", hyperbookJson.logo); const publicPath = path.join(root, "public", hyperbookJson.logo); - + try { await fs.access(bookPath); logoPath = bookPath; @@ -519,7 +553,7 @@ async function runBuild( const rootPath = path.join(root, hyperbookJson.logo); const bookPath = path.join(root, "book", hyperbookJson.logo); const publicPath = path.join(root, "public", hyperbookJson.logo); - + try { await fs.access(rootPath); logoPath = rootPath; @@ -537,11 +571,18 @@ async function runBuild( } } } - + if (logoPath) { try { - const { generateFavicons } = await import("./helpers/generate-favicons"); - await generateFavicons(logoPath, rootOut, hyperbookJson, ASSETS_FOLDER); + const { generateFavicons } = await import( + "./helpers/generate-favicons" + ); + await generateFavicons( + logoPath, + rootOut, + hyperbookJson, + ASSETS_FOLDER, + ); console.log( `${chalk.green(`[${prefix}]`)} Favicons generated successfully.`, );