diff --git a/.changeset/llms-txt-generation.md b/.changeset/llms-txt-generation.md new file mode 100644 index 000000000..fbe4eeea2 --- /dev/null +++ b/.changeset/llms-txt-generation.md @@ -0,0 +1,6 @@ +--- +"hyperbook": minor +"@hyperbook/types": minor +--- + +Add llms.txt file generation feature. When the `llms` property is set to `true` in hyperbook.json, a `llms.txt` file will be generated during build that combines all markdown files in order. The file includes the book name and version in the header. Pages and sections with `hide: true` are automatically excluded from the generated file. diff --git a/packages/hyperbook/build.ts b/packages/hyperbook/build.ts index bfd24b991..6fd3315c4 100644 --- a/packages/hyperbook/build.ts +++ b/packages/hyperbook/build.ts @@ -11,6 +11,9 @@ import { Link, Hyperproject, HyperbookContext, + HyperbookJson, + HyperbookPage, + HyperbookSection, Navigation, } from "@hyperbook/types"; import lunr from "lunr"; @@ -19,6 +22,139 @@ import packageJson from "./package.json"; export const ASSETS_FOLDER = "__hyperbook_assets"; +/** + * Generates an llms.txt file by combining all markdown files in order + */ +async function generateLlmsTxt( + root: string, + rootOut: string, + hyperbookJson: HyperbookJson, + pagesAndSections: Pick, + version: string, +): Promise { + const lines: string[] = []; + + // Add header with book name and version + lines.push(`${hyperbookJson.name} - Version ${version}`); + lines.push(""); // Empty line after header + + // Get all book files once to avoid repeated file system operations + const allFiles = await vfile.listForFolder(root, "book"); + + // Helper function to recursively process sections and pages + const processSection = async ( + section: HyperbookSection, + level: number = 0, + ): Promise => { + // Skip if hidden + if (section.hide) { + return; + } + + // Add section header if it has content + if (section.href && !section.isEmpty) { + const file = allFiles.find((f) => f.path.href === section.href); + if (file) { + // Add section name as a header + lines.push(`# ${section.name}`); + lines.push(""); + + // Get the markdown content without frontmatter + const content = file.markdown.content.trim(); + if (content) { + lines.push(content); + lines.push(""); // Empty line after content + } + } + } + + // Process nested pages + if (section.pages) { + for (const page of section.pages) { + await processPage(page); + } + } + + // Process nested sections + if (section.sections) { + for (const subsection of section.sections) { + await processSection(subsection, level + 1); + } + } + }; + + const processPage = async (page: HyperbookPage): Promise => { + // Skip if hidden or empty + if (page.hide || page.isEmpty) { + return; + } + + if (page.href) { + const file = allFiles.find((f) => f.path.href === page.href); + if (file) { + // Add page name as a header + lines.push(`# ${page.name}`); + lines.push(""); + + // Get the markdown content without frontmatter + const content = file.markdown.content.trim(); + if (content) { + lines.push(content); + lines.push(""); // Empty line after content + } + } + } + }; + + // Process root-level pages first + for (const page of pagesAndSections.pages) { + await processPage(page); + } + + // Process sections + for (const section of pagesAndSections.sections) { + await processSection(section); + } + + // Write the llms.txt file + lines.push(` +When you are writing a hyperbook syntax you must use markdown plus the documented elements, also keep track of colons. Colons act like parentheses in programming languages. Every opening colon must have a closing colon. For example, in the syntax :bold text: the first colon opens the bold formatting and the second colon closes it. If there is a missing colon, it can lead to formatting errors or unexpected behavior in the rendered output. Always ensure that colons are properly paired to maintain the intended structure and appearance of your hyperbook content. + +Single colons are inline elements. + +Example: :t[Test] + +Double colons are block elements. + +Example: ::p5{src="sketch.js"} + +Triple colons are special elements that can contain other elements inside them. + +Example: + +:::::alert{info} + +::::tabs + +:::tab{title="JavaScript"} + +Hi + +::: + + +:::: + +::::: + +When you want to nest elements you need to increase the number of colons by one for each level of nesting. The outer level should have the most colons. + +Also you need to use unique ids when the element supports it. +`); + const llmsTxtContent = lines.join("\n"); + await fs.writeFile(path.join(rootOut, "llms.txt"), llmsTxtContent); +} + export async function runBuildProject( project: Hyperproject, rootProject: Hyperproject, @@ -390,17 +526,17 @@ async function runBuild( if (!faviconExists && hyperbookJson.logo) { console.log(`${chalk.blue(`[${prefix}]`)} Generating favicons from logo.`); - + // Only generate if logo is a local file (not a URL) if (!hyperbookJson.logo.includes("://")) { let logoPath: string | null = null; - + // Resolve logo path by checking multiple locations if (hyperbookJson.logo.startsWith("/")) { // Absolute path starting with / - check book folder, then public folder const bookPath = path.join(root, "book", hyperbookJson.logo); const publicPath = path.join(root, "public", hyperbookJson.logo); - + try { await fs.access(bookPath); logoPath = bookPath; @@ -417,7 +553,7 @@ async function runBuild( const rootPath = path.join(root, hyperbookJson.logo); const bookPath = path.join(root, "book", hyperbookJson.logo); const publicPath = path.join(root, "public", hyperbookJson.logo); - + try { await fs.access(rootPath); logoPath = rootPath; @@ -435,11 +571,18 @@ async function runBuild( } } } - + if (logoPath) { try { - const { generateFavicons } = await import("./helpers/generate-favicons"); - await generateFavicons(logoPath, rootOut, hyperbookJson, ASSETS_FOLDER); + const { generateFavicons } = await import( + "./helpers/generate-favicons" + ); + await generateFavicons( + logoPath, + rootOut, + hyperbookJson, + ASSETS_FOLDER, + ); console.log( `${chalk.green(`[${prefix}]`)} Favicons generated successfully.`, ); @@ -598,5 +741,17 @@ const SEARCH_DOCUMENTS = ${JSON.stringify(documents)}; ), ); + // Generate llms.txt if enabled + if (hyperbookJson.llms) { + console.log(`${chalk.blue(`[${prefix}]`)} Generating llms.txt`); + await generateLlmsTxt( + root, + rootOut, + hyperbookJson, + pagesAndSections, + packageJson.version, + ); + } + console.log(`${chalk.green(`[${prefix}]`)} Build success: ${rootOut}`); } diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 93395f646..172d05886 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -93,6 +93,7 @@ export type HyperbookJson = { search?: boolean; qrcode?: boolean; toc?: boolean; + llms?: boolean; author?: { name?: string; url?: string; diff --git a/platforms/vscode/schemas/hyperbook.schema.json b/platforms/vscode/schemas/hyperbook.schema.json index f9384d637..d397c7a69 100644 --- a/platforms/vscode/schemas/hyperbook.schema.json +++ b/platforms/vscode/schemas/hyperbook.schema.json @@ -280,6 +280,9 @@ }, "type": "array" }, + "llms": { + "type": "boolean" + }, "logo": { "type": "string" }, diff --git a/website/de/book/configuration/book.md b/website/de/book/configuration/book.md index d59cc6992..891d78676 100644 --- a/website/de/book/configuration/book.md +++ b/website/de/book/configuration/book.md @@ -38,6 +38,7 @@ von Optionen, die du definieren kannst. Optionen mit einem "\*" müssen gesetzt | allowDangerousHtml | Erlaube HTML im Hyperbook. Dies kann zu Inkompatibilität in zukünftigen Versionen führen. | | qrcode | Zeigt ein Icon, um einen QR-Code zur aktuellen Seite anzuzeigen. | | toc | Zeige ein Inhaltsverzeichnis. Diese ist standardmäßig aktiviert für Seiten und deaktiviert für Begriffe im Glossar. | +| llms | Wenn auf true gesetzt, wird eine llms.txt-Datei generiert, die alle Markdown-Dateien in Reihenfolge kombiniert. Die Datei enthält den Buchnamen und die Version im Header-Format. | | trailingSlash | Exportiert alle Datei in eigene Verzeichnisse und erzeugt nur index.html Dateien. | | importExport | Ermöglicht das Importieren und Exportieren des Zustands des Hyperbooks als Datei. Schaltflächen zum Importieren und Exportieren befinden sich am unteren Rand der Seite. | diff --git a/website/de/hyperbook.json b/website/de/hyperbook.json index eae6d5a51..0da130f09 100644 --- a/website/de/hyperbook.json +++ b/website/de/hyperbook.json @@ -2,6 +2,7 @@ "name": "Hyperbook Dokumenation", "qrcode": true, "search": true, + "llms": true, "importExport": true, "description": "Dokumentation für Hyperbook erstellt mit Hyperbook", "author": { diff --git a/website/en/book/configuration/book.md b/website/en/book/configuration/book.md index b72a9ba83..14a4e4699 100644 --- a/website/en/book/configuration/book.md +++ b/website/en/book/configuration/book.md @@ -37,6 +37,7 @@ can and part wise must set (indicated by a \*). | allowDangerousHtml | Allow HTML. This can lead to incompatibilities in future versions. | | qrcode | Shows an icon, which opens a qr code to the current page. | | toc | Show or hide a table of content for the page. This is on for pages and off for glossary entries by default | +| llms | When set to true, generates an llms.txt file that combines all markdown files in order. The file includes the book name and version in a header format. | | trailingSlash | Outputs all files into ther own folders and produces only index.html files. | | importExport | Allows to import and export the state of the Hyperbook as a file. Buttons for importing and exporting will be at the bottom of the page. | diff --git a/website/en/hyperbook.json b/website/en/hyperbook.json index 0e675e2d0..14b585c91 100644 --- a/website/en/hyperbook.json +++ b/website/en/hyperbook.json @@ -1,6 +1,7 @@ { "name": "Hyperbook Documentation", "qrcode": true, + "llms": true, "importExport": true, "search": true, "description": "Documentation for Hyperbook created with Hyperbook",