Provide entire docs in a single text file for LLMs. (#962)

lucasguellis · web-flow · commit 27eedd34be4b · 2025-01-20T17:05:55.000+01:00
* feat: script for generating llm files

* feat: adding mdx files on generating llm

* feat: adding llms to sidebar + redirects to latest

* fix: format

* fix: changing to RegExp and String Rescript Core types

* feat: adding redirect for `next` version
diff --git a/.gitignore b/.gitignore
@@ -27,3 +27,8 @@ lib/
 
 src/**/*.mjs
 scripts/**/*.mjs
+
+# Generated via generate-llms script
+public/llms/manual/**/llm*.txt
+public/llms/react/**/llm*.txt
+pages/docs/**/**/llms.mdx
diff --git a/data/sidebar_manual_v1100.json b/data/sidebar_manual_v1100.json
@@ -73,6 +73,7 @@
   "Extra": [
     "newcomer-examples",
     "project-structure",
-    "faq"
+    "faq",
+    "llms"
   ]
 }
diff --git a/data/sidebar_manual_v1200.json b/data/sidebar_manual_v1200.json
@@ -72,6 +72,7 @@
   "Extra": [
     "newcomer-examples",
     "project-structure",
-    "faq"
+    "faq",
+    "llms"
   ]
 }
diff --git a/data/sidebar_react_latest.json b/data/sidebar_react_latest.json
@@ -31,5 +31,8 @@
     "beyond-jsx",
     "forwarding-refs",
     "extensions-of-props"
+  ],
+  "Extra": [
+    "llms"
   ]
 }
diff --git a/data/sidebar_react_v0100.json b/data/sidebar_react_v0100.json
@@ -26,5 +26,8 @@
   "Guides": [
     "beyond-jsx",
     "forwarding-refs"
+  ],
+  "Extra": [
+    "llms"
   ]
 }
diff --git a/data/sidebar_react_v0110.json b/data/sidebar_react_v0110.json
@@ -27,5 +27,8 @@
     "beyond-jsx",
     "forwarding-refs",
     "extensions-of-props"
+  ],
+  "Extra": [
+    "llms"
   ]
 }
diff --git a/next.config.mjs b/next.config.mjs
@@ -136,6 +136,16 @@ const config = {
         destination: `/docs/manual/${process.env.VERSION_NEXT}/:slug*`,
         permanent: false,
       },
+      {
+        source: "/llms/manual/latest/:file*",
+        destination: `/llms/manual/${process.env.VERSION_LATEST}/:file*`,
+        permanent: false,
+      },
+      {
+        source: "/llms/manual/next/:file*",
+        destination: `/llms/manual/${process.env.VERSION_NEXT}/:file*`,
+        permanent: false,
+      },
     ];
   },
 };
diff --git a/package.json b/package.json
@@ -61,7 +61,8 @@
     "build": "rescript && npm run update-index && next build",
     "test": "node scripts/test-examples.mjs && node scripts/test-hrefs.mjs",
     "reanalyze": "reanalyze -all-cmt .",
-    "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml"
+    "update-index": "npm run generate-llms && node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml",
+    "generate-llms": "node scripts/generate_llms.mjs"
   },
   "devDependencies": {
     "@mdx-js/react": "^2.3.0",
diff --git a/public/_redirects b/public/_redirects
@@ -11,3 +11,6 @@
 
 /docs/manual/latest/:slug*                            /docs/manual/v11.0.0/:slug*                            307
 /docs/manual/next/:slug*                              /docs/manual/v12.0.0/:slug*                            307
+
+/llms/manual/latest/:file*                            /llms/manual/v11.0.0/:file*                            307
+/llms/manual/next/:file*                              /llms/manual/v12.0.0/:file*                            307
diff --git a/public/llms/manual/template.mdx b/public/llms/manual/template.mdx
@@ -0,0 +1,23 @@
+---
+title: "LLMs"
+description: "Documentation for LLMs"
+canonical: "/docs/manual/<VERSION>/llms"
+---
+
+# Documentation for LLMs
+
+We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications.
+
+Currently, we have the following files...
+
+- [/docs/manual/llms.txt](/llms/manual/<VERSION>/llms.txt) — a list of the available files for ReScript language.
+- [/docs/manual/llm-full.txt](/llms/manual/<VERSION>/llm-full.txt) — complete documentation for ReScript language.
+- [/docs/manual/llm-small.txt](/llms/manual/<VERSION>/llm-small.txt) — compressed version of the former, without examples.
+
+...and package-level documentation:
+
+- [/docs/react/llms](/docs/react/latest/llms) — the LLms documentation for ReScript React.
+
+## Notes
+
+- The content is automatically generated from the same source as the official documentation for the specific version
diff --git a/public/llms/manual/template.txt b/public/llms/manual/template.txt
@@ -0,0 +1,19 @@
+# ReScript Documentation for LLMs
+
+> ReScript is a robustly typed language that compiles to efficient and human-readable JavaScript. It comes with a lightning fast compiler toolchain that scales to any codebase size.
+
+## Documentation Sets
+
+- [Complete documentation](https://rescript-lang.org/llms/manual/<VERSION>/llm-full.txt): The complete ReScript documentation including all examples and additional content
+- [Abridged documentation](https://rescript-lang.org/llms/manual/<VERSION>/llm-small.txt): A minimal version of the ReScript documentation, with the essential content for quick reference
+
+## Individual Package Documentation
+
+- [ReScript React documentation](https://rescript-lang.org/llms/react/latest/llms.txt): This is the developer documentation for ReScript React.
+
+## Notes
+
+- The abridged documentation excludes the detailed examples, and supplementary information
+- The complete documentation includes all content from the official documentation
+- Package-specific documentation files contain only the content relevant to that package
+- The content is automatically generated from the same source as the official documentation for the specific version
diff --git a/public/llms/react/template.mdx b/public/llms/react/template.mdx
@@ -0,0 +1,23 @@
+---
+title: "LLMs"
+description: "Documentation for LLMs"
+canonical: "/docs/react/<VERSION>/llms"
+---
+
+# Documentation for LLMs
+
+We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications.
+
+Currently, we have the following files...
+
+- [/docs/react/llms.txt](/llms/react/<VERSION>/llms.txt) — a list of the available files for ReScript React.
+- [/docs/react/llms-full.txt](/llms/react/<VERSION>/llm-full.txt) — complete documentation for ReScript React.
+- [/docs/react/llms-small.txt](/llms/react/<VERSION>/llm-small.txt) — compressed version of the former, without examples for ReScript React.
+
+...and the language documentation:
+
+- [/docs/manual/llms](/docs/manual/latest/llms) — the LLms documentation for ReScript.
+
+## Notes
+
+- The content is automatically generated from the same source as the official documentation for the specific version
diff --git a/public/llms/react/template.txt b/public/llms/react/template.txt
@@ -0,0 +1,19 @@
+# ReScript React Documentation for LLMs
+
+> ReScript React is a strongly typed interface for React, designed to integrate seamlessly with modern React (>= v18.0) while compiling to idiomatic JavaScript, enabling robust and scalable React applications.
+
+## ReScript React Documentation
+
+- [Complete documentation](https://rescript-lang.org/llms/react/<VERSION>/llm-full.txt): The complete ReScript React documentation including all examples and additional content
+- [Abridged documentation](https://rescript-lang.org/llms/react/<VERSION>/llm-small.txt): A minimal version of the ReScript React documentation, with the essential content for quick reference
+
+## Language Documentation
+
+- [ReScript documentation](https://rescript-lang.org/llms/manual/latest/llms.txt): This is the developer documentation for ReScript.
+
+## Notes
+
+- The abridged documentation excludes the detailed examples, and supplementary information
+- The complete documentation includes all content from the official documentation
+- Package-specific documentation files contain only the content relevant to that package
+- The content is automatically generated from the same source as the official documentation for the specific version
diff --git a/scripts/generate_llms.res b/scripts/generate_llms.res
@@ -0,0 +1,137 @@
+let readMarkdownFile = (filePath: string): string => {
+  let fileContent = Node.Fs.readFileSync2(filePath, "utf8")
+  fileContent
+}
+
+let rec collectFiles = (dirPath: string): array<string> => {
+  let entries = Node.Fs.readdirSync(dirPath)
+  entries->Array.reduce([], (acc, entry) => {
+    let fullPath = Node.Path.join([dirPath, entry])
+    let stats = Node.Fs.statSync(fullPath)
+    switch stats["isDirectory"]() {
+    | true => acc->Array.concat(collectFiles(fullPath))
+    | false => {
+        acc->Array.push(fullPath)
+        acc
+      }
+    }
+  })
+}
+
+let clearFile = (filePath: string): unit => {
+  Node.Fs.writeFileSync(filePath, "")
+}
+
+let createDirectoryIfNotExists = (dirPath: string): unit => {
+  if !Node.Fs.existsSync(dirPath) {
+    Node.Fs.mkdirSync(dirPath)
+  }
+}
+
+let removeCodeTabTags = (content: string): string => {
+  let regex = RegExp.fromStringWithFlags("<CodeTab.*?>[\\s\\S]*?</CodeTab>", ~flags="g")
+  String.replaceRegExp(content, regex, "")
+}
+
+let removeCodeBlocks = (content: string): string => {
+  let regex = RegExp.fromStringWithFlags("```[a-zA-Z]+\\s*[\\s\\S]*?```", ~flags="g")
+  String.replaceRegExp(content, regex, "")
+}
+
+let removeFileTitle = (content: string): string => {
+  let regex = RegExp.fromStringWithFlags("---\ntitle[\\s\\S]*?---", ~flags="g")
+  String.replaceRegExp(content, regex, "")
+}
+
+let removeUnnecessaryBreaks = (content: string): string => {
+  let regex = RegExp.fromStringWithFlags("^\n{2,}", ~flags="g")
+  String.replaceRegExp(content, regex, "")
+}
+
+let removeToDos = (content: string): string => {
+  let regex = RegExp.fromStringWithFlags("<!-- TODO[\\s\\S]*?-->", ~flags="g")
+  String.replaceRegExp(content, regex, "")
+}
+
+let fillContentWithVersion = (content: string, version: string): string => {
+  let regex = RegExp.fromStringWithFlags("<VERSION>", ~flags="g")
+  String.replaceRegExp(content, regex, version)
+}
+
+let createFullFile = (content: string, filePath: string): unit => {
+  Node.Fs.appendFileSync(filePath, content ++ "\n", "utf8")
+}
+
+let createSmallFile = (content: string, filePath: string): unit => {
+  let smallContent =
+    content
+    ->removeCodeTabTags
+    ->removeFileTitle
+    ->removeToDos
+    ->removeCodeBlocks
+    ->removeUnnecessaryBreaks
+  Node.Fs.appendFileSync(filePath, smallContent, "utf8")
+}
+
+let createLlmsFiles = (version: string, docsDirectory: string, llmsDirectory: string): unit => {
+  let mdxFileTemplatePath = llmsDirectory->Node.Path.join2("template.mdx")
+  let mdxFilePath = docsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.mdx")
+  let txtFileTemplatePath = llmsDirectory->Node.Path.join2("template.txt")
+  let txtFilePath = llmsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.txt")
+
+  Node.Fs.writeFileSync(
+    mdxFilePath,
+    readMarkdownFile(mdxFileTemplatePath)->fillContentWithVersion(version),
+  )
+
+  Node.Fs.writeFileSync(
+    txtFilePath,
+    readMarkdownFile(txtFileTemplatePath)->fillContentWithVersion(version),
+  )
+}
+
+let processVersions = (
+  versions: array<string>,
+  docsDirectory: string,
+  llmsDirectory: string,
+): unit => {
+  let fullFileName = "llm-full.txt"
+  let smallFileName = "llm-small.txt"
+
+  versions->Array.forEach(version => {
+    let versionDir = docsDirectory->Node.Path.join2(version)
+    let llmsDir = llmsDirectory->Node.Path.join2(version)
+    let fullFilePath = llmsDir->Node.Path.join2(fullFileName)
+    let smallFilePath = llmsDir->Node.Path.join2(smallFileName)
+
+    createDirectoryIfNotExists(llmsDir)
+    clearFile(fullFilePath)
+    clearFile(smallFilePath)
+
+    createLlmsFiles(version, docsDirectory, llmsDirectory)
+
+    versionDir
+    ->collectFiles
+    ->Array.forEach(filePath => {
+      if String.endsWith(filePath, ".mdx") {
+        let content = readMarkdownFile(filePath)
+
+        content->createFullFile(fullFilePath)
+
+        content->createSmallFile(smallFilePath)
+      }
+    })
+  })
+}
+
+let manualVersions = ["v12.0.0", "v11.0.0"]
+let reactManualVersions = ["latest", "v0.10.0", "v0.11.0"]
+
+let manualDocsDirectory = "pages/docs/manual"
+let reactDocsDirectory = "pages/docs/react"
+
+let manualLlmsDirectory = "public/llms/manual"
+let reactLlmsDirectory = "public/llms/react"
+
+processVersions(manualVersions, manualDocsDirectory, manualLlmsDirectory)
+processVersions(reactManualVersions, reactDocsDirectory, reactLlmsDirectory)
diff --git a/src/bindings/Node.res b/src/bindings/Node.res
@@ -20,10 +20,13 @@ module Process = {
 
 module Fs = {
   @module("fs") external readFileSync: string => string = "readFileSync"
+  @module("fs") external readFileSync2: (string, string) => string = "readFileSync"
   @module("fs") external readdirSync: string => array<string> = "readdirSync"
   @module("fs") external writeFileSync: (string, string) => unit = "writeFileSync"
+  @module("fs") external appendFileSync: (string, string, string) => unit = "appendFileSync"
   @module("fs") external existsSync: string => bool = "existsSync"
   @module("fs") external mkdirSync: string => unit = "mkdirSync"
+  @module("fs") external statSync: string => {.."isDirectory": unit => bool} = "statSync"
 }
 
 module Buffer = {

Original file line number	Diff line number	Diff line change
`@@ -73,6 +73,7 @@`
`73`	`73`	`"Extra": [`
`74`	`74`	`"newcomer-examples",`
`75`	`75`	`"project-structure",`
`76`		`- "faq"`
	`76`	`+ "faq",`
	`77`	`+ "llms"`
`77`	`78`	`]`
`78`	`79`	`}`
Original file line number	Diff line number	Diff line change
`@@ -72,6 +72,7 @@`
`72`	`72`	`"Extra": [`
`73`	`73`	`"newcomer-examples",`
`74`	`74`	`"project-structure",`
`75`		`- "faq"`
	`75`	`+ "faq",`
	`76`	`+ "llms"`
`76`	`77`	`]`
`77`	`78`	`}`
Original file line number	Diff line number	Diff line change
`@@ -31,5 +31,8 @@`
`31`	`31`	`"beyond-jsx",`
`32`	`32`	`"forwarding-refs",`
`33`	`33`	`"extensions-of-props"`
	`34`	`+ ],`
	`35`	`+ "Extra": [`
	`36`	`+ "llms"`
`34`	`37`	`]`
`35`	`38`	`}`
Original file line number	Diff line number	Diff line change
`@@ -26,5 +26,8 @@`
`26`	`26`	`"Guides": [`
`27`	`27`	`"beyond-jsx",`
`28`	`28`	`"forwarding-refs"`
	`29`	`+ ],`
	`30`	`+ "Extra": [`
	`31`	`+ "llms"`
`29`	`32`	`]`
`30`	`33`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,5 +27,8 @@`
`27`	`27`	`"beyond-jsx",`
`28`	`28`	`"forwarding-refs",`
`29`	`29`	`"extensions-of-props"`
	`30`	`+ ],`
	`31`	`+ "Extra": [`
	`32`	`+ "llms"`
`30`	`33`	`]`
`31`	`34`	`}`