diff --git a/.gitignore b/.gitignore index 4b6ddf80c..d364e12a7 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,8 @@ lib/ src/**/*.mjs scripts/**/*.mjs + +# Generated via generate-llms script +public/llms/manual/**/llm*.txt +public/llms/react/**/llm*.txt +pages/docs/**/**/llms.mdx diff --git a/data/sidebar_manual_v1100.json b/data/sidebar_manual_v1100.json index 6e6c86255..a7672bb98 100644 --- a/data/sidebar_manual_v1100.json +++ b/data/sidebar_manual_v1100.json @@ -73,6 +73,7 @@ "Extra": [ "newcomer-examples", "project-structure", - "faq" + "faq", + "llms" ] } \ No newline at end of file diff --git a/data/sidebar_manual_v1200.json b/data/sidebar_manual_v1200.json index d783a4eb3..a68a5336c 100644 --- a/data/sidebar_manual_v1200.json +++ b/data/sidebar_manual_v1200.json @@ -72,6 +72,7 @@ "Extra": [ "newcomer-examples", "project-structure", - "faq" + "faq", + "llms" ] } \ No newline at end of file diff --git a/data/sidebar_react_latest.json b/data/sidebar_react_latest.json index 758517284..a2a47d615 100644 --- a/data/sidebar_react_latest.json +++ b/data/sidebar_react_latest.json @@ -31,5 +31,8 @@ "beyond-jsx", "forwarding-refs", "extensions-of-props" + ], + "Extra": [ + "llms" ] } \ No newline at end of file diff --git a/data/sidebar_react_v0100.json b/data/sidebar_react_v0100.json index b6bd3b260..43b20d93e 100644 --- a/data/sidebar_react_v0100.json +++ b/data/sidebar_react_v0100.json @@ -26,5 +26,8 @@ "Guides": [ "beyond-jsx", "forwarding-refs" + ], + "Extra": [ + "llms" ] } \ No newline at end of file diff --git a/data/sidebar_react_v0110.json b/data/sidebar_react_v0110.json index f7e96a22c..0876b0888 100644 --- a/data/sidebar_react_v0110.json +++ b/data/sidebar_react_v0110.json @@ -27,5 +27,8 @@ "beyond-jsx", "forwarding-refs", "extensions-of-props" + ], + "Extra": [ + "llms" ] } \ No newline at end of file diff --git a/next.config.mjs b/next.config.mjs index d8b12ed46..eb31e01f3 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -136,6 +136,16 @@ const config = { destination: `/docs/manual/${process.env.VERSION_NEXT}/:slug*`, permanent: false, }, + { + source: "/llms/manual/latest/:file*", + destination: `/llms/manual/${process.env.VERSION_LATEST}/:file*`, + permanent: false, + }, + { + source: "/llms/manual/next/:file*", + destination: `/llms/manual/${process.env.VERSION_NEXT}/:file*`, + permanent: false, + }, ]; }, }; diff --git a/package.json b/package.json index 2d4984b74..0befba73d 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,8 @@ "build": "rescript && npm run update-index && next build", "test": "node scripts/test-examples.mjs && node scripts/test-hrefs.mjs", "reanalyze": "reanalyze -all-cmt .", - "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml" + "update-index": "npm run generate-llms && node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml", + "generate-llms": "node scripts/generate_llms.mjs" }, "devDependencies": { "@mdx-js/react": "^2.3.0", diff --git a/public/_redirects b/public/_redirects index e0e869325..9c18f7af1 100644 --- a/public/_redirects +++ b/public/_redirects @@ -11,3 +11,6 @@ /docs/manual/latest/:slug* /docs/manual/v11.0.0/:slug* 307 /docs/manual/next/:slug* /docs/manual/v12.0.0/:slug* 307 + +/llms/manual/latest/:file* /llms/manual/v11.0.0/:file* 307 +/llms/manual/next/:file* /llms/manual/v12.0.0/:file* 307 diff --git a/public/llms/manual/template.mdx b/public/llms/manual/template.mdx new file mode 100644 index 000000000..975c1a8c3 --- /dev/null +++ b/public/llms/manual/template.mdx @@ -0,0 +1,23 @@ +--- +title: "LLMs" +description: "Documentation for LLMs" +canonical: "/docs/manual//llms" +--- + +# Documentation for LLMs + +We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications. + +Currently, we have the following files... + +- [/docs/manual/llms.txt](/llms/manual//llms.txt) — a list of the available files for ReScript language. +- [/docs/manual/llm-full.txt](/llms/manual//llm-full.txt) — complete documentation for ReScript language. +- [/docs/manual/llm-small.txt](/llms/manual//llm-small.txt) — compressed version of the former, without examples. + +...and package-level documentation: + +- [/docs/react/llms](/docs/react/latest/llms) — the LLms documentation for ReScript React. + +## Notes + +- The content is automatically generated from the same source as the official documentation for the specific version diff --git a/public/llms/manual/template.txt b/public/llms/manual/template.txt new file mode 100644 index 000000000..d4474f163 --- /dev/null +++ b/public/llms/manual/template.txt @@ -0,0 +1,19 @@ +# ReScript Documentation for LLMs + +> ReScript is a robustly typed language that compiles to efficient and human-readable JavaScript. It comes with a lightning fast compiler toolchain that scales to any codebase size. + +## Documentation Sets + +- [Complete documentation](https://rescript-lang.org/llms/manual//llm-full.txt): The complete ReScript documentation including all examples and additional content +- [Abridged documentation](https://rescript-lang.org/llms/manual//llm-small.txt): A minimal version of the ReScript documentation, with the essential content for quick reference + +## Individual Package Documentation + +- [ReScript React documentation](https://rescript-lang.org/llms/react/latest/llms.txt): This is the developer documentation for ReScript React. + +## Notes + +- The abridged documentation excludes the detailed examples, and supplementary information +- The complete documentation includes all content from the official documentation +- Package-specific documentation files contain only the content relevant to that package +- The content is automatically generated from the same source as the official documentation for the specific version \ No newline at end of file diff --git a/public/llms/react/template.mdx b/public/llms/react/template.mdx new file mode 100644 index 000000000..580a23302 --- /dev/null +++ b/public/llms/react/template.mdx @@ -0,0 +1,23 @@ +--- +title: "LLMs" +description: "Documentation for LLMs" +canonical: "/docs/react//llms" +--- + +# Documentation for LLMs + +We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications. + +Currently, we have the following files... + +- [/docs/react/llms.txt](/llms/react//llms.txt) — a list of the available files for ReScript React. +- [/docs/react/llms-full.txt](/llms/react//llm-full.txt) — complete documentation for ReScript React. +- [/docs/react/llms-small.txt](/llms/react//llm-small.txt) — compressed version of the former, without examples for ReScript React. + +...and the language documentation: + +- [/docs/manual/llms](/docs/manual/latest/llms) — the LLms documentation for ReScript. + +## Notes + +- The content is automatically generated from the same source as the official documentation for the specific version diff --git a/public/llms/react/template.txt b/public/llms/react/template.txt new file mode 100644 index 000000000..0023e410a --- /dev/null +++ b/public/llms/react/template.txt @@ -0,0 +1,19 @@ +# ReScript React Documentation for LLMs + +> ReScript React is a strongly typed interface for React, designed to integrate seamlessly with modern React (>= v18.0) while compiling to idiomatic JavaScript, enabling robust and scalable React applications. + +## ReScript React Documentation + +- [Complete documentation](https://rescript-lang.org/llms/react//llm-full.txt): The complete ReScript React documentation including all examples and additional content +- [Abridged documentation](https://rescript-lang.org/llms/react//llm-small.txt): A minimal version of the ReScript React documentation, with the essential content for quick reference + +## Language Documentation + +- [ReScript documentation](https://rescript-lang.org/llms/manual/latest/llms.txt): This is the developer documentation for ReScript. + +## Notes + +- The abridged documentation excludes the detailed examples, and supplementary information +- The complete documentation includes all content from the official documentation +- Package-specific documentation files contain only the content relevant to that package +- The content is automatically generated from the same source as the official documentation for the specific version \ No newline at end of file diff --git a/scripts/generate_llms.res b/scripts/generate_llms.res new file mode 100644 index 000000000..d86c5d9c0 --- /dev/null +++ b/scripts/generate_llms.res @@ -0,0 +1,137 @@ +let readMarkdownFile = (filePath: string): string => { + let fileContent = Node.Fs.readFileSync2(filePath, "utf8") + fileContent +} + +let rec collectFiles = (dirPath: string): array => { + let entries = Node.Fs.readdirSync(dirPath) + entries->Array.reduce([], (acc, entry) => { + let fullPath = Node.Path.join([dirPath, entry]) + let stats = Node.Fs.statSync(fullPath) + switch stats["isDirectory"]() { + | true => acc->Array.concat(collectFiles(fullPath)) + | false => { + acc->Array.push(fullPath) + acc + } + } + }) +} + +let clearFile = (filePath: string): unit => { + Node.Fs.writeFileSync(filePath, "") +} + +let createDirectoryIfNotExists = (dirPath: string): unit => { + if !Node.Fs.existsSync(dirPath) { + Node.Fs.mkdirSync(dirPath) + } +} + +let removeCodeTabTags = (content: string): string => { + let regex = RegExp.fromStringWithFlags("[\\s\\S]*?", ~flags="g") + String.replaceRegExp(content, regex, "") +} + +let removeCodeBlocks = (content: string): string => { + let regex = RegExp.fromStringWithFlags("```[a-zA-Z]+\\s*[\\s\\S]*?```", ~flags="g") + String.replaceRegExp(content, regex, "") +} + +let removeFileTitle = (content: string): string => { + let regex = RegExp.fromStringWithFlags("---\ntitle[\\s\\S]*?---", ~flags="g") + String.replaceRegExp(content, regex, "") +} + +let removeUnnecessaryBreaks = (content: string): string => { + let regex = RegExp.fromStringWithFlags("^\n{2,}", ~flags="g") + String.replaceRegExp(content, regex, "") +} + +let removeToDos = (content: string): string => { + let regex = RegExp.fromStringWithFlags("", ~flags="g") + String.replaceRegExp(content, regex, "") +} + +let fillContentWithVersion = (content: string, version: string): string => { + let regex = RegExp.fromStringWithFlags("", ~flags="g") + String.replaceRegExp(content, regex, version) +} + +let createFullFile = (content: string, filePath: string): unit => { + Node.Fs.appendFileSync(filePath, content ++ "\n", "utf8") +} + +let createSmallFile = (content: string, filePath: string): unit => { + let smallContent = + content + ->removeCodeTabTags + ->removeFileTitle + ->removeToDos + ->removeCodeBlocks + ->removeUnnecessaryBreaks + Node.Fs.appendFileSync(filePath, smallContent, "utf8") +} + +let createLlmsFiles = (version: string, docsDirectory: string, llmsDirectory: string): unit => { + let mdxFileTemplatePath = llmsDirectory->Node.Path.join2("template.mdx") + let mdxFilePath = docsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.mdx") + let txtFileTemplatePath = llmsDirectory->Node.Path.join2("template.txt") + let txtFilePath = llmsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.txt") + + Node.Fs.writeFileSync( + mdxFilePath, + readMarkdownFile(mdxFileTemplatePath)->fillContentWithVersion(version), + ) + + Node.Fs.writeFileSync( + txtFilePath, + readMarkdownFile(txtFileTemplatePath)->fillContentWithVersion(version), + ) +} + +let processVersions = ( + versions: array, + docsDirectory: string, + llmsDirectory: string, +): unit => { + let fullFileName = "llm-full.txt" + let smallFileName = "llm-small.txt" + + versions->Array.forEach(version => { + let versionDir = docsDirectory->Node.Path.join2(version) + let llmsDir = llmsDirectory->Node.Path.join2(version) + let fullFilePath = llmsDir->Node.Path.join2(fullFileName) + let smallFilePath = llmsDir->Node.Path.join2(smallFileName) + + createDirectoryIfNotExists(llmsDir) + clearFile(fullFilePath) + clearFile(smallFilePath) + + createLlmsFiles(version, docsDirectory, llmsDirectory) + + versionDir + ->collectFiles + ->Array.forEach(filePath => { + if String.endsWith(filePath, ".mdx") { + let content = readMarkdownFile(filePath) + + content->createFullFile(fullFilePath) + + content->createSmallFile(smallFilePath) + } + }) + }) +} + +let manualVersions = ["v12.0.0", "v11.0.0"] +let reactManualVersions = ["latest", "v0.10.0", "v0.11.0"] + +let manualDocsDirectory = "pages/docs/manual" +let reactDocsDirectory = "pages/docs/react" + +let manualLlmsDirectory = "public/llms/manual" +let reactLlmsDirectory = "public/llms/react" + +processVersions(manualVersions, manualDocsDirectory, manualLlmsDirectory) +processVersions(reactManualVersions, reactDocsDirectory, reactLlmsDirectory) diff --git a/src/bindings/Node.res b/src/bindings/Node.res index d688f7baa..44bcd522b 100644 --- a/src/bindings/Node.res +++ b/src/bindings/Node.res @@ -20,10 +20,13 @@ module Process = { module Fs = { @module("fs") external readFileSync: string => string = "readFileSync" + @module("fs") external readFileSync2: (string, string) => string = "readFileSync" @module("fs") external readdirSync: string => array = "readdirSync" @module("fs") external writeFileSync: (string, string) => unit = "writeFileSync" + @module("fs") external appendFileSync: (string, string, string) => unit = "appendFileSync" @module("fs") external existsSync: string => bool = "existsSync" @module("fs") external mkdirSync: string => unit = "mkdirSync" + @module("fs") external statSync: string => {.."isDirectory": unit => bool} = "statSync" } module Buffer = {