Add highlightLanguages option

Author: Gerrit0

CHANGELOG.md

@@ -16,6 +16,8 @@
     This means that any projects setting `markedOptions` needs to be updated to use `markdownItOptions`.
     Unlike `marked@4`, `markdown-it` pushes lots of functionality to plugins. To use plugins, a JavaScript config file must be used with the `markdownItLoader` option.
 -   Updated Shiki from 0.14 to 1.x. This should mostly be a transparent update which adds another 23 supported languages and 13 supported themes.
+    As Shiki adds additional languages, the time it takes to load the highlighter increases linearly. To avoid rendering taking longer than necessary,
+    TypeDoc now only loads a few common languages. Additional languages can be loaded by setting the `--highlightLanguages` option.
 -   Renamed `--sitemapBaseUrl` to `--hostedBaseUrl` to reflect that it can be used for more than just the sitemap.
 -   Removed deprecated `navigation.fullTree` option.
 -   (WIP) Removed `--media` option, TypeDoc will now detect image links within your comments and markdown documents and automatically copy them to the site.
@@ -47,6 +49,7 @@
 -   Added three new sort strategies `documents-first`, `documents-last`, and `alphabetical-ignoring-documents` to order markdown documents.
 -   Added new `--alwaysCreateEntryPointModule` option. When set, TypeDoc will always create a `Module` for entry points, even if only one is provided.
     If `--projectDocuments` is used to add documents, this option defaults to `true`, otherwise, defaults to `false`.
+-   Added new `--highlightLanguages` option to control what Shiki language packages are loaded.
 
 ### Bug Fixes
 

src/lib/internationalization/translatable.ts

@@ -113,6 +113,7 @@ export const translatable = {
     // output plugins
     custom_css_file_0_does_not_exist: `Custom CSS file at {0} does not exist.`,
     unsupported_highlight_language_0_not_highlighted_in_comment_for_1: `Unsupported highlight language {0} will not be highlighted in comment for {1}.`,
+    unloaded_language_0_not_highlighted_in_comment_for_1: `Code block with language {0} will not be highlighted in comment for {1} as it was not included in the highlightLanguages option.`,
     yaml_frontmatter_not_an_object: `Expected YAML frontmatter to be an object.`,
 
     // renderer
@@ -218,6 +219,8 @@ export const translatable = {
         "Specify the code highlighting theme in light mode.",
     help_darkHighlightTheme:
         "Specify the code highlighting theme in dark mode.",
+    help_highlightLanguages:
+        "Specify the languages which will be loaded to highlight code when rendering.",
     help_customCss: "Path to a custom CSS file to for the theme to import.",
     help_markdownItOptions:
         "Specify the options passed to markdown-it, the Markdown parser used by TypeDoc.",
@@ -350,6 +353,8 @@ export const translatable = {
     external_symbol_link_mappings_must_be_object:
         "externalSymbolLinkMappings must be a Record<package name, Record<symbol name, link>>",
     highlight_theme_0_must_be_one_of_1: "{0} must be one of the following: {1}",
+    highlightLanguages_contains_invalid_languages_0:
+        "highlightLanguages contains invalid languages: {0}, run typedoc --help for a list of supported languages.",
     hostedBaseUrl_must_start_with_http:
         "hostedBaseUrl must start with http:// or https://",
     option_0_must_be_an_object: "The '{0}' option must be a non-array object.",

src/lib/output/renderer.ts

@@ -20,7 +20,10 @@ import { RendererComponent } from "./components";
 import { Component, ChildableComponent } from "../utils/component";
 import { Option, EventHooks } from "../utils";
 import { loadHighlighter } from "../utils/highlighter";
-import type { BundledTheme as ShikiTheme } from "shiki" with { "resolution-mode": "import" };
+import type {
+    BundledLanguage,
+    BundledTheme as ShikiTheme,
+} from "shiki" with { "resolution-mode": "import" };
 import { Reflection } from "../models";
 import type { JsxElement } from "../utils/jsx.elements";
 import type { DefaultThemeRenderContext } from "./themes/default/DefaultThemeRenderContext";
@@ -187,35 +190,32 @@ export class Renderer extends ChildableComponent<
 
     /** @internal */
     @Option("theme")
-    accessor themeName!: string;
+    private accessor themeName!: string;
 
-    /** @internal */
     @Option("cleanOutputDir")
-    accessor cleanOutputDir!: boolean;
+    private accessor cleanOutputDir!: boolean;
 
-    /** @internal */
     @Option("cname")
-    accessor cname!: string;
+    private accessor cname!: string;
 
-    /** @internal */
     @Option("githubPages")
-    accessor githubPages!: boolean;
+    private accessor githubPages!: boolean;
 
     /** @internal */
     @Option("cacheBust")
     accessor cacheBust!: boolean;
 
-    /** @internal */
     @Option("lightHighlightTheme")
-    accessor lightTheme!: ShikiTheme;
+    private accessor lightTheme!: ShikiTheme;
 
-    /** @internal */
     @Option("darkHighlightTheme")
-    accessor darkTheme!: ShikiTheme;
+    private accessor darkTheme!: ShikiTheme;
+
+    @Option("highlightLanguages")
+    private accessor highlightLanguages!: string[];
 
-    /** @internal */
     @Option("pretty")
-    accessor pretty!: boolean;
+    private accessor pretty!: boolean;
 
     renderStartTime = -1;
 
@@ -299,7 +299,12 @@ export class Renderer extends ChildableComponent<
     }
 
     private async loadHighlighter() {
-        await loadHighlighter(this.lightTheme, this.darkTheme);
+        await loadHighlighter(
+            this.lightTheme,
+            this.darkTheme,
+            // Checked in option validation
+            this.highlightLanguages as BundledLanguage[],
+        );
     }
 
     /**

src/lib/output/themes/MarkedPlugin.tsx

@@ -3,7 +3,7 @@ import markdown from "markdown-it";
 import { Component, ContextAwareRendererComponent } from "../components";
 import { type RendererEvent, MarkdownEvent, type PageEvent } from "../events";
 import { Option, type Logger, renderElement } from "../../utils";
-import { highlight, isSupportedLanguage } from "../../utils/highlighter";
+import { highlight, isLoadedLanguage, isSupportedLanguage } from "../../utils/highlighter";
 import type { BundledTheme } from "shiki" with { "resolution-mode": "import" };
 import { escapeHtml, getTextContent } from "../../utils/html";
 import type { DefaultTheme } from "..";
@@ -72,6 +72,15 @@ export class MarkedPlugin extends ContextAwareRendererComponent {
             );
             return text;
         }
+        if (!isLoadedLanguage(lang)) {
+            this.application.logger.warn(
+                this.application.i18n.unloaded_language_0_not_highlighted_in_comment_for_1(
+                    lang,
+                    this.page?.model.getFriendlyFullName() ?? "(unknown)",
+                ),
+            );
+            return text;
+        }
 
         return highlight(text, lang);
     }

src/lib/utils/highlighter.tsx

@@ -9,6 +9,7 @@ import * as JSX from "./jsx";
 import { unique } from "./array";
 
 const aliases = new Map<string, string>();
+let supportedLanguagesWithoutAliases: string[] = [];
 let supportedLanguages: string[] = [];
 let supportedThemes: string[] = [];
 
@@ -28,6 +29,8 @@ export async function loadShikiMetadata() {
         ...shiki.bundledLanguagesInfo.map((lang) => lang.id),
     ]).sort();
 
+    supportedLanguagesWithoutAliases = unique(["text", ...shiki.bundledLanguagesInfo.map((lang) => lang.id)]);
+
     supportedThemes = Object.keys(shiki.bundledThemes);
 }
 
@@ -40,6 +43,10 @@ class DoubleHighlighter {
         private dark: BundledTheme,
     ) {}
 
+    supports(lang: string) {
+        return this.highlighter.getLoadedLanguages().includes(lang);
+    }
+
     highlight(code: string, lang: string) {
         const tokens = this.highlighter.codeToTokensWithThemes(code, {
             themes: { light: this.light, dark: this.dark },
@@ -122,11 +129,11 @@ class DoubleHighlighter {
 
 let highlighter: DoubleHighlighter | undefined;
 
-export async function loadHighlighter(lightTheme: BundledTheme, darkTheme: BundledTheme) {
+export async function loadHighlighter(lightTheme: BundledTheme, darkTheme: BundledTheme, langs: BundledLanguage[]) {
     if (highlighter) return;
 
     const shiki = await import("shiki");
-    const hl = await shiki.getHighlighter({ themes: [lightTheme, darkTheme], langs: getSupportedLanguages() });
+    const hl = await shiki.getHighlighter({ themes: [lightTheme, darkTheme], langs });
     highlighter = new DoubleHighlighter(hl, lightTheme, darkTheme);
 }
 
@@ -139,16 +146,22 @@ export function getSupportedLanguages(): string[] {
     return supportedLanguages;
 }
 
+export function getSupportedLanguagesWithoutAliases(): string[] {
+    ok(supportedLanguagesWithoutAliases.length > 0, "loadShikiMetadata has not been called");
+    return supportedLanguages;
+}
+
 export function getSupportedThemes(): string[] {
     ok(supportedThemes.length > 0, "loadShikiMetadata has not been called");
     return supportedThemes;
 }
 
+export function isLoadedLanguage(lang: string): boolean {
+    return highlighter?.supports(lang) ?? false;
+}
+
 export function highlight(code: string, lang: string): string {
     assert(highlighter, "Tried to highlight with an uninitialized highlighter");
-    if (!isSupportedLanguage(lang)) {
-        return code;
-    }
 
     if (lang === "text") {
         return JSX.renderElement(<>{code}</>);

src/lib/utils/options/declaration.ts

@@ -135,6 +135,7 @@ export interface TypeDocOptionMap {
     theme: string;
     lightHighlightTheme: ShikiTheme;
     darkHighlightTheme: ShikiTheme;
+    highlightLanguages: string[];
     customCss: string;
     markdownItOptions: ManuallyValidatedOption<Record<string, unknown>>;
     /**

src/lib/utils/options/help.ts

@@ -5,7 +5,10 @@ import {
     ParameterType,
     type DeclarationOption,
 } from "./declaration";
-import { getSupportedLanguages, getSupportedThemes } from "../highlighter";
+import {
+    getSupportedLanguagesWithoutAliases,
+    getSupportedThemes,
+} from "../highlighter";
 import type { TranslationProxy } from "../../internationalization/internationalization";
 
 export interface ParameterHelp {
@@ -102,7 +105,7 @@ export function getOptionsHelp(
     output.push(
         "",
         "Supported highlighting languages:",
-        ...toEvenColumns(getSupportedLanguages(), 80),
+        ...toEvenColumns(getSupportedLanguagesWithoutAliases(), 80),
     );
 
     output.push(

src/lib/utils/options/sources/typedoc.ts

@@ -12,8 +12,15 @@ import { ReflectionKind } from "../../../models/reflections/kind";
 import * as Validation from "../../validation";
 import { blockTags, inlineTags, modifierTags } from "../tsdoc-defaults";
 import { getEnumKeys } from "../../enum";
-import type { BundledTheme } from "shiki" with { "resolution-mode": "import" };
-import { getSupportedThemes } from "../../highlighter";
+import type {
+    BundledLanguage,
+    BundledTheme,
+} from "shiki" with { "resolution-mode": "import" };
+import {
+    getSupportedLanguagesWithoutAliases,
+    getSupportedThemes,
+} from "../../highlighter";
+import { setDifference } from "../../set";
 
 // For convenience, added in the same order as they are documented on the website.
 export function addTypeDocOptions(options: Pick<Options, "addDeclaration">) {
@@ -319,6 +326,35 @@ export function addTypeDocOptions(options: Pick<Options, "addDeclaration">) {
             }
         },
     });
+    options.addDeclaration({
+        name: "highlightLanguages",
+        help: (i18n) => i18n.help_highlightLanguages(),
+        type: ParameterType.Array,
+        defaultValue: [
+            "bash",
+            "console",
+            "css",
+            "html",
+            "javascript",
+            "json",
+            "jsonc",
+            "tsx",
+            "typescript",
+        ] satisfies BundledLanguage[],
+        validate(value, i18n) {
+            const invalid = setDifference(
+                value,
+                getSupportedLanguagesWithoutAliases(),
+            );
+            if (invalid.size) {
+                throw new Error(
+                    i18n.highlightLanguages_contains_invalid_languages_0(
+                        Array.from(invalid).join(", "),
+                    ),
+                );
+            }
+        },
+    });
 
     options.addDeclaration({
         name: "customCss",

src/lib/utils/set.ts

@@ -1,4 +1,4 @@
-export function setIntersection<T>(a: Set<T>, b: Set<T>): Set<T> {
+export function setIntersection<T>(a: Iterable<T>, b: Set<T>): Set<T> {
     const result = new Set<T>();
     for (const elem of a) {
         if (b.has(elem)) {
@@ -7,3 +7,11 @@ export function setIntersection<T>(a: Set<T>, b: Set<T>): Set<T> {
     }
     return result;
 }
+
+export function setDifference<T>(a: Iterable<T>, b: Iterable<T>): Set<T> {
+    const result = new Set(a);
+    for (const elem of b) {
+        result.delete(elem);
+    }
+    return result;
+}