wip specify locales and write fallback imgs when localized are absent

Author: hatton

.vscode/settings.json

@@ -4,7 +4,8 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "cSpell.words": [
-    "callouts"
+    "callouts",
+    "unlocalized"
   ],
   "workbench.colorCustomizations": {
     "statusBar.background": "#d649ca",

package.json

@@ -15,7 +15,7 @@
     "notion-download": "node dist/index.js",
     "cmdhelp": "ts-node --compiler-options \"{\\\"module\\\": \\\"commonjs\\\"}\" src/index.ts",
     "// test out with my sample notion db": "",
-    "sample": "cross-var ts-node --compiler-options \"{\\\"module\\\": \\\"commonjs\\\"}\" src/index.ts -n %NOTION_PULL_INTEGRATION_TOKEN% -r %NOTION_PULL_ROOT_PAGE% -m ./sample --log-level verbose",
+    "sample": "cross-var ts-node --compiler-options \"{\\\"module\\\": \\\"commonjs\\\"}\" src/index.ts -n %NOTION_PULL_INTEGRATION_TOKEN% -r %NOTION_PULL_ROOT_PAGE% -m ./sample --locales en,es,fr,de --log-level verbose",
     "sample-with-paths": "cross-var ts-node --compiler-options \"{\\\"module\\\": \\\"commonjs\\\"}\" src/index.ts -n %NOTION_PULL_INTEGRATION_TOKEN% -r %NOTION_PULL_ROOT_PAGE% -m ./sample --img-output-path ./sample_img"
   },
   "repository": {

src/NotionImage.ts

@@ -8,12 +8,19 @@ import { logDebug, verbose, info } from "./log";
 let existingImagesNotSeenYetInPull: string[] = [];
 let imageOutputPath = ""; // default to putting in the same directory as the document referring to it.
 let imagePrefix = ""; // default to "./"
+let locales: string[];
 
-// we parse a notion image and its caption into what we need, which includes any urls to localized versions of the image that may be embedded in the caption
+// we parse a notion image and its caption into what we need, which includes any urls to localized versions
+// of the image that may be embedded in the caption.
 export type ImageSet = {
   // We get these from parseImageBlock():
   primaryUrl: string;
+  // caption may contain a caption and/or URLs to localized versions
   caption?: string;
+  // We use entries in localizedUrls whether or not we have a url, because if we don't have
+  // a localized image, we then need to copy the primary image in, instead, to
+  // get image fallback. In that case, the placeholder at least tells us what languages
+  // are being supported.
   localizedUrls: Array<{ iso632Code: string; url: string }>;
 
   // then we fill this in from processImageBlock():
@@ -32,12 +39,14 @@ export type ImageSet = {
 
 export async function initImageHandling(
   prefix: string,
-  outputPath: string
+  outputPath: string,
+  incomingLocales: string[]
 ): Promise<void> {
   // If they gave us a trailing slash, remove it because we add it back later.
   // Note that it's up to the caller to have a *leading* slash or not.
   imagePrefix = prefix.replace(/\/$/, "");
   imageOutputPath = outputPath;
+  locales = incomingLocales;
 
   // Currently we don't delete the image directory, because if an image
   // changes, it gets a new id. This way can then prevent downloading
@@ -60,42 +69,52 @@ async function saveImage(imageSet: ImageSet): Promise<void> {
 
   let foundLocalizedImage = false;
 
-  // if there are localized images, save them too, using the same
-  // name as the primary but with their language code attached
   for (const localizedImage of imageSet.localizedUrls) {
-    verbose(`Retrieving ${localizedImage.iso632Code} version...`);
-    const response = await fetch(localizedImage.url);
-    const arrayBuffer = await response.arrayBuffer();
-    const buffer = Buffer.from(arrayBuffer);
+    let buffer = imageSet.primaryBuffer!;
+    // if we have a urls for the localized screenshot, download it
+    if (localizedImage?.url.length > 0) {
+      verbose(`Retrieving ${localizedImage.iso632Code} version...`);
+      const response = await fetch(localizedImage.url);
+      const arrayBuffer = await response.arrayBuffer();
+      buffer = Buffer.from(arrayBuffer);
+    } else {
+      verbose(
+        `No localized image specified for ${localizedImage.iso632Code}, will use primary image.`
+      );
+      // otherwise, we're going to fall back to outputting the primary image here
+    }
     const directory = `./i18n/${
       localizedImage.iso632Code
     }/docusaurus-plugin-content-docs/current/${imageSet.relativePathToParentDocument!}`;
     if (!foundLocalizedImage) {
       foundLocalizedImage = true;
-      info(
-        "*** found at least one localized image, so /i18n directory will be created and filled with localized image files."
-      );
     }
     writeImageIfNew(directory + "/" + imageSet.outputFileName!, buffer);
   }
 }
 
 function writeImageIfNew(path: string, buffer: Buffer) {
   imageWasSeen(path);
-  if (!fs.pathExistsSync(path)) {
+
+  // Note: it's tempting to not spend time writing this out if we already have
+  // it from a previous run. But we don't really know it's the same. A) it
+  // could just have the same name, B) it could have been previously
+  // unlocalized and thus filled with a copy of the primary language image
+  // while and now is localized.
+  if (fs.pathExistsSync(path)) {
+    verbose("Replacing image " + path);
+  } else {
     verbose("Adding image " + path);
     fs.mkdirsSync(Path.dirname(path));
-    fs.createWriteStream(path).write(buffer); // async but we're not waiting
-  } else {
-    verbose(`image already filled: ${path}`);
   }
+  fs.createWriteStream(path).write(buffer); // async but we're not waiting
 }
 
 export function parseImageBlock(b: any): ImageSet {
   const imageSet: ImageSet = {
     primaryUrl: "",
     caption: "",
-    localizedUrls: [],
+    localizedUrls: locales.map(l => ({ iso632Code: l, url: "" })),
   };
 
   if ("file" in b.image) {

src/index.ts

@@ -13,18 +13,23 @@ program
   .requiredOption("-n, --notion-token <string>", "notion api token")
   .requiredOption(
     "-r, --root-page <string>",
-    "the 31 character ID of the page which is the root of your notion docs"
+    "The 31 character ID of the page which is the root of your notion docs"
   )
   .requiredOption(
     "-m, --markdown-output-path  <string>",
-    "root of the hierarchy for md files. WARNING: node-pull-mdx will delete files from this directory. Note also that if it finds localized images, it will create an i18n/ directory as a sibling.",
+    "Root of the hierarchy for md files. WARNING: node-pull-mdx will delete files from this directory. Note also that if it finds localized images, it will create an i18n/ directory as a sibling.",
     "./docs"
   )
   .option(
     "-t, --status-tag  <string>",
     "Database pages without a Notion page property 'status' matching this will be ignored. Use '*' to ignore status altogether.",
     "Publish"
   )
+  .option(
+    "--locales  <codes>",
+    "Comma-separated list of iso 639-2 codes, the same list as in docusaurus.config.js, minus the primary (i.e. 'en'). This is needed for image localization.",
+    parseLocales
+  )
   .addOption(
     new Option("-l, --log-level <level>", "Log level").choices([
       "info",
@@ -34,15 +39,15 @@ program
   )
   .option(
     "-i, --img-output-path  <string>",
-    "path to directory where images will be stored. If this is not included, images will be placed in the same directory as the document that uses them, which then allows for localization of screenshots."
+    "Path to directory where images will be stored. If this is not included, images will be placed in the same directory as the document that uses them, which then allows for localization of screenshots."
   )
   // .option(
   //   "-l, --internal-link-prefix <string>",
   //   "when converting a link from one page to another, prefix the with this path instead of the default, which is rooted at the markdown-output-path."
   // )
   .option(
     "-p, --img-prefix-in-markdown <string>",
-    "when referencing an image from markdown, prefix with this path instead of the full img-output-path. Should be used only in conjunction with --img-output-path."
+    "When referencing an image from markdown, prefix with this path instead of the full img-output-path. Should be used only in conjunction with --img-output-path."
   );
 
 program.showHelpAfterError();
@@ -53,3 +58,7 @@ setLogLevel(program.opts().logLevel);
 void notionPull(program.opts() as Options).then(() =>
   console.log("notion-pull-mdx Finished.")
 );
+
+function parseLocales(value: string): string[] {
+  return value.split(",").map(l => l.trim().toLowerCase());
+}

src/makeImagePersistencePlan.spec.ts

@@ -31,3 +31,11 @@ test("primary file with defaults for image output path and prefix", () => {
   );
   expect(imageSet.filePathToUseInMarkdown).toBe("./463556435.png");
 });
+
+// In order to make image fallback work with other languages, we have to have
+// a file for each image, in each Docusaurus language directory. This is true
+// whether we have a localized version of the image or not.
+// The imageSet is initially populated with placeholders for each language.
+// This test ensures that these placeholders are replaced with actual urls
+// when localized versions of the image are listed.
+// TODO write this test

src/pull.ts

@@ -21,6 +21,7 @@ import { error, info, verbose, warning } from "./log";
 export type Options = {
   notionToken: string;
   rootPage: string;
+  locales: string[];
   markdownOutputPath: string;
   imgOutputPath: string;
   imgPrefixInMarkdown: string;
@@ -47,7 +48,8 @@ export async function notionPull(incomingOptions: Options): Promise<void> {
   verbose(JSON.stringify(optionsForLogging, null, 2));
   await initImageHandling(
     options.imgPrefixInMarkdown || options.imgOutputPath || "",
-    options.imgOutputPath || ""
+    options.imgOutputPath || "",
+    options.locales
   );
 
   const notionClient = initNotionClient(options.notionToken);