[#46] Extract documents within adventures into separate files (#61)

Co-authored-by: fyorl <kim.mantas@gmail.com>

Author: arbron

CHANGELOG.md

@@ -4,6 +4,7 @@
  - Attempting to unpack a directory that is not a valid LevelDB database now throws an error.
  - (BoltsJ) Added the `--config` command-line flag. Configuration options read from this file will be merged with the global `.fvttrc.yml` configuration options.
  - (Jeff Hitchcock) Added the `--folders` command-line flag, and corresponding `folders` parameter to `extractPack`. When used, this option writes the pack's entries to a directory structure matching the pack's internal Folder document structure.
+ - (Jeff Hitchcock) Added the `--expandAdventures` command-line flags, and corresponding `expandAdventures` parameter to `extractPack`. When used, this option writes each Adventure document's embedded documents to their own files.
 
 ### Fixes
  - (Jakob Törmä) Fixed launch command assuming electron directory structure.

README.md

@@ -213,6 +213,7 @@ Extract the contents of a compendium pack into individual source files for each
     * **transformEntry:** *(entry: object): Promise<false|void>* A function that is called on every entry. Returning *false* indicates that the entry should be discarded.
     * **transformName:** *(entry: object): Promise<string|void>* A function that is called on every entry. The value returned from this will be used as the entry's filename and must include the appropriate file extension. If nothing is returned, an auto-generated name will be used instead.
     * **transformFolderName:** *(entry: object): Promise<string|void>* A function used to generate a directory name for an extracted Folder document when the `folders` option is used.
+    * **expandAdventures:** *boolean* Write documents emebdded in Adventures to their own files. If the `folders` option is also supplied, the Adventure is treated like a folder, and written to `_Adventure.{yml|json}` instead of `_Folder.{yml|json}`.
     * **jsonOptions:** *object*
         * **replacer:** *(key: string, value: any): any|Array<string|number>* A replacer function or an array of property names in the object to include in the resulting string.
         * **space:** *string|number* A number of spaces or a string to use as indentation.

commands/package.mjs

@@ -28,6 +28,8 @@ import { compilePack, extractPack, TYPE_COLLECTION_MAP } from "../lib/package.mj
  * @property {boolean} [clean]                          When unpacking, delete the destination directory first.
  * @property {boolean} [folders]                        When unpacking, create a directory structure that matches the
  *                                                      compendium folders.
+ * @property {boolean} [expandAdventures]               When unpacking, extract adventure documents into a folder with
+ *                                                      each contained document as its own entry in a folder.
  */
 
 /**
@@ -132,6 +134,11 @@ export function getCommand() {
         type: "boolean"
       });
 
+      yargs.option("expandAdventures", {
+        describe: "When unpacking, extract documents embedded inside Adventures to their own files. If supplied alongside the --folders option, the Adventure is treated like a folder.",
+        type: "boolean"
+      });
+
       return yargs;
     },
     handler: async argv => {
@@ -381,7 +388,7 @@ async function handleUnpack(argv) {
   }
 
   let documentType;
-  const { nedb, yaml, clean, folders } = argv;
+  const { nedb, yaml, clean, folders, expandAdventures } = argv;
   if ( nedb ) {
     documentType = determineDocumentType(pack, argv);
     if ( !documentType ) {
@@ -401,7 +408,7 @@ async function handleUnpack(argv) {
   console.log(`[${dbMode}] Unpacking "${chalk.blue(pack)}" to "${chalk.blue(source)}"`);
 
   try {
-    await extractPack(pack, source, { nedb, yaml, documentType, clean, folders, log: true });
+    await extractPack(pack, source, { nedb, yaml, documentType, clean, folders, expandAdventures, log: true });
   } catch ( err ) {
     console.error(err);
     process.exitCode = 1;

lib/package.mjs

@@ -46,6 +46,9 @@ import { ClassicLevel } from "classic-level";
  * @property {DocumentType} [documentType]            Required only for NeDB packs in order to generate a correct key.
  * @property {boolean} [clean]                        Delete the destination directory before unpacking.
  * @property {boolean} [folders]                      Create a directory structure that matches the compendium folders.
+ * @property {boolean} [expandAdventures]             Write documents embedded in Adventures to their own files. If the
+ *                                                    folders option is also supplied, the Adventure is treated like a
+ *                                                    folder.
  * @property {DocumentCollection} [collection]        Required only for NeDB packs in order to generate a correct key.
  *                                                    Can be used instead of documentType if known.
  * @property {NameTransformer} [transformName]        A function that is used to generate a filename for the extracted
@@ -64,6 +67,12 @@ import { ClassicLevel } from "classic-level";
  * @property {string|number} [space]                         A number of spaces or a string to use as indentation.
  */
 
+/**
+ * @typedef FolderDescriptor
+ * @property {string} name      The folder's filename.
+ * @property {string} [folder]  A parent folder ID.
+ */
+
 /**
  * @callback JSONReplacer
  * @param {string} key  The key being stringified.
@@ -73,17 +82,25 @@ import { ClassicLevel } from "classic-level";
 
 /**
  * @callback EntryTransformer
- * @param {object} entry           The entry data.
- * @returns {Promise<false|void>}  Return boolean false to indicate that this entry should be discarded.
+ * @param {object} entry                      The entry data.
+ * @param {TransformerContext} [context]      Optional context information for the document being transformed.
+ * @returns {Promise<false|void>}             Return boolean false to indicate that this entry should be discarded.
  */
 
 /**
  * @callback NameTransformer
- * @param {object} entry             The entry data.
- * @param {object} [context]
- * @param {string} [context.folder]  Folder path if this entry is in a folder and the folders option is enabled.
- * @returns {Promise<string|void>}   If a string is returned, it is used as the filename that the entry will be written
- *                                   to.
+ * @param {object} entry                      The entry data.
+ * @param {TransformerContext} [context]      Optional context information for the document being transformed.
+ * @returns {Promise<string|void>}            If a string is returned, it is used as the filename that the entry will
+ *                                            be written to.
+ */
+
+/**
+ * @typedef TransformerContext
+ * @property {object} [adventure]       Data on an adventure if document is stored within an adventure.
+ * @property {object} [adventure.doc]   The entire adventure document.
+ * @property {string} [adventure.path]  The path where the adventure will be extracted.
+ * @property {string} [folder]          Folder path if this entry is in a folder and the folders option is enabled.
  */
 
 /**
@@ -154,6 +171,14 @@ const HIERARCHY = {
   }
 };
 
+/**
+ * Document collections stored with adventure documents.
+ * @type {string[]}
+ */
+const ADVENTURE_DOCS = [
+  "actors", "cards", "combats", "folders", "items", "journal", "playlists", "scenes", "tables", "macros"
+];
+
 /**
  * A mapping of primary document types to collection names.
  * @type {Record<DocumentType, DocumentCollection>}
@@ -221,7 +246,7 @@ async function compileNedb(pack, files, { log, transformEntry }={}) {
   const seenKeys = new Set();
   const packDoc = applyHierarchy(doc => {
     if ( seenKeys.has(doc._key) ) {
-      throw new Error(`An entry with key '${key}' was already packed and would be overwritten by this entry.`);
+      throw new Error(`An entry with key '${doc._key}' was already packed and would be overwritten by this entry.`);
     }
     seenKeys.add(doc._key);
     delete doc._key;
@@ -234,6 +259,10 @@ async function compileNedb(pack, files, { log, transformEntry }={}) {
       const ext = path.extname(file);
       const isYaml = ext === ".yml" || ext === ".yaml";
       const doc = isYaml ? YAML.load(contents) : JSON.parse(contents);
+      if ( !doc._key ) continue;
+      if ( doc._key.startsWith("!adventures") ) await reconstructAdventure(path.dirname(file), doc, {
+        transformEntry, log
+      });
       const key = doc._key;
       const [, collection] = key.split("!");
       // If the key starts with !folders, we should skip packing it as NeDB doesn't support folders.
@@ -290,6 +319,10 @@ async function compileClassicLevel(pack, files, { log, transformEntry }={}) {
       const ext = path.extname(file);
       const isYaml = ext === ".yml" || ext === ".yaml";
       const doc = isYaml ? YAML.load(contents) : JSON.parse(contents);
+      if ( !doc._key ) continue;
+      if ( doc._key.startsWith("!adventures") ) await reconstructAdventure(path.dirname(file), doc, {
+        transformEntry, log
+      });
       const [, collection] = doc._key.split("!");
       if ( await transformEntry?.(doc) === false ) continue;
       await packDoc(doc, collection);
@@ -315,6 +348,40 @@ async function compileClassicLevel(pack, files, { log, transformEntry }={}) {
 
 /* -------------------------------------------- */
 
+/**
+ * Collect any documents linked within an adventure.
+ * @param {string} src  The Adventure document's source directory.
+ * @param {object} doc  Adventure document being reconstructed.
+ * @param {Partial<PackageOptions>} [options]
+ * @returns {Promise<void>}
+ */
+async function reconstructAdventure(src, doc, { transformEntry, log }={}) {
+  const context = { adventure: doc };
+  for ( const embeddedCollectionName of ADVENTURE_DOCS ) {
+    const entries = [];
+    for ( let entry of doc[embeddedCollectionName] ?? [] ) {
+      if ( typeof entry === "string" ) {
+        const file = path.join(src, entry);
+        let contents;
+        try {
+          contents = fs.readFileSync(file, "utf8");
+        } catch ( err ) {
+          if ( log ) console.error(`Failed to pack ${chalk.red(file)} as part of Adventure reconstruction.`);
+          throw err;
+        }
+        const ext = path.extname(file);
+        const isYaml = ext === ".yml" || ext === ".yaml";
+        entry = isYaml ? YAML.load(contents) : JSON.parse(contents);
+        if ( await transformEntry?.(entry, context) === false ) continue;
+      }
+      entries.push(entry);
+    }
+    doc[embeddedCollectionName] = entries;
+  }
+}
+
+/* -------------------------------------------- */
+
 /**
  * Flushes the log of the given database to create compressed binary tables.
  * @param {ClassicLevel} db The database to compress.
@@ -346,7 +413,7 @@ async function compactClassicLevel(db) {
  */
 export async function extractPack(src, dest, {
   nedb=false, yaml=false, yamlOptions={}, jsonOptions={}, log=false, documentType, collection, clean, folders,
-  transformEntry, transformName, transformFolderName
+  expandAdventures, transformEntry, transformName, transformFolderName
 }={}) {
   if ( nedb && (path.extname(src) !== ".db") ) {
     throw new Error("The nedb option was passed to extractPacks, but the target pack does not have a .db extension.");
@@ -362,7 +429,7 @@ export async function extractPack(src, dest, {
     return extractNedb(src, dest, { yaml, yamlOptions, jsonOptions, log, collection, transformEntry, transformName });
   }
   return extractClassicLevel(src, dest, {
-    yaml, log, yamlOptions, jsonOptions, folders, transformEntry, transformName, transformFolderName
+    yaml, log, yamlOptions, jsonOptions, folders, expandAdventures, transformEntry, transformName, transformFolderName
   });
 }
 
@@ -413,26 +480,26 @@ async function extractNedb(pack, dest, {
  * @returns {Promise<void>}
  */
 async function extractClassicLevel(pack, dest, {
-  yaml, yamlOptions, jsonOptions, log, folders, transformEntry, transformName, transformFolderName
-}) {
+  yaml, yamlOptions, jsonOptions, log, folders, expandAdventures, transformEntry, transformName, transformFolderName
+}={}) {
   // Load the directory as a ClassicLevel DB.
   const db = new ClassicLevel(pack, { keyEncoding: "utf8", valueEncoding: "json", createIfMissing: false });
 
   // Build up the folder structure
+  const folderMap = new Map();
   if ( folders ) {
-    folders = new Map();
     for await ( const [key, doc] of db.iterator() ) {
       if ( !key.startsWith("!folders") ) continue;
       let name = await transformFolderName?.(doc);
       if ( !name ) name = doc.name ? `${getSafeFilename(doc.name)}_${doc._id}` : key;
-      folders.set(doc._id, { name, folder: doc.folder });
+      folderMap.set(doc._id, { name, folder: doc.folder });
     }
-    for ( const folder of folders.values() ) {
-      let parent = folders.get(folder.folder);
+    for ( const folder of folderMap.values() ) {
+      let parent = folderMap.get(folder.folder);
       folder.path = folder.name;
       while ( parent ) {
         folder.path = path.join(parent.name, folder.path);
-        parent = folders.get(parent.folder);
+        parent = folderMap.get(parent.folder);
       }
     }
   }
@@ -453,11 +520,17 @@ async function extractClassicLevel(pack, dest, {
     if ( collection.includes(".") ) continue; // This is not a primary document, skip it.
     await unpackDoc(doc, collection);
     if ( await transformEntry?.(doc) === false ) continue;
-    const folder = folders?.get(doc.folder)?.path;
+    if ( key.startsWith("!adventures") && expandAdventures ) {
+      await extractAdventure(doc, dest, { folderMap }, {
+        yaml, yamlOptions, jsonOptions, log, folders, transformEntry, transformName
+      });
+      continue;
+    }
+    const folder = folderMap.get(doc.folder)?.path;
     let name = await transformName?.(doc, { folder });
     if ( !name ) {
-      if ( key.startsWith("!folders") && folders?.has(doc._id) ) {
-        const folder = folders.get(doc._id);
+      if ( key.startsWith("!folders") && folderMap.has(doc._id) ) {
+        const folder = folderMap.get(doc._id);
         name = path.join(folder.name, `_Folder.${yaml ? "yml" : "json"}`);
       } else {
         name = `${doc.name ? `${getSafeFilename(doc.name)}_${id}` : key}.${yaml ? "yml" : "json"}`;
@@ -472,6 +545,60 @@ async function extractClassicLevel(pack, dest, {
   await db.close();
 }
 
+/* -------------------------------------------- */
+
+/**
+ * Split an adventure document into separate files.
+ * @param {object} doc                                The Document being operated on.
+ * @param {string} dest                               The root output directory.
+ * @param {object} [adventureOptions]                 Options to configure adventure extraction behavior.
+ * @param {Map<string, FolderDescriptor>} [adventureOptions.folderMap]  Folder hierarchy.
+ * @param {Partial<ExtractOptions>} [extractOptions]  Options to configure serialization behavior.
+ */
+async function extractAdventure(doc, dest, { folderMap }={}, {
+  yaml, yamlOptions, jsonOptions, log, folders, transformEntry, transformName
+}={}) {
+  let adventureFolder;
+
+  // Prepare name for the adventure
+  const folder = folderMap?.get(doc.folder)?.path;
+  let name = await transformName?.(doc, { folder });
+  adventureFolder = folders ? path.join(folder ?? "", `${getSafeFilename(doc.name)}_${doc._id}`) : folder;
+  if ( !name ) {
+    if ( folders ) {
+      name = path.join(adventureFolder, `_Adventure.${yaml ? "yml" : "json"}`);
+    } else {
+      name = `${doc.name ? `${getSafeFilename(doc.name)}_${doc._id}` : doc._id}.${yaml ? "yml" : "json"}`;
+      if ( folder ) name = path.join(folder, name);
+    }
+  }
+
+  // Write all documents contained in the adventure
+  const context = { adventure: { doc, path: name } };
+  for ( const embeddedCollectionName of ADVENTURE_DOCS ) {
+    const paths = [];
+    for ( const embeddedDoc of doc[embeddedCollectionName] ?? [] ) {
+      if ( await transformEntry?.(embeddedDoc, context) === false ) continue;
+      let embeddedName = await transformName?.(embeddedDoc, context);
+      if ( !embeddedName ) {
+        const { name, _id: id } = embeddedDoc;
+        embeddedName = `${name ? `${getSafeFilename(name)}_${id}` : doc._id}.${yaml ? "yml" : "json"}`;
+        if ( adventureFolder ) embeddedName = path.join(adventureFolder, embeddedName);
+      }
+      const filename = path.join(dest, embeddedName);
+      paths.push(path.basename(embeddedName));
+      serializeDocument(embeddedDoc, filename, { yaml, yamlOptions, jsonOptions });
+      if ( log ) console.log(`Wrote ${chalk.blue(embeddedName)}`);
+    }
+    doc[embeddedCollectionName] = paths;
+  }
+
+  // Write the adventure itself
+  const filename = path.join(dest, name);
+  serializeDocument(doc, filename, { yaml, yamlOptions, jsonOptions });
+  if ( log ) console.log(`Wrote ${chalk.blue(name)}`);
+}
+
 /* -------------------------------------------- */
 /*  Utilities                                   */
 /* -------------------------------------------- */