From 543ecbd3062aabc59e6f2c514d1f06c2e75c509b Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Sat, 2 Nov 2024 02:19:05 +0500 Subject: [PATCH 1/7] Use unified to parse markdown --- node-zerox/src/openAI.ts | 73 ++ package-lock.json | 1444 +++++++++++++++++++++++++++++++++++--- package.json | 4 + 3 files changed, 1420 insertions(+), 101 deletions(-) diff --git a/node-zerox/src/openAI.ts b/node-zerox/src/openAI.ts index ffd58c20..97919dfd 100644 --- a/node-zerox/src/openAI.ts +++ b/node-zerox/src/openAI.ts @@ -1,6 +1,76 @@ import { CompletionArgs, CompletionResponse } from "./types"; import { convertKeysToSnakeCase, encodeImageToBase64 } from "./utils"; import axios from "axios"; +import { nanoid } from "nanoid"; + +const markdownToJson = async (markdownString: string) => { + /** + * Bypassing typescript transpiler using eval to use dynamic imports + * + * Source: https://stackoverflow.com/a/70546326 + */ + const { unified } = await eval(`import('unified')`); + const { default: remarkParse } = await eval(`import('remark-parse')`); + const { remarkGfm } = await eval(`import('remark-gfm')`); + + const parsedMd = unified() + .use(remarkParse) // Parse Markdown to AST + .use(remarkGfm) + .parse(markdownString); + + const parentIdManager: string[] = []; + + let depths = [0]; + + const jsonObj = parsedMd.children.map((node: any) => { + const isHeading = node.type === "heading"; + if (isHeading && node.depth <= (depths.at(-1) || 0)) { + parentIdManager.pop(); + // TODO: keep removing depth number till it reaches the one less than node.depth + depths.pop(); + } + const processedNode = processNode(node, parentIdManager.at(-1)); + + if (isHeading) { + parentIdManager.push(processedNode.id); + if (depths.at(-1) !== node.depth) depths.push(node.depth); + } + + return processedNode; + }); + + return jsonObj; +}; + +const type: Record = { + heading: "heading", + text: "text", +}; + +const processNode = (node: any, parentId?: string) => { + let value: any; + + if (node.type === "heading") { + value = node.children + .map((childNode: any) => processText(childNode)) + .join(" "); + } else if (node.type === "paragraph") { + value = node.children + .map((childNode: any) => processText(childNode)) + .join(" "); + } + + return { + id: nanoid(), + parentId, + type: type[node.type as string] || type.text, + value, + }; +}; + +const processText = (node: any) => { + return node.value; +}; export const getCompletion = async ({ apiKey, @@ -58,6 +128,9 @@ export const getCompletion = async ({ const data = response.data; + // const jsonOutput = await markdownToJson(data.choices[0].message.content); + // console.log("====>>>>", JSON.stringify(jsonOutput, null, 2)); + return { content: data.choices[0].message.content, inputTokens: data.usage.prompt_tokens, diff --git a/package-lock.json b/package-lock.json index 3b25ba21..78a192d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,6 @@ "": { "name": "zerox", "version": "1.0.34", - "hasInstallScript": true, "license": "MIT", "dependencies": { "axios": "^1.7.2", @@ -15,12 +14,16 @@ "fs-extra": "^11.2.0", "libreoffice-convert": "^1.6.0", "mime-types": "^2.1.35", + "nanoid": "^3.3.7", "os": "^0.1.2", "p-limit": "^3.1.0", "path": "^0.12.7", "pdf2pic": "^3.1.1", + "remark-gfm": "^1.0.0", + "remark-parse": "^11.0.0", "sharp": "^0.33.5", "tesseract.js": "^5.1.1", + "unified": "^11.0.5", "util": "^0.12.5" }, "devDependencies": { @@ -466,6 +469,14 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "dev": true }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/fs-extra": { "version": "11.0.4", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", @@ -487,12 +498,25 @@ "@types/node": "*" } }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "dependencies": { + "@types/unist": "*" + } + }, "node_modules/@types/mime-types": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==", "dev": true }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==" + }, "node_modules/@types/node": { "version": "20.14.11", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz", @@ -513,6 +537,11 @@ "kleur": "^3.0.3" } }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==" + }, "node_modules/acorn": { "version": "8.14.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", @@ -590,6 +619,15 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/bmp-js": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", @@ -614,6 +652,42 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ccount": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-1.1.0.tgz", + "integrity": "sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz", + "integrity": "sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz", + "integrity": "sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/child_process": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/child_process/-/child_process-1.0.2.tgz", @@ -697,6 +771,18 @@ "ms": "^2.1.1" } }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -722,6 +808,14 @@ "node": ">=0.4.0" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "engines": { + "node": ">=6" + } + }, "node_modules/detect-libc": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", @@ -731,6 +825,18 @@ "node": ">=8" } }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -772,6 +878,22 @@ "node": ">= 0.4" } }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/follow-redirects": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", @@ -956,6 +1078,28 @@ "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", "license": "ISC" }, + "node_modules/is-alphabetical": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz", + "integrity": "sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz", + "integrity": "sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==", + "dependencies": { + "is-alphabetical": "^1.0.0", + "is-decimal": "^1.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-arguments": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", @@ -988,6 +1132,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-decimal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.4.tgz", + "integrity": "sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-electron": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/is-electron/-/is-electron-2.2.2.tgz", @@ -1008,6 +1161,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-hexadecimal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz", + "integrity": "sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-typed-array": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", @@ -1067,6 +1240,15 @@ "node": ">=6" } }, + "node_modules/longest-streak": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-2.0.4.tgz", + "integrity": "sha512-vM6rUVCVUJJt33bnmHiZEvr7wPT78ztX7rojL+LW51bHtLh6HTjx84LA5W4+oa6aKEJA7jJu5LR6vQRBpA5DVg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/lru-cache": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", @@ -1083,150 +1265,1073 @@ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" + "node_modules/markdown-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz", + "integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==", + "dependencies": { + "repeat-string": "^1.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "node_modules/mdast-util-find-and-replace": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-1.1.1.tgz", + "integrity": "sha512-9cKl33Y21lyckGzpSmEQnIDjEfeeWelN5s1kUW1LwdB0Fkuq2u+4GdqcGEygYxJE8GVqCl0741bYXHgamfWAZA==", "dependencies": { - "mime-db": "1.52.0" + "escape-string-regexp": "^4.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" }, - "engines": { - "node": ">= 0.6" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "node_modules/mdast-util-from-markdown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.2.tgz", + "integrity": "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", + "node_modules/mdast-util-gfm": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-0.1.2.tgz", + "integrity": "sha512-NNkhDx/qYcuOWB7xHUGWZYVXvjPFFd6afg6/e2g+SV4r9q5XUcCbV4Wfa3DLYIiD+xAEZc6K4MGaE/m0KDcPwQ==", "dependencies": { - "whatwg-url": "^5.0.0" + "mdast-util-gfm-autolink-literal": "^0.1.0", + "mdast-util-gfm-strikethrough": "^0.2.0", + "mdast-util-gfm-table": "^0.1.0", + "mdast-util-gfm-task-list-item": "^0.1.0", + "mdast-util-to-markdown": "^0.6.1" }, - "engines": { - "node": "4.x || >=6.0.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-0.1.3.tgz", + "integrity": "sha512-GjmLjWrXg1wqMIO9+ZsRik/s7PLwTaeCHVB7vRxUwLntZc8mzmTsLVr6HW1yLokcnhfURsn5zmSVdi3/xWWu1A==", + "dependencies": { + "ccount": "^1.0.0", + "mdast-util-find-and-replace": "^1.1.0", + "micromark": "^2.11.3" }, - "peerDependencies": { - "encoding": "^0.1.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" }, "peerDependenciesMeta": { - "encoding": { + "supports-color": { "optional": true } } }, - "node_modules/opencollective-postinstall": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", - "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", - "license": "MIT", - "bin": { - "opencollective-postinstall": "index.js" + "node_modules/mdast-util-gfm-autolink-literal/node_modules/micromark": { + "version": "2.11.4", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", + "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "debug": "^4.0.0", + "parse-entities": "^2.0.0" } }, - "node_modules/os": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/os/-/os-0.1.2.tgz", - "integrity": "sha512-ZoXJkvAnljwvc56MbvhtKVWmSkzV712k42Is2mA0+0KTSRakq5XXuXpjZjgAt9ctzl51ojhQWakQQpmOvXWfjQ==", - "license": "MIT" - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "node_modules/mdast-util-gfm-strikethrough": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-0.2.3.tgz", + "integrity": "sha512-5OQLXpt6qdbttcDG/UxYY7Yjj3e8P7X16LzvpX8pIQPYJ/C2Z1qFGMmcw+1PZMUM3Z8wt8NRfYTvCni93mgsgA==", "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" + "mdast-util-to-markdown": "^0.6.0" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, - "node_modules/path": { - "version": "0.12.7", - "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", - "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==", - "license": "MIT", + "node_modules/mdast-util-gfm-table": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-0.1.6.tgz", + "integrity": "sha512-j4yDxQ66AJSBwGkbpFEp9uG/LS1tZV3P33fN1gkyRB2LoRL+RR3f76m0HPHaby6F4Z5xr9Fv1URmATlRRUIpRQ==", "dependencies": { - "process": "^0.11.1", - "util": "^0.10.3" + "markdown-table": "^2.0.0", + "mdast-util-to-markdown": "~0.6.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, - "node_modules/path/node_modules/util": { - "version": "0.10.4", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", - "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "node_modules/mdast-util-gfm-task-list-item": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-0.1.6.tgz", + "integrity": "sha512-/d51FFIfPsSmCIRNp7E6pozM9z1GYPIkSy1urQ8s/o4TC22BZ7DqfHFWiqBD23bc7J3vV1Fc9O4QIHBlfuit8A==", "dependencies": { - "inherits": "2.0.3" + "mdast-util-to-markdown": "~0.6.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, - "node_modules/pdf2pic": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/pdf2pic/-/pdf2pic-3.1.1.tgz", - "integrity": "sha512-AHXsYi9EcYlSm3uUANz7h5WSktHiyTnUeHqdWmyRdjdMhgq9LgZ8pggl9FOUGuCLVfe+NKxp2k9sEMCH3tHIEg==", - "license": "MIT", + "node_modules/mdast-util-to-markdown": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-0.6.5.tgz", + "integrity": "sha512-XeV9sDE7ZlOQvs45C9UKMtfTcctcaj/pGwH8YLbMHoMOXNNCn2LsqVQOqrF1+/NU8lKDAqozme9SCXWyo9oAcQ==", "dependencies": { - "gm": "^1.25.0" + "@types/unist": "^2.0.0", + "longest-streak": "^2.0.0", + "mdast-util-to-string": "^2.0.0", + "parse-entities": "^2.0.0", + "repeat-string": "^1.0.0", + "zwitch": "^1.0.0" }, - "engines": { - "node": ">=14" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==" + }, + "node_modules/mdast-util-to-markdown/node_modules/mdast-util-to-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-2.0.0.tgz", + "integrity": "sha512-AW4DRS3QbBayY/jJmD8437V1Gombjf8RSOUCMFBuo5iHi58AGEgVCKQ+ezHkZZDpAQS75hcBMpLqjpJTjtUL7w==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "dependencies": { + "@types/mdast": "^4.0.0" }, "funding": { - "type": "paypal", - "url": "https://www.paypal.me/yakovmeister" + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, - "node_modules/possible-typed-array-names": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", - "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", - "engines": { - "node": ">= 0.4" + "node_modules/micromark": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.0.tgz", + "integrity": "sha512-o/sd0nMof8kYff+TqcDx3VSrgBTcZpSvYcAHIfHhv5VAuNmisCxjhx6YmxS8PFEpb9z5WKWKPdzf0jM23ro3RQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.1.tgz", + "integrity": "sha512-CUQyKr1e///ZODyD1U3xit6zXwy1a8q2a1S1HKtIlmgvurrEpaw/Y9y6KSIbF8P59cn/NjzHyO+Q2fAyYLQrAA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-0.3.3.tgz", + "integrity": "sha512-oVN4zv5/tAIA+l3GbMi7lWeYpJ14oQyJ3uEim20ktYFAcfX1x3LNlFGGlmrZHt7u9YlKExmyJdDGaTt6cMSR/A==", + "dependencies": { + "micromark": "~2.11.0", + "micromark-extension-gfm-autolink-literal": "~0.5.0", + "micromark-extension-gfm-strikethrough": "~0.6.5", + "micromark-extension-gfm-table": "~0.4.0", + "micromark-extension-gfm-tagfilter": "~0.3.0", + "micromark-extension-gfm-task-list-item": "~0.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, - "node_modules/process": { - "version": "0.11.10", - "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", - "license": "MIT", - "engines": { - "node": ">= 0.6.0" + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-0.5.7.tgz", + "integrity": "sha512-ePiDGH0/lhcngCe8FtH4ARFoxKTUelMp4L7Gg2pujYD5CSMb9PbblnyL+AAMud/SNMyusbS2XDSiPIRcQoNFAw==", + "dependencies": { + "micromark": "~2.11.3" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, - "node_modules/prompts": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", - "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", - "dev": true, + "node_modules/micromark-extension-gfm-autolink-literal/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", "dependencies": { - "kleur": "^3.0.3", - "sisteransi": "^1.0.5" + "ms": "^2.1.3" }, "engines": { - "node": ">= 6" + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" + "node_modules/micromark-extension-gfm-autolink-literal/node_modules/micromark": { + "version": "2.11.4", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", + "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "debug": "^4.0.0", + "parse-entities": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-0.6.5.tgz", + "integrity": "sha512-PpOKlgokpQRwUesRwWEp+fHjGGkZEejj83k9gU5iXCbDG+XBA92BqnRKYJdfqfkrRcZRgGuPuXb7DaK/DmxOhw==", + "dependencies": { + "micromark": "~2.11.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/micromark-extension-gfm-strikethrough/node_modules/micromark": { + "version": "2.11.4", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", + "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "debug": "^4.0.0", + "parse-entities": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-0.4.3.tgz", + "integrity": "sha512-hVGvESPq0fk6ALWtomcwmgLvH8ZSVpcPjzi0AjPclB9FsVRgMtGZkUcpE0zgjOCFAznKepF4z3hX8z6e3HODdA==", + "dependencies": { + "micromark": "~2.11.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/micromark-extension-gfm-table/node_modules/micromark": { + "version": "2.11.4", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", + "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "debug": "^4.0.0", + "parse-entities": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-0.3.0.tgz", + "integrity": "sha512-9GU0xBatryXifL//FJH+tAZ6i240xQuFrSL7mYi8f4oZSbc+NvXjkrHemeYP0+L4ZUT+Ptz3b95zhUZnMtoi/Q==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-0.3.3.tgz", + "integrity": "sha512-0zvM5iSLKrc/NQl84pZSjGo66aTGd57C1idmlWmE87lkMcXrTxg1uXa/nXomxJytoje9trP0NDLvw4bZ/Z/XCQ==", + "dependencies": { + "micromark": "~2.11.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/micromark-extension-gfm-task-list-item/node_modules/micromark": { + "version": "2.11.4", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", + "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "debug": "^4.0.0", + "parse-entities": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/micromark-extension-gfm/node_modules/micromark": { + "version": "2.11.4", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", + "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "debug": "^4.0.0", + "parse-entities": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.0.tgz", + "integrity": "sha512-j9DGrQLm/Uhl2tCzcbLhy5kXsgkHUrjJHg4fFAeoMRwJmJerT9aw4FEhIbZStWN8A3qMwOp1uzHr4UL8AInxtA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.0.tgz", + "integrity": "sha512-RR3i96ohZGde//4WSe/dJsxOX6vxIg9TimLAS3i4EhBAFx8Sm5SmqVfR8E87DPSR31nEAjZfbt91OMZWcNgdZw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.0.tgz", + "integrity": "sha512-TKr+LIDX2pkBJXFLzpyPyljzYK3MtmllMUMODTQJIUfDGncESaqB90db9IAUcz4AZAJFdd8U9zOp9ty1458rxg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.0.tgz", + "integrity": "sha512-jY8CSxmpWLOxS+t8W+FG3Xigc0RDQA9bKMY/EwILvsesiRniiVMejYTE4wumNc2f4UbAa4WsHqe3J1QS1sli+A==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.0.tgz", + "integrity": "sha512-28kbwaBjc5yAI1XadbdPYHX/eDnqaUFVikLwrO7FDnKG7lpgxnvk/XGRhX/PN0mOZ+dBSZ+LgunHS+6tYQAzhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.0.tgz", + "integrity": "sha512-KvOVV+X1yLBfs9dCBSopq/+G1PcgT3lAK07mC4BzXi5E7ahzMAF8oIupDDJ6mievI6F+lAATkbQQlQixJfT3aQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.0.tgz", + "integrity": "sha512-anK8SWmNphkXdaKgz5hJvGa7l00qmcaUQoMYsBwDlSKFKjc6gjGXPDw3FNL3Nbwq5L8gE+RCbGqTw49FK5Qyvg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.0.tgz", + "integrity": "sha512-S0ze2R9GH+fu41FA7pbSqNWObo/kzwf8rN/+IGlW/4tC6oACOs8B++bh+i9bVyNnwCcuksbFwsBme5OCKXCwIw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.0.tgz", + "integrity": "sha512-vZZio48k7ON0fVS3CUgFatWHoKbbLTK/rT7pzpJ4Bjp5JjkZeasRfrS9wsBdDJK2cJLHMckXZdzPSSr1B8a4oQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.1.tgz", + "integrity": "sha512-bmkNc7z8Wn6kgjZmVHOX3SowGmVdhYS7yBpMnuMnPzDq/6xwVA604DuOXMZTO1lvq01g+Adfa0pE2UKGlxL1XQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.0.tgz", + "integrity": "sha512-r4Sc6leeUTn3P6gk20aFMj2ntPwn6qpDZqWvYmAG6NgvFTIlj4WtrAudLi65qYoaGdXYViXYw2pkmn7QnIFasA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.0.tgz", + "integrity": "sha512-pS+ROfCXAGLWCOc8egcBvT0kf27GoWMqtdarNfDcjb6YLuV5cM3ioG45Ys2qOVqeqSbjaKg72vU+Wby3eddPsA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.0.tgz", + "integrity": "sha512-xNn4Pqkj2puRhKdKTm8t1YHC/BAjx6CEwRFXntTaRf/x16aqka6ouVoutm+QdkISTlT7e2zU7U4ZdlDLJd2Mcw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.0.tgz", + "integrity": "sha512-2xhYT0sfo85FMrUPtHcPo2rrp1lwbDEEzpx7jiH2xXJLqBuy4H0GgXk5ToU8IEwoROtXuL8ND0ttVa4rNqYK3w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.0.tgz", + "integrity": "sha512-6KU6qO7DZ7GJkaCgwBNtplXCvGkJToU86ybBAUdavvgsCiG8lSSvYxr9MhwmQ+udpzywHsl4RpGJsYWG1pDOcA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.0.tgz", + "integrity": "sha512-WhYv5UEcZrbAtlsnPuChHUAsu/iBPOVaEVsntLBIdpibO0ddy8OzavZz3iL2xVvBZOpolujSliP65Kq0/7KIYw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.0.1.tgz", + "integrity": "sha512-jZNtiFl/1aY73yS3UGQkutD0UbhTt68qnRpw2Pifmz5wV9h8gOVsN70v+Lq/f1rKaU/W8pxRe8y8Q9FX1AOe1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.0.tgz", + "integrity": "sha512-8JZt9ElZ5kyTnO94muPxIGS8oyElRJaiJO8EzV6ZSyGQ1Is8xwl4Q45qU5UOg+bGH4AikWziz0iN4sFLWs8PGw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.0.tgz", + "integrity": "sha512-oNh6S2WMHWRZrmutsRmDDfkzKtxF+bc2VxLC9dvtrDIRFln627VsFP6fLMgTryGDljgLPjkrzQSDcPrjPyDJ5w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, + "node_modules/os": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/os/-/os-0.1.2.tgz", + "integrity": "sha512-ZoXJkvAnljwvc56MbvhtKVWmSkzV712k42Is2mA0+0KTSRakq5XXuXpjZjgAt9ctzl51ojhQWakQQpmOvXWfjQ==", + "license": "MIT" + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse-entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", + "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", + "dependencies": { + "character-entities": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "character-reference-invalid": "^1.0.0", + "is-alphanumerical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-hexadecimal": "^1.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/character-entities": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", + "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/path": { + "version": "0.12.7", + "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", + "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==", + "license": "MIT", + "dependencies": { + "process": "^0.11.1", + "util": "^0.10.3" + } + }, + "node_modules/path/node_modules/util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", + "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "dependencies": { + "inherits": "2.0.3" + } + }, + "node_modules/pdf2pic": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/pdf2pic/-/pdf2pic-3.1.1.tgz", + "integrity": "sha512-AHXsYi9EcYlSm3uUANz7h5WSktHiyTnUeHqdWmyRdjdMhgq9LgZ8pggl9FOUGuCLVfe+NKxp2k9sEMCH3tHIEg==", + "license": "MIT", + "dependencies": { + "gm": "^1.25.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "paypal", + "url": "https://www.paypal.me/yakovmeister" + } + }, + "node_modules/possible-typed-array-names": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", + "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dev": true, + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" }, "node_modules/pseudomap": { "version": "1.0.2", @@ -1240,6 +2345,42 @@ "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", "license": "MIT" }, + "node_modules/remark-gfm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-1.0.0.tgz", + "integrity": "sha512-KfexHJCiqvrdBZVbQ6RopMZGwaXz6wFJEfByIuEwGf0arvITHjiKKZ1dpXujjH9KZdm1//XJQwgfnJ3lmXaDPA==", + "dependencies": { + "mdast-util-gfm": "^0.1.0", + "micromark-extension-gfm": "^0.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -1361,6 +2502,15 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "license": "MIT" }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/ts-node": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", @@ -1432,6 +2582,63 @@ "dev": true, "license": "MIT" }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==" + }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", @@ -1459,6 +2666,32 @@ "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", "dev": true }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz", + "integrity": "sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/wasm-feature-detect": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", @@ -1545,6 +2778,15 @@ "engines": { "node": "*" } + }, + "node_modules/zwitch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-1.0.5.tgz", + "integrity": "sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } } } } diff --git a/package.json b/package.json index 06825fc5..5146bc3a 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,16 @@ "fs-extra": "^11.2.0", "libreoffice-convert": "^1.6.0", "mime-types": "^2.1.35", + "nanoid": "^3.3.7", "os": "^0.1.2", "p-limit": "^3.1.0", "path": "^0.12.7", "pdf2pic": "^3.1.1", + "remark-gfm": "^1.0.0", + "remark-parse": "^11.0.0", "sharp": "^0.33.5", "tesseract.js": "^5.1.1", + "unified": "^11.0.5", "util": "^0.12.5" }, "devDependencies": { From c608b96350128b5bc536ad1e27c8cbf16b6eda71 Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Mon, 4 Nov 2024 19:20:48 +0500 Subject: [PATCH 2/7] Add support for lists --- node-zerox/src/openAI.ts | 76 +---------------------- node-zerox/src/utils.ts | 131 +++++++++++++++++++++++++++++++++++++++ package-lock.json | 1 + 3 files changed, 135 insertions(+), 73 deletions(-) diff --git a/node-zerox/src/openAI.ts b/node-zerox/src/openAI.ts index 97919dfd..df044569 100644 --- a/node-zerox/src/openAI.ts +++ b/node-zerox/src/openAI.ts @@ -1,76 +1,6 @@ import { CompletionArgs, CompletionResponse } from "./types"; -import { convertKeysToSnakeCase, encodeImageToBase64 } from "./utils"; +import { convertKeysToSnakeCase, encodeImageToBase64, markdownToJson } from "./utils"; import axios from "axios"; -import { nanoid } from "nanoid"; - -const markdownToJson = async (markdownString: string) => { - /** - * Bypassing typescript transpiler using eval to use dynamic imports - * - * Source: https://stackoverflow.com/a/70546326 - */ - const { unified } = await eval(`import('unified')`); - const { default: remarkParse } = await eval(`import('remark-parse')`); - const { remarkGfm } = await eval(`import('remark-gfm')`); - - const parsedMd = unified() - .use(remarkParse) // Parse Markdown to AST - .use(remarkGfm) - .parse(markdownString); - - const parentIdManager: string[] = []; - - let depths = [0]; - - const jsonObj = parsedMd.children.map((node: any) => { - const isHeading = node.type === "heading"; - if (isHeading && node.depth <= (depths.at(-1) || 0)) { - parentIdManager.pop(); - // TODO: keep removing depth number till it reaches the one less than node.depth - depths.pop(); - } - const processedNode = processNode(node, parentIdManager.at(-1)); - - if (isHeading) { - parentIdManager.push(processedNode.id); - if (depths.at(-1) !== node.depth) depths.push(node.depth); - } - - return processedNode; - }); - - return jsonObj; -}; - -const type: Record = { - heading: "heading", - text: "text", -}; - -const processNode = (node: any, parentId?: string) => { - let value: any; - - if (node.type === "heading") { - value = node.children - .map((childNode: any) => processText(childNode)) - .join(" "); - } else if (node.type === "paragraph") { - value = node.children - .map((childNode: any) => processText(childNode)) - .join(" "); - } - - return { - id: nanoid(), - parentId, - type: type[node.type as string] || type.text, - value, - }; -}; - -const processText = (node: any) => { - return node.value; -}; export const getCompletion = async ({ apiKey, @@ -128,8 +58,8 @@ export const getCompletion = async ({ const data = response.data; - // const jsonOutput = await markdownToJson(data.choices[0].message.content); - // console.log("====>>>>", JSON.stringify(jsonOutput, null, 2)); + const jsonOutput = await markdownToJson(data.choices[0].message.content); + console.log("====>>>>", JSON.stringify(jsonOutput)); return { content: data.choices[0].message.content, diff --git a/node-zerox/src/utils.ts b/node-zerox/src/utils.ts index 871f79be..5da1eb5b 100644 --- a/node-zerox/src/utils.ts +++ b/node-zerox/src/utils.ts @@ -9,6 +9,7 @@ import fs from "fs-extra"; import mime from "mime-types"; import path from "path"; import sharp from "sharp"; +import { nanoid } from "nanoid"; const convertAsync = promisify(convert); @@ -311,3 +312,133 @@ export const convertKeysToSnakeCase = ( Object.entries(obj).map(([key, value]) => [camelToSnakeCase(key), value]) ); }; + +interface ProcessedNode { + id: string; + parentId: string | undefined; + type: string; + value: any; +} +interface parentId { + id: string; + depth: number; +} + +export const markdownToJson = async (markdownString: string) => { + /** + * Bypassing typescript transpiler using eval to use dynamic imports + * + * Source: https://stackoverflow.com/a/70546326 + */ + const { unified } = await eval(`import('unified')`); + const { default: remarkParse } = await eval(`import('remark-parse')`); + const { remarkGfm } = await eval(`import('remark-gfm')`); + + const parsedMd = unified() + .use(remarkParse) // Parse Markdown to AST + .use(remarkGfm) + .parse(markdownString); + + console.log(JSON.stringify(parsedMd)); + + const parentIdManager: parentId[] = []; + + const jsonObj: ProcessedNode[] = []; + parsedMd.children.forEach((node: any) => { + const isHeading = node.type === "heading"; + + if (isHeading && node.depth <= (parentIdManager.at(-1)?.depth || 0)) { + for (let i = parentIdManager.length; i > 0; i--) { + parentIdManager.pop(); + if (node.depth > (parentIdManager.at(-1)?.depth || 0)) { + break; + } + } + } + const processedNode = processNode(node, parentIdManager.at(-1)?.id); + + if (isHeading) { + parentIdManager.push({ id: processedNode[0].id, depth: node.depth }); + } + + jsonObj.push(...processedNode); + }); + + return jsonObj; +}; + +const type: Record = { + heading: "heading", + text: "text", + list: "list", +}; + +const processNode = (node: any, parentId?: string): ProcessedNode[] => { + let value: any; + let siblingNodes: ProcessedNode[] = []; + + if (node.type === "heading") { + value = node.children + .map((childNode: any) => processText(childNode)) + .join(" "); + } else if (node.type === "paragraph") { + value = node.children + .map((childNode: any) => processText(childNode)) + .join(" "); + } else if (node.type === "list") { + const processedNodes = node.children.map((childNode: any) => + processListItem(childNode) + ); + value = []; + processedNodes.forEach((pn: any) => { + value.push(...pn.node); + siblingNodes.push(...pn.siblings); + }); + } + + return [ + { + id: nanoid(), + parentId, + type: type[node.type as string] || type.text, + value, + }, + ...(siblingNodes || []), + ]; +}; + +const processText = (node: any) => { + return node.value; +}; + +const processListItem = (node: any) => { + let newNode: ProcessedNode[] = []; + let siblings: ProcessedNode[] = []; + + node.children.forEach((childNode: any) => { + if (childNode.type !== "list") { + const processedNode = processNode(childNode); + if (newNode.length > 0) { + newNode[0].value += processedNode.map(({ value }) => value).join(", "); + } else { + newNode[0] = processedNode[0]; + } + siblings.push(...processedNode.slice(1)); + } else { + if (newNode.length == 0) { + newNode = [ + { + id: nanoid(), + type: "text", + value: "", + parentId: undefined, + }, + ]; + } + const processedNode = processNode(childNode, newNode[0].id); + siblings.push(...processedNode); + } + }); + + return { node: newNode, siblings }; +}; diff --git a/package-lock.json b/package-lock.json index 78a192d5..83dc5e0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "zerox", "version": "1.0.34", + "hasInstallScript": true, "license": "MIT", "dependencies": { "axios": "^1.7.2", From fab05db26cacc117589317e6ef19aa28883345cd Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Mon, 4 Nov 2024 22:25:24 +0500 Subject: [PATCH 3/7] Improve typing, add page number to nodes and a bit of clean up --- node-zerox/src/index.ts | 11 ++-- node-zerox/src/openAI.ts | 19 +++++-- node-zerox/src/types.ts | 51 +++++++++++++++++++ node-zerox/src/utils.ts | 105 +++++++++++++++++++++++---------------- node-zerox/tsconfig.json | 3 +- 5 files changed, 138 insertions(+), 51 deletions(-) diff --git a/node-zerox/src/index.ts b/node-zerox/src/index.ts index c7528536..5bbfe818 100644 --- a/node-zerox/src/index.ts +++ b/node-zerox/src/index.ts @@ -97,7 +97,7 @@ export const zerox = async ({ if (maintainFormat) { // Use synchronous processing - for (const image of images) { + for (const [idx, image] of images.entries()) { const imagePath = path.join(tempDirectory, image); try { const { content, inputTokens, outputTokens } = await getCompletion({ @@ -106,6 +106,7 @@ export const zerox = async ({ llmParams, maintainFormat, model, + pageNumber: idx + 1, priorPage, }); const formattedMarkdown = formatMarkdown(content); @@ -124,7 +125,10 @@ export const zerox = async ({ } } else { // Process in parallel with a limit on concurrent pages - const processPage = async (image: string): Promise => { + const processPage = async ( + image: string, + pageNumber: number + ): Promise => { const imagePath = path.join(tempDirectory, image); try { const { content, inputTokens, outputTokens } = await getCompletion({ @@ -133,6 +137,7 @@ export const zerox = async ({ llmParams, maintainFormat, model, + pageNumber, priorPage, }); const formattedMarkdown = formatMarkdown(content); @@ -156,7 +161,7 @@ export const zerox = async ({ const promises = images.map((image, index) => limit(() => - processPage(image).then((result) => { + processPage(image, index + 1).then((result) => { results[index] = result; }) ) diff --git a/node-zerox/src/openAI.ts b/node-zerox/src/openAI.ts index df044569..92135d93 100644 --- a/node-zerox/src/openAI.ts +++ b/node-zerox/src/openAI.ts @@ -1,5 +1,9 @@ import { CompletionArgs, CompletionResponse } from "./types"; -import { convertKeysToSnakeCase, encodeImageToBase64, markdownToJson } from "./utils"; +import { + convertKeysToSnakeCase, + encodeImageToBase64, + markdownToJson, +} from "./utils"; import axios from "axios"; export const getCompletion = async ({ @@ -8,6 +12,7 @@ export const getCompletion = async ({ llmParams, maintainFormat, model, + pageNumber, priorPage, }: CompletionArgs): Promise => { const systemPrompt = ` @@ -58,13 +63,21 @@ export const getCompletion = async ({ const data = response.data; - const jsonOutput = await markdownToJson(data.choices[0].message.content); - console.log("====>>>>", JSON.stringify(jsonOutput)); + const jsonOutput = await markdownToJson( + data.choices[0].message.content, + pageNumber + ); + + // TODO: remove this + // Only for development + console.log('======') + console.log(JSON.stringify(jsonOutput)); return { content: data.choices[0].message.content, inputTokens: data.usage.prompt_tokens, outputTokens: data.usage.completion_tokens, + structuredContent: jsonOutput, }; } catch (err) { console.error("Error in OpenAI completion", err); diff --git a/node-zerox/src/types.ts b/node-zerox/src/types.ts index ad701700..33391692 100644 --- a/node-zerox/src/types.ts +++ b/node-zerox/src/types.ts @@ -36,6 +36,7 @@ export interface CompletionResponse { content: string; inputTokens: number; outputTokens: number; + structuredContent: ProcessedNode[]; } export interface CompletionArgs { @@ -44,6 +45,7 @@ export interface CompletionArgs { llmParams?: LLMParams; maintainFormat: boolean; model: ModelOptions | string; + pageNumber: number; priorPage: string; } @@ -54,3 +56,52 @@ export interface LLMParams { temperature?: number; topP?: number; } + +export enum MdNodeType { + break = "break", + heading = "heading", + list = "list", + paragraph = "paragraph", + strong = "strong", + table = "table", + text = "text", + thematicBreak = "thematicBreak", +} + +export enum ConvertedNodeType { + heading = "heading", + list = "list", + text = "text", +} +export interface BaseNode { + id: string; + page?: number; + parentId?: string; +} + +export interface TextNode extends BaseNode { + type: ConvertedNodeType.text; + value: string; +} + +export interface HeadingNode extends BaseNode { + type: ConvertedNodeType.heading; + value: string; +} + +export interface ListNode extends BaseNode { + type: ConvertedNodeType.list; + value: ListItem[]; +} + +export interface ListItem { + id: string; + value: string; +} + +export type ProcessedNode = TextNode | HeadingNode | ListNode; + +export interface ParentId { + depth: number; + id: string; +} diff --git a/node-zerox/src/utils.ts b/node-zerox/src/utils.ts index 5da1eb5b..6757ab9b 100644 --- a/node-zerox/src/utils.ts +++ b/node-zerox/src/utils.ts @@ -1,6 +1,12 @@ import { convert } from "libreoffice-convert"; import { fromPath } from "pdf2pic"; -import { LLMParams } from "./types"; +import { + ConvertedNodeType, + LLMParams, + MdNodeType, + ParentId, + ProcessedNode, +} from "./types"; import { pipeline } from "stream/promises"; import { promisify } from "util"; import * as Tesseract from "tesseract.js"; @@ -313,18 +319,13 @@ export const convertKeysToSnakeCase = ( ); }; -interface ProcessedNode { - id: string; - parentId: string | undefined; - type: string; - value: any; -} -interface parentId { - id: string; - depth: number; -} - -export const markdownToJson = async (markdownString: string) => { +/** + * + * @param markdownString String - Markdown text + * @param page Number - Page number + * @returns ProcessedNode[] - Array of processed nodes + */ +export const markdownToJson = async (markdownString: string, page: number) => { /** * Bypassing typescript transpiler using eval to use dynamic imports * @@ -341,57 +342,64 @@ export const markdownToJson = async (markdownString: string) => { console.log(JSON.stringify(parsedMd)); - const parentIdManager: parentId[] = []; + const parentIdManager: ParentId[] = []; - const jsonObj: ProcessedNode[] = []; - parsedMd.children.forEach((node: any) => { - const isHeading = node.type === "heading"; + const processedNodes: ProcessedNode[] = []; + parsedMd.children.forEach((sourceNode: any) => { + const isHeading = sourceNode.type === MdNodeType.heading; - if (isHeading && node.depth <= (parentIdManager.at(-1)?.depth || 0)) { + if (isHeading && sourceNode.depth <= (parentIdManager.at(-1)?.depth || 0)) { for (let i = parentIdManager.length; i > 0; i--) { parentIdManager.pop(); - if (node.depth > (parentIdManager.at(-1)?.depth || 0)) { + if (sourceNode.depth > (parentIdManager.at(-1)?.depth || 0)) { break; } } } - const processedNode = processNode(node, parentIdManager.at(-1)?.id); + const processedNode = processNode( + sourceNode, + page, + parentIdManager.at(-1)?.id + ); if (isHeading) { - parentIdManager.push({ id: processedNode[0].id, depth: node.depth }); + parentIdManager.push({ + id: processedNode[0].id, + depth: sourceNode.depth, + }); } - jsonObj.push(...processedNode); + processedNodes.push(...processedNode); }); - return jsonObj; -}; - -const type: Record = { - heading: "heading", - text: "text", - list: "list", + return processedNodes; }; -const processNode = (node: any, parentId?: string): ProcessedNode[] => { +const processNode = ( + node: any, + page: number, + parentId?: string +): ProcessedNode[] => { let value: any; let siblingNodes: ProcessedNode[] = []; - if (node.type === "heading") { - value = node.children - .map((childNode: any) => processText(childNode)) - .join(" "); - } else if (node.type === "paragraph") { + if ( + node.type === MdNodeType.heading || + node.type === MdNodeType.paragraph || + node.type === MdNodeType.strong + ) { value = node.children .map((childNode: any) => processText(childNode)) .join(" "); - } else if (node.type === "list") { + } else if (node.type === MdNodeType.list) { const processedNodes = node.children.map((childNode: any) => - processListItem(childNode) + processListItem(childNode, page) ); value = []; processedNodes.forEach((pn: any) => { value.push(...pn.node); + + // Store nested list nodes siblingNodes.push(...pn.siblings); }); } @@ -399,25 +407,34 @@ const processNode = (node: any, parentId?: string): ProcessedNode[] => { return [ { id: nanoid(), + page, parentId, - type: type[node.type as string] || type.text, + type: + ConvertedNodeType[node.type as ConvertedNodeType] || + ConvertedNodeType.text, value, }, ...(siblingNodes || []), ]; }; +const ignoreNodeTypes = new Set([MdNodeType.break, MdNodeType.thematicBreak]); + const processText = (node: any) => { - return node.value; + if (ignoreNodeTypes.has(node.type)) return ""; + + return node.type === MdNodeType.text + ? node.value + : node.children.map((child: any) => processText(child)).join(" "); }; -const processListItem = (node: any) => { +const processListItem = (node: any, page: number) => { let newNode: ProcessedNode[] = []; let siblings: ProcessedNode[] = []; node.children.forEach((childNode: any) => { - if (childNode.type !== "list") { - const processedNode = processNode(childNode); + if (childNode.type !== MdNodeType.list) { + const processedNode = processNode(childNode, page); if (newNode.length > 0) { newNode[0].value += processedNode.map(({ value }) => value).join(", "); } else { @@ -429,13 +446,13 @@ const processListItem = (node: any) => { newNode = [ { id: nanoid(), - type: "text", + type: ConvertedNodeType.text, value: "", parentId: undefined, }, ]; } - const processedNode = processNode(childNode, newNode[0].id); + const processedNode = processNode(childNode, page, newNode[0].id); siblings.push(...processedNode); } }); diff --git a/node-zerox/tsconfig.json b/node-zerox/tsconfig.json index f14c077a..b843df01 100644 --- a/node-zerox/tsconfig.json +++ b/node-zerox/tsconfig.json @@ -6,7 +6,8 @@ "outDir": "./dist", "strict": true, "esModuleInterop": true, - "skipLibCheck": true + "skipLibCheck": true, + "downlevelIteration": true, }, "include": ["src/**/*"], "exclude": ["node_modules", "**/*.test.ts"] From 219887fea312ff18e8c3c66102947e54eb0c848b Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Tue, 5 Nov 2024 19:53:34 +0500 Subject: [PATCH 4/7] Handle more markdown node types and refactor --- node-zerox/src/types.ts | 17 +++- node-zerox/src/utils.ts | 167 +++++++++++++++++++++++++++------------- 2 files changed, 129 insertions(+), 55 deletions(-) diff --git a/node-zerox/src/types.ts b/node-zerox/src/types.ts index 33391692..900e7d7b 100644 --- a/node-zerox/src/types.ts +++ b/node-zerox/src/types.ts @@ -57,15 +57,28 @@ export interface LLMParams { topP?: number; } +// Source: https://github.com/syntax-tree/mdast?tab=readme-ov-file export enum MdNodeType { - break = "break", + blockquote = "blockquote", + break = "break", // ignored + code = "code", + definition = "definition", // ignored + emphasis = "emphasis", heading = "heading", + html = "html", + image = "image", // ignored + imageReference = "imageReference", // ignored + inlineCode = "inlineCode", + link = "link", + linkReference = "linkReference", // ignored list = "list", + listItem = "listItem", paragraph = "paragraph", + root = "root", strong = "strong", table = "table", text = "text", - thematicBreak = "thematicBreak", + thematicBreak = "thematicBreak", // ignored } export enum ConvertedNodeType { diff --git a/node-zerox/src/utils.ts b/node-zerox/src/utils.ts index 6757ab9b..18aba35f 100644 --- a/node-zerox/src/utils.ts +++ b/node-zerox/src/utils.ts @@ -2,6 +2,7 @@ import { convert } from "libreoffice-convert"; import { fromPath } from "pdf2pic"; import { ConvertedNodeType, + ListItem, LLMParams, MdNodeType, ParentId, @@ -356,7 +357,7 @@ export const markdownToJson = async (markdownString: string, page: number) => { } } } - const processedNode = processNode( + const processedNode = processRootNode( sourceNode, page, parentIdManager.at(-1)?.id @@ -375,85 +376,145 @@ export const markdownToJson = async (markdownString: string, page: number) => { return processedNodes; }; -const processNode = ( +// MD (mdast) type source: https://github.com/syntax-tree/mdast?tab=readme-ov-file + +// All of the parent nodes are composed of these literals. +const literals = new Set([ + MdNodeType.code, + MdNodeType.html, + MdNodeType.inlineCode, + MdNodeType.text, +]); + +// No data will be pulled from these nodes. +const ignoreNodeTypes = new Set([ + MdNodeType.break, + MdNodeType.definition, + MdNodeType.image, + MdNodeType.imageReference, + MdNodeType.thematicBreak, +]); + +const gatherValue = (result: any) => + result.filter((r: any) => Boolean(r)).map((r: any) => r.mainNode.value); +const gatherSiblings = (result: any) => + result.reduce( + (acc: any[], r: any) => (Boolean(r) ? acc.concat(r.siblings) : acc), + [] + ); + +const nodeProcessors = { + // Default processor for "literal" nodes + literal: (node: any) => ({ value: node.value, siblings: [] }), + [MdNodeType.list]: (node: any, page: number) => { + const result = node.children.map((childNode: any) => + processNode(childNode, page) + ); + + return { + value: gatherValue(result), + siblings: gatherSiblings(result), + }; + }, + [MdNodeType.listItem]: (node: any, page: number) => { + const result = processListItem(node, page); + return { + value: result.node, + siblings: result.siblings, + }; + }, + // Default processor for "parent" nodes + parent: (node: any, page: number) => { + const result: string[] = node.children.map((childNode: any) => + processNode(childNode, page) + ); + + return { + value: gatherValue(result).join(" "), + siblings: gatherSiblings(result), + }; + }, +}; + +const processRootNode = ( node: any, page: number, parentId?: string ): ProcessedNode[] => { - let value: any; - let siblingNodes: ProcessedNode[] = []; + const nodes = processNode(node, page, parentId); - if ( - node.type === MdNodeType.heading || - node.type === MdNodeType.paragraph || - node.type === MdNodeType.strong - ) { - value = node.children - .map((childNode: any) => processText(childNode)) - .join(" "); - } else if (node.type === MdNodeType.list) { - const processedNodes = node.children.map((childNode: any) => - processListItem(childNode, page) - ); - value = []; - processedNodes.forEach((pn: any) => { - value.push(...pn.node); + if (!nodes) return []; - // Store nested list nodes - siblingNodes.push(...pn.siblings); - }); + return [nodes.mainNode, ...nodes.siblings]; +}; + +const processNode = ( + node: any, + page: number, + parentId?: string +): { mainNode: ProcessedNode; siblings: ProcessedNode[] } | undefined => { + let value: string | ListItem[]; + let siblingNodes: ProcessedNode[] = []; + let result; + + if (literals.has(node.type)) { + result = nodeProcessors["literal"](node); + } else if (node.type in nodeProcessors) { + result = nodeProcessors[node.type as keyof typeof nodeProcessors]( + node, + page + ); + } else if (!ignoreNodeTypes.has(node.type)) { + result = nodeProcessors["parent"](node, page); + } else { + return; } - return [ - { + value = result.value; + siblingNodes = result.siblings; + + return { + mainNode: { id: nanoid(), page, parentId, type: ConvertedNodeType[node.type as ConvertedNodeType] || ConvertedNodeType.text, - value, + value: value as any, }, - ...(siblingNodes || []), - ]; -}; - -const ignoreNodeTypes = new Set([MdNodeType.break, MdNodeType.thematicBreak]); - -const processText = (node: any) => { - if (ignoreNodeTypes.has(node.type)) return ""; - - return node.type === MdNodeType.text - ? node.value - : node.children.map((child: any) => processText(child)).join(" "); + siblings: siblingNodes, + }; }; const processListItem = (node: any, page: number) => { - let newNode: ProcessedNode[] = []; + let newNode: ProcessedNode | undefined; let siblings: ProcessedNode[] = []; node.children.forEach((childNode: any) => { if (childNode.type !== MdNodeType.list) { const processedNode = processNode(childNode, page); - if (newNode.length > 0) { - newNode[0].value += processedNode.map(({ value }) => value).join(", "); + if (!processedNode) return; + + if (newNode) { + newNode.value += processedNode.mainNode.value as string; } else { - newNode[0] = processedNode[0]; + newNode = processedNode.mainNode; } - siblings.push(...processedNode.slice(1)); + siblings.push(...processedNode.siblings); } else { - if (newNode.length == 0) { - newNode = [ - { - id: nanoid(), - type: ConvertedNodeType.text, - value: "", - parentId: undefined, - }, - ]; + if (!newNode) { + newNode = { + id: nanoid(), + parentId: undefined, + type: ConvertedNodeType.text, + value: "", + }; } - const processedNode = processNode(childNode, page, newNode[0].id); - siblings.push(...processedNode); + + const processedNode = processNode(childNode, page, newNode?.id); + if (processedNode) + siblings.push(processedNode.mainNode, ...processedNode.siblings); } }); From 26b58f10739299f4ecbbf5d0c8726b586eb8d2e4 Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Tue, 5 Nov 2024 23:51:37 +0500 Subject: [PATCH 5/7] Handle tables --- node-zerox/src/types.ts | 13 +- node-zerox/src/utils.ts | 34 ++- package-lock.json | 616 ++++++++++++++-------------------------- package.json | 2 +- 4 files changed, 252 insertions(+), 413 deletions(-) diff --git a/node-zerox/src/types.ts b/node-zerox/src/types.ts index 900e7d7b..4079be01 100644 --- a/node-zerox/src/types.ts +++ b/node-zerox/src/types.ts @@ -77,6 +77,8 @@ export enum MdNodeType { root = "root", strong = "strong", table = "table", + tableCell = "tableCell", + tableRow = "tableRow", text = "text", thematicBreak = "thematicBreak", // ignored } @@ -84,6 +86,7 @@ export enum MdNodeType { export enum ConvertedNodeType { heading = "heading", list = "list", + table = "table", text = "text", } export interface BaseNode { @@ -112,7 +115,15 @@ export interface ListItem { value: string; } -export type ProcessedNode = TextNode | HeadingNode | ListNode; +export interface TableNode extends BaseNode { + type: ConvertedNodeType.table; + value: { + header: string[]; + rows: Record[]; + }; +} + +export type ProcessedNode = HeadingNode | ListNode | TableNode | TextNode; export interface ParentId { depth: number; diff --git a/node-zerox/src/utils.ts b/node-zerox/src/utils.ts index 18aba35f..21559827 100644 --- a/node-zerox/src/utils.ts +++ b/node-zerox/src/utils.ts @@ -334,7 +334,7 @@ export const markdownToJson = async (markdownString: string, page: number) => { */ const { unified } = await eval(`import('unified')`); const { default: remarkParse } = await eval(`import('remark-parse')`); - const { remarkGfm } = await eval(`import('remark-gfm')`); + const { default: remarkGfm } = await eval(`import('remark-gfm')`); const parsedMd = unified() .use(remarkParse) // Parse Markdown to AST @@ -423,6 +423,38 @@ const nodeProcessors = { siblings: result.siblings, }; }, + [MdNodeType.table]: (node: any, page: number) => { + const [headerNodes, ...rowNodes] = node.children; + + const headers = headerNodes.children.map((child: any) => { + const pn = processNode(child, page)?.mainNode; + return pn ? { value: pn.value, id: pn.id } : {}; + }); + + const rows = rowNodes.map((rowNode: any) => { + return Object.fromEntries( + rowNode.children.map((cellNode: any, idx: number) => { + const headerId = headers[idx].id; + if (!headerId) return; + + return [ + headerId, + cellNode.children + .map((child: any) => processNode(child, page)?.mainNode.value) + .join(" "), + ]; + }) + ); + }); + + return { + value: { + headers, + rows, + }, + siblings: [], + }; + }, // Default processor for "parent" nodes parent: (node: any, page: number) => { const result: string[] = node.children.map((childNode: any) => diff --git a/package-lock.json b/package-lock.json index 83dc5e0f..53a658a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "p-limit": "^3.1.0", "path": "^0.12.7", "pdf2pic": "^3.1.1", - "remark-gfm": "^1.0.0", + "remark-gfm": "^4.0.0", "remark-parse": "^11.0.0", "sharp": "^0.33.5", "tesseract.js": "^5.1.1", @@ -654,9 +654,9 @@ } }, "node_modules/ccount": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/ccount/-/ccount-1.1.0.tgz", - "integrity": "sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -671,24 +671,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/character-entities-legacy": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz", - "integrity": "sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-reference-invalid": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz", - "integrity": "sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/child_process": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/child_process/-/child_process-1.0.2.tgz", @@ -880,11 +862,11 @@ } }, "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", "engines": { - "node": ">=10" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -1079,28 +1061,6 @@ "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", "license": "ISC" }, - "node_modules/is-alphabetical": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz", - "integrity": "sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/is-alphanumerical": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz", - "integrity": "sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==", - "dependencies": { - "is-alphabetical": "^1.0.0", - "is-decimal": "^1.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/is-arguments": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", @@ -1133,15 +1093,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-decimal": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.4.tgz", - "integrity": "sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/is-electron": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/is-electron/-/is-electron-2.2.2.tgz", @@ -1162,15 +1113,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-hexadecimal": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz", - "integrity": "sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/is-plain-obj": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", @@ -1242,9 +1184,9 @@ } }, "node_modules/longest-streak": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-2.0.4.tgz", - "integrity": "sha512-vM6rUVCVUJJt33bnmHiZEvr7wPT78ztX7rojL+LW51bHtLh6HTjx84LA5W4+oa6aKEJA7jJu5LR6vQRBpA5DVg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -1267,25 +1209,23 @@ "dev": true }, "node_modules/markdown-table": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz", - "integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==", - "dependencies": { - "repeat-string": "^1.0.0" - }, + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" } }, "node_modules/mdast-util-find-and-replace": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-1.1.1.tgz", - "integrity": "sha512-9cKl33Y21lyckGzpSmEQnIDjEfeeWelN5s1kUW1LwdB0Fkuq2u+4GdqcGEygYxJE8GVqCl0741bYXHgamfWAZA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.1.tgz", + "integrity": "sha512-SG21kZHGC3XRTSUhtofZkBzZTJNM5ecCi0SK2IMKmSXR8vO3peL+kb1O0z7Zl83jKtutG4k5Wv/W7V3/YHvzPA==", "dependencies": { - "escape-string-regexp": "^4.0.0", - "unist-util-is": "^4.0.0", - "unist-util-visit-parents": "^3.0.0" + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" }, "funding": { "type": "opencollective", @@ -1316,15 +1256,17 @@ } }, "node_modules/mdast-util-gfm": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-0.1.2.tgz", - "integrity": "sha512-NNkhDx/qYcuOWB7xHUGWZYVXvjPFFd6afg6/e2g+SV4r9q5XUcCbV4Wfa3DLYIiD+xAEZc6K4MGaE/m0KDcPwQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.0.0.tgz", + "integrity": "sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw==", "dependencies": { - "mdast-util-gfm-autolink-literal": "^0.1.0", - "mdast-util-gfm-strikethrough": "^0.2.0", - "mdast-util-gfm-table": "^0.1.0", - "mdast-util-gfm-task-list-item": "^0.1.0", - "mdast-util-to-markdown": "^0.6.1" + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" }, "funding": { "type": "opencollective", @@ -1332,60 +1274,45 @@ } }, "node_modules/mdast-util-gfm-autolink-literal": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-0.1.3.tgz", - "integrity": "sha512-GjmLjWrXg1wqMIO9+ZsRik/s7PLwTaeCHVB7vRxUwLntZc8mzmTsLVr6HW1yLokcnhfURsn5zmSVdi3/xWWu1A==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", "dependencies": { - "ccount": "^1.0.0", - "mdast-util-find-and-replace": "^1.1.0", - "micromark": "^2.11.3" + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, - "node_modules/mdast-util-gfm-autolink-literal/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "node_modules/mdast-util-gfm-footnote": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.0.0.tgz", + "integrity": "sha512-5jOT2boTSVkMnQ7LTrd6n/18kqwjmuYqo7JUPe+tRCY6O7dAuTFMtTPauYYrMPpox9hlN0uOx/FL8XvEfG9/mQ==", "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/mdast-util-gfm-autolink-literal/node_modules/micromark": { - "version": "2.11.4", - "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", - "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "dependencies": { - "debug": "^4.0.0", - "parse-entities": "^2.0.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, "node_modules/mdast-util-gfm-strikethrough": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-0.2.3.tgz", - "integrity": "sha512-5OQLXpt6qdbttcDG/UxYY7Yjj3e8P7X16LzvpX8pIQPYJ/C2Z1qFGMmcw+1PZMUM3Z8wt8NRfYTvCni93mgsgA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", "dependencies": { - "mdast-util-to-markdown": "^0.6.0" + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" }, "funding": { "type": "opencollective", @@ -1393,12 +1320,15 @@ } }, "node_modules/mdast-util-gfm-table": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-0.1.6.tgz", - "integrity": "sha512-j4yDxQ66AJSBwGkbpFEp9uG/LS1tZV3P33fN1gkyRB2LoRL+RR3f76m0HPHaby6F4Z5xr9Fv1URmATlRRUIpRQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", "dependencies": { - "markdown-table": "^2.0.0", - "mdast-util-to-markdown": "~0.6.0" + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" }, "funding": { "type": "opencollective", @@ -1406,43 +1336,48 @@ } }, "node_modules/mdast-util-gfm-task-list-item": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-0.1.6.tgz", - "integrity": "sha512-/d51FFIfPsSmCIRNp7E6pozM9z1GYPIkSy1urQ8s/o4TC22BZ7DqfHFWiqBD23bc7J3vV1Fc9O4QIHBlfuit8A==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", "dependencies": { - "mdast-util-to-markdown": "~0.6.0" + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, - "node_modules/mdast-util-to-markdown": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-0.6.5.tgz", - "integrity": "sha512-XeV9sDE7ZlOQvs45C9UKMtfTcctcaj/pGwH8YLbMHoMOXNNCn2LsqVQOqrF1+/NU8lKDAqozme9SCXWyo9oAcQ==", + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", "dependencies": { - "@types/unist": "^2.0.0", - "longest-streak": "^2.0.0", - "mdast-util-to-string": "^2.0.0", - "parse-entities": "^2.0.0", - "repeat-string": "^1.0.0", - "zwitch": "^1.0.0" + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, - "node_modules/mdast-util-to-markdown/node_modules/@types/unist": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", - "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==" - }, - "node_modules/mdast-util-to-markdown/node_modules/mdast-util-to-string": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-2.0.0.tgz", - "integrity": "sha512-AW4DRS3QbBayY/jJmD8437V1Gombjf8RSOUCMFBuo5iHi58AGEgVCKQ+ezHkZZDpAQS75hcBMpLqjpJTjtUL7w==", + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" @@ -1528,16 +1463,18 @@ } }, "node_modules/micromark-extension-gfm": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-0.3.3.tgz", - "integrity": "sha512-oVN4zv5/tAIA+l3GbMi7lWeYpJ14oQyJ3uEim20ktYFAcfX1x3LNlFGGlmrZHt7u9YlKExmyJdDGaTt6cMSR/A==", - "dependencies": { - "micromark": "~2.11.0", - "micromark-extension-gfm-autolink-literal": "~0.5.0", - "micromark-extension-gfm-strikethrough": "~0.6.5", - "micromark-extension-gfm-table": "~0.4.0", - "micromark-extension-gfm-tagfilter": "~0.3.0", - "micromark-extension-gfm-task-list-item": "~0.3.0" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" }, "funding": { "type": "opencollective", @@ -1545,237 +1482,100 @@ } }, "node_modules/micromark-extension-gfm-autolink-literal": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-0.5.7.tgz", - "integrity": "sha512-ePiDGH0/lhcngCe8FtH4ARFoxKTUelMp4L7Gg2pujYD5CSMb9PbblnyL+AAMud/SNMyusbS2XDSiPIRcQoNFAw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", "dependencies": { - "micromark": "~2.11.3" + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, - "node_modules/micromark-extension-gfm-autolink-literal/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/micromark-extension-gfm-autolink-literal/node_modules/micromark": { - "version": "2.11.4", - "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", - "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "dependencies": { - "debug": "^4.0.0", - "parse-entities": "^2.0.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, "node_modules/micromark-extension-gfm-strikethrough": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-0.6.5.tgz", - "integrity": "sha512-PpOKlgokpQRwUesRwWEp+fHjGGkZEejj83k9gU5iXCbDG+XBA92BqnRKYJdfqfkrRcZRgGuPuXb7DaK/DmxOhw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", "dependencies": { - "micromark": "~2.11.0" + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, - "node_modules/micromark-extension-gfm-strikethrough/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/micromark-extension-gfm-strikethrough/node_modules/micromark": { - "version": "2.11.4", - "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", - "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "dependencies": { - "debug": "^4.0.0", - "parse-entities": "^2.0.0" - } - }, "node_modules/micromark-extension-gfm-table": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-0.4.3.tgz", - "integrity": "sha512-hVGvESPq0fk6ALWtomcwmgLvH8ZSVpcPjzi0AjPclB9FsVRgMtGZkUcpE0zgjOCFAznKepF4z3hX8z6e3HODdA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.0.tgz", + "integrity": "sha512-Ub2ncQv+fwD70/l4ou27b4YzfNaCJOvyX4HxXU15m7mpYY+rjuWzsLIPZHJL253Z643RpbcP1oeIJlQ/SKW67g==", "dependencies": { - "micromark": "~2.11.0" + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, - "node_modules/micromark-extension-gfm-table/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" + "micromark-util-types": "^2.0.0" }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/micromark-extension-gfm-table/node_modules/micromark": { - "version": "2.11.4", - "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", - "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "dependencies": { - "debug": "^4.0.0", - "parse-entities": "^2.0.0" - } - }, - "node_modules/micromark-extension-gfm-tagfilter": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-0.3.0.tgz", - "integrity": "sha512-9GU0xBatryXifL//FJH+tAZ6i240xQuFrSL7mYi8f4oZSbc+NvXjkrHemeYP0+L4ZUT+Ptz3b95zhUZnMtoi/Q==", "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, "node_modules/micromark-extension-gfm-task-list-item": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-0.3.3.tgz", - "integrity": "sha512-0zvM5iSLKrc/NQl84pZSjGo66aTGd57C1idmlWmE87lkMcXrTxg1uXa/nXomxJytoje9trP0NDLvw4bZ/Z/XCQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", "dependencies": { - "micromark": "~2.11.0" + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, - "node_modules/micromark-extension-gfm-task-list-item/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/micromark-extension-gfm-task-list-item/node_modules/micromark": { - "version": "2.11.4", - "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", - "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "dependencies": { - "debug": "^4.0.0", - "parse-entities": "^2.0.0" - } - }, - "node_modules/micromark-extension-gfm/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/micromark-extension-gfm/node_modules/micromark": { - "version": "2.11.4", - "resolved": "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz", - "integrity": "sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "dependencies": { - "debug": "^4.0.0", - "parse-entities": "^2.0.0" - } - }, "node_modules/micromark-factory-destination": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.0.tgz", @@ -2238,32 +2038,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/parse-entities": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", - "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", - "dependencies": { - "character-entities": "^1.0.0", - "character-entities-legacy": "^1.0.0", - "character-reference-invalid": "^1.0.0", - "is-alphanumerical": "^1.0.0", - "is-decimal": "^1.0.0", - "is-hexadecimal": "^1.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/parse-entities/node_modules/character-entities": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", - "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/path": { "version": "0.12.7", "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", @@ -2347,12 +2121,16 @@ "license": "MIT" }, "node_modules/remark-gfm": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-1.0.0.tgz", - "integrity": "sha512-KfexHJCiqvrdBZVbQ6RopMZGwaXz6wFJEfByIuEwGf0arvITHjiKKZ1dpXujjH9KZdm1//XJQwgfnJ3lmXaDPA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.0.tgz", + "integrity": "sha512-U92vJgBPkbw4Zfu/IiW2oTZLSL3Zpv+uI7My2eq8JxKgqraFdU8YUGicEJCEgSbeaG+QDFqIcwwfMTOEelPxuA==", "dependencies": { - "mdast-util-gfm": "^0.1.0", - "micromark-extension-gfm": "^0.3.0" + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" }, "funding": { "type": "opencollective", @@ -2374,12 +2152,18 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/repeat-string": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", - "engines": { - "node": ">=0.10" + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" } }, "node_modules/semver": { @@ -2602,9 +2386,12 @@ } }, "node_modules/unist-util-is": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", - "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", + "integrity": "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==", + "dependencies": { + "@types/unist": "^3.0.0" + }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" @@ -2622,23 +2409,32 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/unist-util-visit-parents": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", - "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "node_modules/unist-util-visit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.0.0.tgz", + "integrity": "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==", "dependencies": { - "@types/unist": "^2.0.0", - "unist-util-is": "^4.0.0" + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/unified" } }, - "node_modules/unist-util-visit-parents/node_modules/@types/unist": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", - "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==" + "node_modules/unist-util-visit-parents": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.1.tgz", + "integrity": "sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } }, "node_modules/universalify": { "version": "2.0.1", @@ -2781,9 +2577,9 @@ } }, "node_modules/zwitch": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-1.0.5.tgz", - "integrity": "sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" diff --git a/package.json b/package.json index 5146bc3a..bf56fda4 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "p-limit": "^3.1.0", "path": "^0.12.7", "pdf2pic": "^3.1.1", - "remark-gfm": "^1.0.0", + "remark-gfm": "^4.0.0", "remark-parse": "^11.0.0", "sharp": "^0.33.5", "tesseract.js": "^5.1.1", From 1fd18a2577173f56fac457aac18e4036e7af7552 Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Wed, 6 Nov 2024 04:16:16 +0500 Subject: [PATCH 6/7] Remove console logs --- node-zerox/src/openAI.ts | 5 ----- node-zerox/src/utils.ts | 2 -- 2 files changed, 7 deletions(-) diff --git a/node-zerox/src/openAI.ts b/node-zerox/src/openAI.ts index 92135d93..15d7d349 100644 --- a/node-zerox/src/openAI.ts +++ b/node-zerox/src/openAI.ts @@ -68,11 +68,6 @@ export const getCompletion = async ({ pageNumber ); - // TODO: remove this - // Only for development - console.log('======') - console.log(JSON.stringify(jsonOutput)); - return { content: data.choices[0].message.content, inputTokens: data.usage.prompt_tokens, diff --git a/node-zerox/src/utils.ts b/node-zerox/src/utils.ts index 21559827..ed7a5508 100644 --- a/node-zerox/src/utils.ts +++ b/node-zerox/src/utils.ts @@ -341,8 +341,6 @@ export const markdownToJson = async (markdownString: string, page: number) => { .use(remarkGfm) .parse(markdownString); - console.log(JSON.stringify(parsedMd)); - const parentIdManager: ParentId[] = []; const processedNodes: ProcessedNode[] = []; From dc890bf1a19b4b9798fb83fc7537821f0813954d Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Thu, 7 Nov 2024 03:02:01 +0500 Subject: [PATCH 7/7] Add chunk option that returns JSON array --- README.md | 2 ++ node-zerox/src/index.ts | 73 +++++++++++++++++++++++++++------------- node-zerox/src/openAI.ts | 16 +++++---- node-zerox/src/types.ts | 10 +++++- node-zerox/src/utils.ts | 4 +-- 5 files changed, 72 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 17c4d462..6c504b01 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ const result = await zerox({ openaiAPIKey: process.env.OPENAI_API_KEY, // Optional + chunk: false, // Return JSON array of elements on each page cleanup: true, // Clear images from tmp after run. concurrency: 10, // Number of pages to run at a time. correctOrientation: true, // True by default, attempts to identify and correct page orientation. @@ -97,6 +98,7 @@ Request #3 => page_2_markdown + page_3_image ```js { + chunks: [], completionTime: 10038, fileName: 'invoice_36258', inputTokens: 25543, diff --git a/node-zerox/src/index.ts b/node-zerox/src/index.ts index 5bbfe818..63538d26 100644 --- a/node-zerox/src/index.ts +++ b/node-zerox/src/index.ts @@ -6,7 +6,13 @@ import { isString, } from "./utils"; import { getCompletion } from "./openAI"; -import { ModelOptions, ZeroxArgs, ZeroxOutput } from "./types"; +import { + ModelOptions, + ProcessedNode, + ProcessPageResponseBody, + ZeroxArgs, + ZeroxOutput, +} from "./types"; import { validateLLMParams } from "./utils"; import fs from "fs-extra"; import os from "os"; @@ -14,6 +20,7 @@ import path from "path"; import pLimit, { Limit } from "p-limit"; export const zerox = async ({ + chunk = false, cleanup = true, concurrency = 10, correctOrientation = true, @@ -94,21 +101,24 @@ export const zerox = async ({ // Get list of converted images const files = await fs.readdir(tempDirectory); const images = files.filter((file) => file.endsWith(".png")); + const chunks: ProcessedNode[] = []; if (maintainFormat) { // Use synchronous processing for (const [idx, image] of images.entries()) { const imagePath = path.join(tempDirectory, image); try { - const { content, inputTokens, outputTokens } = await getCompletion({ - apiKey: openaiAPIKey, - imagePath, - llmParams, - maintainFormat, - model, - pageNumber: idx + 1, - priorPage, - }); + const { chunks: pageChunks, content, inputTokens, outputTokens } = + await getCompletion({ + apiKey: openaiAPIKey, + chunk, + imagePath, + llmParams, + maintainFormat, + model, + pageNumber: idx + 1, + priorPage, + }); const formattedMarkdown = formatMarkdown(content); inputTokenCount += inputTokens; outputTokenCount += outputTokens; @@ -118,6 +128,8 @@ export const zerox = async ({ // Add all markdown results to array aggregatedMarkdown.push(formattedMarkdown); + + chunks.push(...pageChunks); } catch (error) { console.error(`Failed to process image ${image}:`, error); throw error; @@ -128,18 +140,20 @@ export const zerox = async ({ const processPage = async ( image: string, pageNumber: number - ): Promise => { + ): Promise => { const imagePath = path.join(tempDirectory, image); try { - const { content, inputTokens, outputTokens } = await getCompletion({ - apiKey: openaiAPIKey, - imagePath, - llmParams, - maintainFormat, - model, - pageNumber, - priorPage, - }); + const { chunks, content, inputTokens, outputTokens } = + await getCompletion({ + chunk, + apiKey: openaiAPIKey, + imagePath, + llmParams, + maintainFormat, + model, + pageNumber, + priorPage, + }); const formattedMarkdown = formatMarkdown(content); inputTokenCount += inputTokens; outputTokenCount += outputTokens; @@ -148,7 +162,7 @@ export const zerox = async ({ priorPage = formattedMarkdown; // Add all markdown results to array - return formattedMarkdown; + return { formattedMarkdown, chunks }; } catch (error) { console.error(`Failed to process image ${image}:`, error); throw error; @@ -157,7 +171,7 @@ export const zerox = async ({ // Function to process pages with concurrency limit const processPagesInBatches = async (images: string[], limit: Limit) => { - const results: (string | null)[] = []; + const results: ProcessPageResponseBody[] = []; const promises = images.map((image, index) => limit(() => @@ -173,8 +187,18 @@ export const zerox = async ({ const limit = pLimit(concurrency); const results = await processPagesInBatches(images, limit); - const filteredResults = results.filter(isString); - aggregatedMarkdown.push(...filteredResults); + const filteredResults = results.filter( + (r) => r && isString(r.formattedMarkdown) + ); + aggregatedMarkdown.push( + ...filteredResults.map((r) => r!.formattedMarkdown) + ); + chunks.push( + ...filteredResults.reduce((acc: ProcessedNode[], r) => { + acc.push(...r!.chunks); + return acc; + }, []) + ); } // Write the aggregated markdown to a file @@ -208,6 +232,7 @@ export const zerox = async ({ }); return { + chunks, completionTime, fileName, inputTokens: inputTokenCount, diff --git a/node-zerox/src/openAI.ts b/node-zerox/src/openAI.ts index 15d7d349..ae8049de 100644 --- a/node-zerox/src/openAI.ts +++ b/node-zerox/src/openAI.ts @@ -1,4 +1,4 @@ -import { CompletionArgs, CompletionResponse } from "./types"; +import { CompletionArgs, CompletionResponse, ProcessedNode } from "./types"; import { convertKeysToSnakeCase, encodeImageToBase64, @@ -8,6 +8,7 @@ import axios from "axios"; export const getCompletion = async ({ apiKey, + chunk, imagePath, llmParams, maintainFormat, @@ -62,17 +63,20 @@ export const getCompletion = async ({ ); const data = response.data; + let chunks: ProcessedNode[] = []; - const jsonOutput = await markdownToJson( - data.choices[0].message.content, - pageNumber - ); + if (chunk) { + chunks = await markdownToJson( + data.choices[0].message.content, + pageNumber + ); + } return { + chunks, content: data.choices[0].message.content, inputTokens: data.usage.prompt_tokens, outputTokens: data.usage.completion_tokens, - structuredContent: jsonOutput, }; } catch (err) { console.error("Error in OpenAI completion", err); diff --git a/node-zerox/src/types.ts b/node-zerox/src/types.ts index 4079be01..9b435abd 100644 --- a/node-zerox/src/types.ts +++ b/node-zerox/src/types.ts @@ -1,4 +1,5 @@ export interface ZeroxArgs { + chunk?: boolean; cleanup?: boolean; concurrency?: number; correctOrientation?: boolean; @@ -30,17 +31,19 @@ export interface ZeroxOutput { inputTokens: number; outputTokens: number; pages: Page[]; + chunks: ProcessedNode[]; } export interface CompletionResponse { content: string; inputTokens: number; outputTokens: number; - structuredContent: ProcessedNode[]; + chunks: ProcessedNode[]; } export interface CompletionArgs { apiKey: string; + chunk: boolean; imagePath: string; llmParams?: LLMParams; maintainFormat: boolean; @@ -57,6 +60,11 @@ export interface LLMParams { topP?: number; } +export type ProcessPageResponseBody = { + formattedMarkdown: string; + chunks: ProcessedNode[]; +} | null + // Source: https://github.com/syntax-tree/mdast?tab=readme-ov-file export enum MdNodeType { blockquote = "blockquote", diff --git a/node-zerox/src/utils.ts b/node-zerox/src/utils.ts index ed7a5508..5cc118be 100644 --- a/node-zerox/src/utils.ts +++ b/node-zerox/src/utils.ts @@ -432,8 +432,8 @@ const nodeProcessors = { const rows = rowNodes.map((rowNode: any) => { return Object.fromEntries( rowNode.children.map((cellNode: any, idx: number) => { - const headerId = headers[idx].id; - if (!headerId) return; + const headerId = headers[idx]?.id; + if (!headerId) return []; return [ headerId,