Add ability to change model for completion and better formatting

core/dist/index.d.ts

@@ -1,2 +1,2 @@
-import { DocumindArgs, DocumindOutput } from "./types";
+import { type DocumindArgs, type DocumindOutput } from "./types";
 export declare const documind: ({ cleanup, concurrency, filePath, llmParams, maintainFormat, model, openaiAPIKey, outputDir, pagesToConvertAsImages, tempDir, }: DocumindArgs) => Promise<DocumindOutput>;

core/dist/index.js

@@ -4,14 +4,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.documind = void 0;
-const utils_1 = require("./utils");
-const openAI_1 = require("./openAI");
-const types_1 = require("./types");
-const utils_2 = require("./utils");
-const fs_extra_1 = __importDefault(require("fs-extra"));
 const os_1 = __importDefault(require("os"));
 const path_1 = __importDefault(require("path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
 const p_limit_1 = __importDefault(require("p-limit"));
+const openAI_1 = require("./openAI");
+const types_1 = require("./types");
+const utils_1 = require("./utils");
+const utils_2 = require("./utils");
 const documind = async ({ cleanup = true, concurrency = 10, filePath, llmParams = {}, maintainFormat = false, model, //= ModelOptions.gpt_4o_mini,
 openaiAPIKey = "", outputDir, pagesToConvertAsImages = -1, tempDir = os_1.default.tmpdir(), }) => {
     const baseUrl = process.env.BASE_URL || "https://api.openai.com/v1";

core/dist/openAI.d.ts

@@ -1,2 +1,2 @@
-import { CompletionArgs, CompletionResponse } from "./types";
+import type { CompletionArgs, CompletionResponse } from "./types";
 export declare const getCompletion: ({ apiKey, imagePath, llmParams, maintainFormat, model, priorPage, }: CompletionArgs) => Promise<CompletionResponse>;

core/dist/openAI.js

@@ -4,13 +4,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getCompletion = void 0;
-const utils_1 = require("./utils");
 const axios_1 = __importDefault(require("axios"));
+const utils_1 = require("./utils");
 const getCompletion = async ({ apiKey, imagePath, llmParams, maintainFormat, model, priorPage, }) => {
-    const validModelsForCustomBaseUrl = [
-        "llava",
-        "llama3.2-vision",
-    ];
+    const validModelsForCustomBaseUrl = ["llava", "llama3.2-vision"];
     const validModelsForOpenAi = ["gpt-4o", "gpt-4o-mini"];
     const baseUrl = process.env.BASE_URL || "https://api.openai.com/v1";
     if (baseUrl !== "https://api.openai.com/v1") {
@@ -24,7 +21,7 @@ const getCompletion = async ({ apiKey, imagePath, llmParams, maintainFormat, mod
         }
     }
     const systemPrompt = `
-    Convert the following image/document  to markdown. 
+    Convert the following image/document  to markdown.
     Return only the markdown with no explanation text. Do not include deliminators like '''markdown.
     You must include all information on the page. Do not exclude headers, footers, or subtext.
   `;

core/dist/types.d.ts

@@ -11,6 +11,7 @@ export interface DocumindArgs {
     tempDir?: string;
 }
 export declare enum ModelOptions {
+    gpt_o3_mini = "o3-mini",
     gpt_4o = "gpt-4o",
     gpt_4o_mini = "gpt-4o-mini",
     llava = "llava",

core/dist/types.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ModelOptions = void 0;
 var ModelOptions;
 (function (ModelOptions) {
+    ModelOptions["gpt_o3_mini"] = "o3-mini";
     ModelOptions["gpt_4o"] = "gpt-4o";
     ModelOptions["gpt_4o_mini"] = "gpt-4o-mini";
     ModelOptions["llava"] = "llava";

core/dist/utils.d.ts

@@ -1,4 +1,4 @@
-import { LLMParams } from "./types";
+import type { LLMParams } from "./types";
 export declare const validateLLMParams: (params: Partial<LLMParams>) => LLMParams;
 export declare const encodeImageToBase64: (imagePath: string) => Promise<string>;
 export declare const formatMarkdown: (text: string) => string;

core/dist/utils.js

@@ -27,16 +27,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.convertKeysToSnakeCase = exports.convertFileToPdf = exports.convertPdfToImages = exports.getTextFromImage = exports.downloadFile = exports.isValidUrl = exports.isString = exports.formatMarkdown = exports.encodeImageToBase64 = exports.validateLLMParams = void 0;
-const libreoffice_convert_1 = require("libreoffice-convert");
-const pdf2pic_1 = require("pdf2pic");
-const promises_1 = require("stream/promises");
+const promises_1 = require("node:stream/promises");
+const path_1 = __importDefault(require("path"));
 const util_1 = require("util");
-const Tesseract = __importStar(require("tesseract.js"));
 const axios_1 = __importDefault(require("axios"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
+const libreoffice_convert_1 = require("libreoffice-convert");
 const mime_types_1 = __importDefault(require("mime-types"));
-const path_1 = __importDefault(require("path"));
+const pdf2pic_1 = require("pdf2pic");
 const sharp_1 = __importDefault(require("sharp"));
+const Tesseract = __importStar(require("tesseract.js"));
 const convertAsync = (0, util_1.promisify)(libreoffice_convert_1.convert);
 const defaultLLMParams = {
     frequencyPenalty: 0, // OpenAI defaults to 0

core/package.json

@@ -1,36 +1,36 @@
 {
-  "name": "core",
-  "version": "1.0.0",
-  "description": "Core package for document conversion",
-  "main": "dist/index.js",
-  "type": "commonjs",
-  "files": [
-    "dist"
-  ],
-  "scripts": {
-    "build": "tsc"
-  },
-  "dependencies": {
-    "axios": "^1.7.2",
-    "child_process": "^1.0.2",
-    "dotenv": "^16.4.5",
-    "fs-extra": "^11.2.0",
-    "libreoffice-convert": "^1.6.0",
-    "mime-types": "^2.1.35",
-    "openai": "^4.68.4",
-    "os": "^0.1.2",
-    "p-limit": "^3.1.0",
-    "path": "^0.12.7",
-    "pdf2pic": "^3.1.1",
-    "sharp": "^0.33.5",
-    "tesseract.js": "^5.1.1",
-    "util": "^0.12.5",
-    "zod": "^3.23.8"
-  },
-  "devDependencies": {
-    "@types/fs-extra": "^11.0.4",
-    "@types/mime-types": "^2.1.4",
-    "@types/node": "^20.14.11",
-    "typescript": "^5.6.3"
-  }
+	"name": "core",
+	"version": "1.0.0",
+	"description": "Core package for document conversion",
+	"main": "dist/index.js",
+	"type": "commonjs",
+	"files": [
+		"dist"
+	],
+	"scripts": {
+		"build": "tsc"
+	},
+	"dependencies": {
+		"axios": "^1.7.2",
+		"child_process": "^1.0.2",
+		"dotenv": "^16.4.5",
+		"fs-extra": "^11.2.0",
+		"libreoffice-convert": "^1.6.0",
+		"mime-types": "^2.1.35",
+		"openai": "^4.68.4",
+		"os": "^0.1.2",
+		"p-limit": "^3.1.0",
+		"path": "^0.12.7",
+		"pdf2pic": "^3.1.1",
+		"sharp": "^0.33.5",
+		"tesseract.js": "^5.1.1",
+		"util": "^0.12.5",
+		"zod": "^3.23.8"
+	},
+	"devDependencies": {
+		"@types/fs-extra": "^11.0.4",
+		"@types/mime-types": "^2.1.4",
+		"@types/node": "^20.14.11",
+		"typescript": "^5.6.3"
+	}
 }

core/scripts/install-dependencies.js

@@ -1,87 +1,99 @@
-const { exec } = require("child_process");
-const { promisify } = require("util");
-const fs = require("fs");
+const { exec } = require("node:child_process");
+const { promisify } = require("node:util");
+const fs = require("node:fs");
 
 const execPromise = promisify(exec);
 
 const installPackage = async (command, packageName) => {
-  try {
-    const { stdout, stderr } = await execPromise(command);
-    if (stderr) {
-      throw new Error(`Failed to install ${packageName}: ${stderr}`);
-    }
-    return stdout;
-  } catch (error) {
-    throw new Error(`Failed to install ${packageName}: ${error.message}`);
-  }
+	try {
+		const { stdout, stderr } = await execPromise(command);
+		if (stderr) {
+			throw new Error(`Failed to install ${packageName}: ${stderr}`);
+		}
+		return stdout;
+	} catch (error) {
+		throw new Error(`Failed to install ${packageName}: ${error.message}`);
+	}
 };
 
 const isSudoAvailable = async () => {
-  try {
-    // Try running a sudo command (more relevant for Unix-like systems)
-    await execPromise("sudo -n true");
-    return true;
-  } catch {
-    return false;
-  }
+	try {
+		// Try running a sudo command (more relevant for Unix-like systems)
+		await execPromise("sudo -n true");
+		return true;
+	} catch {
+		return false;
+	}
 };
 
 const checkInstalled = async (command, appName) => {
-  try {
-    const { stdout } = await execPromise(command);
-    console.log(`${appName} is installed: ${stdout}`);
-    return true;
-  } catch (error) {
-    console.log(`${appName} is not installed. Error: ${error.message}`);
-    return false;
-  }
+	try {
+		const { stdout } = await execPromise(command);
+		console.log(`${appName} is installed: ${stdout}`);
+		return true;
+	} catch (error) {
+		console.log(`${appName} is not installed. Error: ${error.message}`);
+		return false;
+	}
 };
 
 const checkAndInstall = async () => {
-  try {
-    const sudoAvailable = await isSudoAvailable();
+	try {
+		const sudoAvailable = await isSudoAvailable();
 
-    // Check for Ghostscript
-    const ghostscriptInstalled = await checkInstalled("gswin64c --version", "Ghostscript") ||
-                                  await checkInstalled("gswin32c --version", "Ghostscript");
-    if (!ghostscriptInstalled) {
-      console.log("Please install Ghostscript from https://www.ghostscript.com/download.html");
-    }
+		// Check for Ghostscript
+		const ghostscriptInstalled =
+			(await checkInstalled("gswin64c --version", "Ghostscript")) ||
+			(await checkInstalled("gswin32c --version", "Ghostscript"));
+		if (!ghostscriptInstalled) {
+			console.log(
+				"Please install Ghostscript from https://www.ghostscript.com/download.html",
+			);
+		}
 
-    // Check for GraphicsMagick
-    const graphicsMagickInstalled = await checkInstalled("gm -version", "GraphicsMagick");
-    if (!graphicsMagickInstalled) {
-      if (process.platform === "darwin") {
-        await installPackage("brew install graphicsmagick", "GraphicsMagick");
-      } else if (process.platform === "linux") {
-        const command = sudoAvailable
-          ? "sudo apt-get update && sudo apt-get install -y graphicsmagick"
-          : "apt-get update && apt-get install -y graphicsmagick";
-        await installPackage(command, "GraphicsMagick");
-      } else {
-        console.log("Please install GraphicsMagick from http://www.graphicsmagick.org/download.html");
-      }
-    }
+		// Check for GraphicsMagick
+		const graphicsMagickInstalled = await checkInstalled(
+			"gm -version",
+			"GraphicsMagick",
+		);
+		if (!graphicsMagickInstalled) {
+			if (process.platform === "darwin") {
+				await installPackage("brew install graphicsmagick", "GraphicsMagick");
+			} else if (process.platform === "linux") {
+				const command = sudoAvailable
+					? "sudo apt-get update && sudo apt-get install -y graphicsmagick"
+					: "apt-get update && apt-get install -y graphicsmagick";
+				await installPackage(command, "GraphicsMagick");
+			} else {
+				console.log(
+					"Please install GraphicsMagick from http://www.graphicsmagick.org/download.html",
+				);
+			}
+		}
 
-    // Check for LibreOffice
-    const libreOfficeInstalled = await checkInstalled("soffice --version", "LibreOffice");
-    if (!libreOfficeInstalled) {
-      if (process.platform === "darwin") {
-        await installPackage("brew install --cask libreoffice", "LibreOffice");
-      } else if (process.platform === "linux") {
-        const command = sudoAvailable
-          ? "sudo apt-get update && sudo apt-get install -y libreoffice"
-          : "apt-get update && apt-get install -y libreoffice";
-        await installPackage(command, "LibreOffice");
-      } else {
-        console.log("Please install LibreOffice from https://www.libreoffice.org/download/download");
-      }
-    }
-
-  } catch (err) {
-    console.error(`Error during installation: ${err.message}`);
-    process.exit(1);
-  }
+		// Check for LibreOffice
+		const libreOfficeInstalled = await checkInstalled(
+			"soffice --version",
+			"LibreOffice",
+		);
+		if (!libreOfficeInstalled) {
+			if (process.platform === "darwin") {
+				await installPackage("brew install --cask libreoffice", "LibreOffice");
+			} else if (process.platform === "linux") {
+				const command = sudoAvailable
+					? "sudo apt-get update && sudo apt-get install -y libreoffice"
+					: "apt-get update && apt-get install -y libreoffice";
+				await installPackage(command, "LibreOffice");
+			} else {
+				console.log(
+					"Please install LibreOffice from https://www.libreoffice.org/download/download",
+				);
+			}
+		}
+	} catch (err) {
+		console.error(`Error during installation: ${err.message}`);
+		process.exit(1);
+	}
 };
 
 checkAndInstall();