Merge pull request #19 from DocumindHQ/multi-files

Support for more files and schema field types

Author: Tammilore

core/dist/openAI.js

@@ -24,7 +24,7 @@ const getCompletion = async ({ apiKey, imagePath, llmParams, maintainFormat, mod
         }
     }
     const systemPrompt = `
-    Convert the following document page to markdown.
+    Convert the following image/document  to markdown. 
     Return only the markdown with no explanation text. Do not include deliminators like '''markdown.
     You must include all information on the page. Do not exclude headers, footers, or subtext.
   `;

core/src/openAI.ts

@@ -33,7 +33,7 @@ export const getCompletion = async ({
   }
 
   const systemPrompt = `
-    Convert the following document page to markdown.
+    Convert the following image/document  to markdown. 
     Return only the markdown with no explanation text. Do not include deliminators like '''markdown.
     You must include all information on the page. Do not exclude headers, footers, or subtext.
   `;

extractor/src/converter.js

@@ -13,6 +13,7 @@ export const convertFile = async (filePath, model) => {
     const totalPages = pages.length;
 
     const markdown = await generateMarkdownDocument(pages);
+    console.log('Markdown generated', markdown);
 
     return { markdown, totalPages, fileName };
   } catch (error) {

extractor/src/services/extract.js

@@ -1,5 +1,5 @@
 import { extractData } from '../main-extractor.js'; 
-import { isPdfFile } from '../utils/pdfValidator.js';
+import { isValidFile } from '../utils/fileValidator.js';
 import { validateSchema } from '../utils/schemaValidator.js';
 import { getTemplate } from './templates.js';
 
@@ -19,8 +19,8 @@ export async function extract({ file, schema, template, model, autoSchema }) {
       throw new Error('File is required.');
     }
 
-    if (!isPdfFile(file)) {
-      throw new Error('File must be a PDF.');
+    if (!isValidFile(file)) {
+      throw new Error('File must be a valid format: PDF, PNG, JPG, TXT, DOCX, or HTML.');
     }
 
     let finalSchema = null;

extractor/src/utils/fileValidator.js

@@ -0,0 +1,36 @@
+import axios from 'axios';
+
+/**
+ * Function to check if a file is valid based on its URL or MIME type
+ * @param {string} file - The URL to the file
+ * @returns {Promise<boolean>} - Resolves to true if the file is valid, false otherwise
+ */
+export async function isValidFile(file) {
+    const allowedExtensions = ['pdf', 'png', 'jpg', 'jpeg', 'txt', 'docx', 'html'];
+    const allowedMimeTypes = {
+        pdf: 'application/pdf',
+        png: 'image/png',
+        jpg: 'image/jpeg',
+        jpeg: 'image/jpeg',
+        txt: 'text/plain',
+        docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        html: 'text/html',
+    };
+
+    const urlPath = new URL(file).pathname;
+    const extensionRegex = new RegExp(`\\.(${allowedExtensions.join('|')})$`, 'i');
+
+    if (!extensionRegex.test(urlPath)) {
+        return false;
+    }
+
+    // Optional: Check the MIME type if query parameters are used
+    try {
+        const response = await axios.head(file);
+        const contentType = response.headers['content-type'];
+        return Object.values(allowedMimeTypes).some(mime => contentType.startsWith(mime));
+    } catch (error) {
+        console.error('Error checking MIME type:', error);
+        return false;
+    }
+}

extractor/src/utils/schemaValidator.js

@@ -4,7 +4,7 @@
  * @returns {Object} - { isValid: boolean, errors: Array<string> }
  */
 export function validateSchema(schema) {
-    const validTypes = ["string", "number", "array", "object"];
+    const validTypes = ["string", "number", "array", "object", "boolean", "enum"];
     let errors = [];
 
     if (!Array.isArray(schema)) {
@@ -36,7 +36,17 @@ export function validateSchema(schema) {
           field.children.forEach((child, index) => validateField(child, `${path}.children[${index}]`));
         }
       }
+
+      // Additional checks for enum
+    if (field.type === "enum") {
+      if (!field.hasOwnProperty("values") || !Array.isArray(field.values) || field.values.length === 0) {
+        errors.push(`"values" is required and must be a non-empty array for enum at ${path}`);
+      } else if (!field.values.every((value) => typeof value === "string")) {
+        errors.push(`"values" for enum at ${path} must be an array of strings`);
+      }
     }
+
+  }
 
     schema.forEach((field, index) => validateField(field, `schema[${index}]`));