fix(backend): enforce encrypt script

packages/backend/package.json

@@ -88,6 +88,7 @@
     "node-pdftk": "^2.1.3",
     "nodemailer": "^6.10.0",
     "nunjucks": "^3.2.4",
+    "p-limit": "^6.2.0",
     "passport": "^0.7.0",
     "passport-jwt": "^4.0.1",
     "pg": "^8.13.3",

packages/backend/src/_migrations/1748264444999-encrypt-structure-docs-migration.ts → packages/backend/src/_migrations/1748264445999-encrypt-structure-docs-migration.ts

@@ -5,24 +5,28 @@ import { join } from "node:path";
 import { FileManagerService } from "../util/file-manager/file-manager.service";
 import { CommonDoc, StructureDoc } from "@domifa/common";
 import { createHash } from "node:crypto";
+import { Readable } from "node:stream";
 
 type Docs = StructureDoc & {
   structureUuid: string;
 };
-export class EncryptStructureDocsMigration1748264444999
+export class EncryptStructureDocsMigration1748264445999
   implements MigrationInterface
 {
   public fileManagerService: FileManagerService;
+  public processedCount = 0;
+  public notFoundCount = 0;
+  public errorCount = 0;
+  public errors: Array<{ filePath: string; error: string }> = [];
+  public processedSinceLastPause = 0;
 
   constructor() {
     this.fileManagerService = new FileManagerService();
   }
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   public async up(_queryRunner: QueryRunner): Promise<void> {
     appLogger.warn("[MIGRATION] Encrypt structure docs");
-
     await structureDocRepository.update({}, { encryptionContext: null });
-
     const unencryptedDocs: Docs[] = await structureDocRepository
       .createQueryBuilder("structure_doc")
       .leftJoin(
@@ -44,28 +48,71 @@ export class EncryptStructureDocsMigration1748264444999
     appLogger.warn(
       `[MIGRATION] ${unencryptedDocs.length} documents to encrypt`
     );
+    const pLimit = (await import("p-limit")).default;
 
-    let processedCount = 0;
-    let notFoundCount = 0;
-    let errorCount = 0;
+    const limit = pLimit(3);
 
-    const errors: Array<{ filePath: string; error: string }> = [];
+    const promises = unencryptedDocs.map((doc) =>
+      limit(() => this.processWithPause(doc))
+    );
 
-    for (const doc of unencryptedDocs) {
-      const filePath = join(
-        "structure-documents",
-        cleanPath(`${doc.structureId}`),
-        doc.path
-      );
+    await Promise.all(promises);
 
-      try {
-        const fileExists = await this.fileManagerService.fileExists(filePath);
-        if (!fileExists) {
-          appLogger.error(`🔴 File not found: ${filePath}`);
-          notFoundCount++;
-          continue;
-        }
+    const totalDocs = unencryptedDocs.length;
+    const successRate =
+      totalDocs > 0
+        ? ((this.processedCount / totalDocs) * 100).toFixed(1)
+        : "0";
+
+    const migrationSummary = {
+      "Total documents": totalDocs,
+      "✅ Success": this.processedCount,
+      "🔴 Files not found": this.notFoundCount,
+      "🟠 Processing errors": this.errorCount,
+      "📊 Success rate": `${successRate}%`,
+    };
+
+    appLogger.warn("MIGRATION SUMMARY");
+    console.table(migrationSummary);
+
+    if (this.errors.length > 0) {
+      appLogger.error("Error details:");
+      console.table(this.errors);
+    }
+
+    if (this.processedCount === totalDocs) {
+      appLogger.info("🎉 Migration completed successfully!");
+    } else {
+      appLogger.warn("⚠️ Migration completed with errors");
+    }
+    throw new Error("pokpo");
+  }
+
+  public processWithPause = async (doc: Docs) => {
+    await this.processDoc(doc);
+    this.processedSinceLastPause++;
+
+    if (this.processedSinceLastPause >= 6) {
+      this.processedSinceLastPause = 0;
 
+      appLogger.info("⏸️ Wait one second ");
+      await new Promise((resolve) => setTimeout(resolve, 1500));
+    }
+  };
+
+  public async processDoc(doc: Docs) {
+    const filePath = join(
+      "structure-documents",
+      cleanPath(`${doc.structureId}`),
+      doc.path
+    );
+
+    try {
+      const fileExists = await this.fileManagerService.fileExists(filePath);
+      if (!fileExists) {
+        appLogger.error(`🔴 File not found: ${filePath}`);
+        await this.incrementCounter("notFoundCount");
+      } else {
         appLogger.info(`⌛ Processing : ${filePath}`);
 
         const encryptionContext = crypto.randomUUID();
@@ -75,22 +122,28 @@ export class EncryptStructureDocsMigration1748264444999
           `${doc.path}.sfe`
         );
 
-        const object = await this.fileManagerService.getFileBody(filePath);
+        let object = await this.fileManagerService.getFileBody(filePath);
         await this.fileManagerService.saveEncryptedFile(
           newFilePath,
           { ...doc, encryptionContext },
           object
         );
 
+        object = null;
+
         const isIntact = await this.compareFileIntegrity(
           filePath,
           newFilePath,
-          { ...doc, encryptionContext }
+          {
+            ...doc,
+            encryptionContext,
+          }
         );
 
         if (!isIntact) {
-          errorCount++;
+          await this.incrementCounter("errorCount");
         }
+
         await structureDocRepository.update(
           { uuid: doc.uuid },
           {
@@ -100,55 +153,29 @@ export class EncryptStructureDocsMigration1748264444999
         );
 
         appLogger.info(`✅ encypt done : ${doc.structureUuid} ${filePath}`);
-        processedCount++;
-      } catch (error) {
-        appLogger.error(
-          `🟠 Error during encryption: ${doc.structureUuid}   ${filePath} - ${error.message}`
-        );
-        errors.push({ filePath, error: error.message });
-        errorCount++;
+        await this.incrementCounter("processedCount");
       }
-    }
-
-    const totalDocs = unencryptedDocs.length;
-    const successRate =
-      totalDocs > 0 ? ((processedCount / totalDocs) * 100).toFixed(1) : "0";
-
-    const migrationSummary = {
-      "Total documents": totalDocs,
-      "✅ Success": processedCount,
-      "🔴 Files not found": notFoundCount,
-      "🟠 Processing errors": errorCount,
-      "📊 Success rate": `${successRate}%`,
-    };
-
-    appLogger.warn("MIGRATION SUMMARY");
-    console.table(migrationSummary);
-
-    if (errors.length > 0) {
-      appLogger.error("Error details:");
-      console.table(errors);
-    }
+    } catch (error) {
+      appLogger.error(
+        `🟠 Error during encryption: ${doc.structureUuid}   ${filePath} - ${error.message}`
+      );
+      this.errors.push({ filePath, error: error.message });
 
-    if (processedCount === totalDocs) {
-      appLogger.info("🎉 Migration completed successfully!");
-    } else {
-      appLogger.warn("⚠️ Migration completed with errors");
+      await this.incrementCounter("errorCount");
     }
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   public async down(_queryRunner: QueryRunner): Promise<void> {}
 
-  public calculateHash(data: string | Buffer): string {
+  public async calculateStreamHash(stream: Readable): Promise<string> {
     const hash = createHash("sha256");
-    if (typeof data === "string") {
-      hash.update(data, "binary");
-    } else {
-      hash.update(data);
+    for await (const chunk of stream) {
+      hash.update(chunk);
     }
     return hash.digest("hex");
   }
+
   private async compareFileIntegrity(
     originalPath: string,
     encryptedPath: string,
@@ -158,35 +185,22 @@ export class EncryptStructureDocsMigration1748264444999
       const originalStream = await this.fileManagerService.getFileBody(
         originalPath
       );
-      const originalChunks: Buffer[] = [];
-      for await (const chunk of originalStream) {
-        originalChunks.push(
-          Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
-        );
-      }
-      const originalBuffer = Buffer.concat(originalChunks);
-      const originalHash = this.calculateHash(originalBuffer);
-
-      const decryptedContent =
-        await this.fileManagerService.getDecryptedFileContent(
+      const decryptedStream =
+        await this.fileManagerService.getDecryptedFileStream(
           encryptedPath,
           doc
         );
-      const decryptedHash = this.calculateHash(decryptedContent);
+
+      const [originalHash, decryptedHash] = await Promise.all([
+        this.calculateStreamHash(originalStream),
+        this.calculateStreamHash(decryptedStream),
+      ]);
 
       const isIntact = originalHash === decryptedHash;
 
       appLogger.info(`🔍 ${originalPath}`);
-      appLogger.info(
-        `   Original:  ${originalHash.substring(0, 16)}... (${
-          originalBuffer.length
-        } bytes)`
-      );
-      appLogger.info(
-        `   Decrypted: ${decryptedHash.substring(0, 16)}... (${
-          decryptedContent.length
-        } chars)`
-      );
+      appLogger.info(`   Original:  ${originalHash.substring(0, 16)}...`);
+      appLogger.info(`   Decrypted: ${decryptedHash.substring(0, 16)}...`);
       appLogger.info(`   Status: ${isIntact ? "✅ OK" : "❌ CORRUPTED"}`);
 
       if (!isIntact) {
@@ -201,4 +215,9 @@ export class EncryptStructureDocsMigration1748264444999
       return false;
     }
   }
+  private async incrementCounter(
+    counter: "processedCount" | "errorCount" | "notFoundCount"
+  ): Promise<void> {
+    this[counter] = this[counter] + 1;
+  }
 }

packages/backend/src/util/file-manager/file-manager.service.ts

@@ -245,4 +245,14 @@ export class FileManagerService {
       throw error;
     }
   }
+
+  public async getDecryptedFileStream(
+    filePath: string,
+    doc: Pick<CommonDoc, "encryptionContext">
+  ): Promise<Readable> {
+    const encryptedStream = await this.getFileBody(filePath);
+    const mainSecret = domifaConfig().security.mainSecret;
+
+    return encryptedStream.pipe(decryptFile(mainSecret, doc.encryptionContext));
+  }
 }

yarn.lock

@@ -3837,6 +3837,7 @@ __metadata:
     nodemailer: "npm:^6.10.0"
     nodemon: "npm:^3.1.9"
     nunjucks: "npm:^3.2.4"
+    p-limit: "npm:^6.2.0"
     passport: "npm:^0.7.0"
     passport-jwt: "npm:^4.0.1"
     pg: "npm:^8.13.3"
@@ -22148,6 +22149,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"p-limit@npm:^6.2.0":
+  version: 6.2.0
+  resolution: "p-limit@npm:6.2.0"
+  dependencies:
+    yocto-queue: "npm:^1.1.1"
+  checksum: 10/70e8df3e5f1c173c9bd9fa8390a3c5c2797505240ae42973536992b1f5f59a922153c2f35ff1bf36fb72a0f025b0f13fca062a4233e830adad446960d56b4d84
+  languageName: node
+  linkType: hard
+
 "p-locate@npm:^2.0.0":
   version: 2.0.0
   resolution: "p-locate@npm:2.0.0"
@@ -28245,6 +28255,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"yocto-queue@npm:^1.1.1":
+  version: 1.2.1
+  resolution: "yocto-queue@npm:1.2.1"
+  checksum: 10/0843d6c2c0558e5c06e98edf9c17942f25c769e21b519303a5c2adefd5b738c9b2054204dc856ac0cd9d134b1bc27d928ce84fd23c9e2423b7e013d5a6f50577
+  languageName: node
+  linkType: hard
+
 "yoctocolors-cjs@npm:^2.1.2":
   version: 2.1.2
   resolution: "yoctocolors-cjs@npm:2.1.2"