fix(backend): analyze filesize documents

pYassine · pYassine · commit 507667520fde · 2025-09-16T16:50:54.000+02:00
diff --git a/.talismanrc b/.talismanrc
@@ -8,9 +8,9 @@ fileignoreconfig:
 - filename: _docs/dev/0.run-local-env.md
   checksum: 855b6f29ace746c86597d36382dbe7e40f771137734e3bf5eb53c948d6592a74
 - filename: _scripts/db/dumps/domifa_test.postgres.restore-data-only.sql
-  checksum: f106304727d41ce64bb581f8772896ed8664df3a5e41489587be64afab48f453
+  checksum: 77fe0574f840079590823778e65a418f3ec05925d1e7ec4c1c9925446b7bccbd
 - filename: _scripts/db/dumps/domifa_test.postgres.truncate-restore-data-only.sql
-  checksum: 4d2f01ba7fbd8c1109994a0831576e6341dc7e925fb75b9f447e2b3e1c9ef1c6
+  checksum: 4319e1dd76aa224c9d3f05c8834726b98eda791fe972458a5c46743b434f51eb
 - filename: packages/backend/package.json
   checksum: 2f5813dbc35cdc3659516beabf9f93e2978eb62d1b5403cfbb2907f22bb25355
 - filename: packages/backend/src/_migrations/1757532174131-auto-migration.ts
diff --git a/_scripts/db/dumps/domifa_test.postgres.custom.gz b/_scripts/db/dumps/domifa_test.postgres.custom.gz
diff --git a/_scripts/db/dumps/domifa_test.postgres.restore-data-only.sql b/_scripts/db/dumps/domifa_test.postgres.restore-data-only.sql
@@ -209,10 +209,10 @@ COPY public.typeorm_metadata (type, schema, name, value) FROM stdin;
 -- Data for Name: usager_docs; Type: TABLE DATA; Schema: public; Owner: -
 --
 
-COPY public.usager_docs (uuid, "createdAt", "updatedAt", version, "usagerUUID", "structureId", "usagerRef", path, label, filetype, "createdBy", "encryptionContext", "encryptionVersion", shared) FROM stdin;
-2eb5e74d-4b25-4aa6-ad2a-7d963ae66072	2019-10-07 20:51:31.578+02	2022-06-28 22:52:59.282479+02	1	860ffa4c-88c4-4e1c-ad42-5a05cdf39830	1	1	373144a3d9d0b3f4c84bd527a5cff880.jpg	CNI	image/jpeg	Patrick Roméro	ffe17c48-7a1a-42c9-8494-0b72ca8b3686	0	f
-a77729a9-1b28-4090-bda8-760590bff982	2019-10-07 20:53:32.922+02	2022-06-28 22:52:59.28757+02	1	4dcdcddc-fad2-4827-aac5-0acf1df7b5bc	1	5	8242ba1bc7f3c3971f761b6a347fc1c4.jpg	Carte identité	image/jpeg	Patrick Roméro	ffe17c48-7a1a-42c9-8494-0b72ca8b3686	0	f
-542a0da1-ea1c-48ab-8026-67a4248b1c47	2024-10-30 22:35:12.346+01	2024-10-30 22:35:13.112661+01	1	b2c26e55-ab37-457d-b307-6fe161050a9b	1	7	ddda9bf211821bec99b90230bd0f52dc.jpg	Document à éditer par la suite	image/jpeg	Patrick Roméro	f36cb085-f3a5-4333-afbf-ba6c1c28c5d8	0	f
+COPY public.usager_docs (uuid, "createdAt", "updatedAt", version, "usagerUUID", "structureId", "usagerRef", path, label, filetype, "createdBy", "encryptionContext", "encryptionVersion", shared, filesize) FROM stdin;
+2eb5e74d-4b25-4aa6-ad2a-7d963ae66072	2019-10-07 20:51:31.578+02	2025-09-16 16:47:26.946388+02	2	860ffa4c-88c4-4e1c-ad42-5a05cdf39830	1	1	373144a3d9d0b3f4c84bd527a5cff880.jpg	CNI	image/jpeg	Patrick Roméro	ffe17c48-7a1a-42c9-8494-0b72ca8b3686	0	f	0
+a77729a9-1b28-4090-bda8-760590bff982	2019-10-07 20:53:32.922+02	2025-09-16 16:47:27.115593+02	2	4dcdcddc-fad2-4827-aac5-0acf1df7b5bc	1	5	8242ba1bc7f3c3971f761b6a347fc1c4.jpg	Carte identité	image/jpeg	Patrick Roméro	ffe17c48-7a1a-42c9-8494-0b72ca8b3686	0	f	0
+542a0da1-ea1c-48ab-8026-67a4248b1c47	2024-10-30 22:35:12.346+01	2025-09-16 16:47:27.271753+02	2	b2c26e55-ab37-457d-b307-6fe161050a9b	1	7	ddda9bf211821bec99b90230bd0f52dc.jpg	Document à éditer par la suite	image/jpeg	Patrick Roméro	f36cb085-f3a5-4333-afbf-ba6c1c28c5d8	0	f	0
 \.
 
 
diff --git a/_scripts/db/dumps/domifa_test.postgres.truncate-restore-data-only.sql b/_scripts/db/dumps/domifa_test.postgres.truncate-restore-data-only.sql
@@ -245,10 +245,10 @@ COPY public.typeorm_metadata (type, schema, name, value) FROM stdin;
 -- Data for Name: usager_docs; Type: TABLE DATA; Schema: public; Owner: -
 --
 
-COPY public.usager_docs (uuid, "createdAt", "updatedAt", version, "usagerUUID", "structureId", "usagerRef", path, label, filetype, "createdBy", "encryptionContext", "encryptionVersion", shared) FROM stdin;
-2eb5e74d-4b25-4aa6-ad2a-7d963ae66072	2019-10-07 20:51:31.578+02	2022-06-28 22:52:59.282479+02	1	860ffa4c-88c4-4e1c-ad42-5a05cdf39830	1	1	373144a3d9d0b3f4c84bd527a5cff880.jpg	CNI	image/jpeg	Patrick Roméro	ffe17c48-7a1a-42c9-8494-0b72ca8b3686	0	f
-a77729a9-1b28-4090-bda8-760590bff982	2019-10-07 20:53:32.922+02	2022-06-28 22:52:59.28757+02	1	4dcdcddc-fad2-4827-aac5-0acf1df7b5bc	1	5	8242ba1bc7f3c3971f761b6a347fc1c4.jpg	Carte identité	image/jpeg	Patrick Roméro	ffe17c48-7a1a-42c9-8494-0b72ca8b3686	0	f
-542a0da1-ea1c-48ab-8026-67a4248b1c47	2024-10-30 22:35:12.346+01	2024-10-30 22:35:13.112661+01	1	b2c26e55-ab37-457d-b307-6fe161050a9b	1	7	ddda9bf211821bec99b90230bd0f52dc.jpg	Document à éditer par la suite	image/jpeg	Patrick Roméro	f36cb085-f3a5-4333-afbf-ba6c1c28c5d8	0	f
+COPY public.usager_docs (uuid, "createdAt", "updatedAt", version, "usagerUUID", "structureId", "usagerRef", path, label, filetype, "createdBy", "encryptionContext", "encryptionVersion", shared, filesize) FROM stdin;
+2eb5e74d-4b25-4aa6-ad2a-7d963ae66072	2019-10-07 20:51:31.578+02	2025-09-16 16:47:26.946388+02	2	860ffa4c-88c4-4e1c-ad42-5a05cdf39830	1	1	373144a3d9d0b3f4c84bd527a5cff880.jpg	CNI	image/jpeg	Patrick Roméro	ffe17c48-7a1a-42c9-8494-0b72ca8b3686	0	f	0
+a77729a9-1b28-4090-bda8-760590bff982	2019-10-07 20:53:32.922+02	2025-09-16 16:47:27.115593+02	2	4dcdcddc-fad2-4827-aac5-0acf1df7b5bc	1	5	8242ba1bc7f3c3971f761b6a347fc1c4.jpg	Carte identité	image/jpeg	Patrick Roméro	ffe17c48-7a1a-42c9-8494-0b72ca8b3686	0	f	0
+542a0da1-ea1c-48ab-8026-67a4248b1c47	2024-10-30 22:35:12.346+01	2025-09-16 16:47:27.271753+02	2	b2c26e55-ab37-457d-b307-6fe161050a9b	1	7	ddda9bf211821bec99b90230bd0f52dc.jpg	Document à éditer par la suite	image/jpeg	Patrick Roméro	f36cb085-f3a5-4333-afbf-ba6c1c28c5d8	0	f	0
 \.
 
 
diff --git a/packages/backend/src/_migrations/1758029089533-auto-migration.ts b/packages/backend/src/_migrations/1758029089533-auto-migration.ts
@@ -0,0 +1,30 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+import { domifaConfig } from "../config";
+
+export class AutoMigration1758029089533 implements MigrationInterface {
+  name = "AutoMigration1758029089533";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    if (
+      domifaConfig().envId === "prod" ||
+      domifaConfig().envId === "preprod" ||
+      domifaConfig().envId === "local"
+    ) {
+      await queryRunner.query(
+        `ALTER TABLE "usager_docs" ADD "filesize" integer`
+      );
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    if (
+      domifaConfig().envId === "prod" ||
+      domifaConfig().envId === "preprod" ||
+      domifaConfig().envId === "local"
+    ) {
+      await queryRunner.query(
+        `ALTER TABLE "usager_docs" DROP COLUMN "filesize"`
+      );
+    }
+  }
+}
diff --git a/packages/backend/src/_migrations/1758029089535-auto-migration.ts b/packages/backend/src/_migrations/1758029089535-auto-migration.ts
@@ -0,0 +1,123 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+import { domifaConfig } from "../config";
+import { usagerDocsRepository } from "../database";
+import { appLogger, cleanPath, FileManagerService } from "../util";
+import { join } from "node:path";
+
+export class AutoMigration1758029089535 implements MigrationInterface {
+  name = "AutoMigration1758029089535";
+
+  public async up(): Promise<void> {
+    if (
+      domifaConfig().envId === "prod" ||
+      domifaConfig().envId === "preprod" ||
+      domifaConfig().envId === "local"
+    ) {
+      await this.processExistingDocuments();
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "usager_docs" DROP COLUMN "filesize"`);
+  }
+
+  private async processExistingDocuments(): Promise<void> {
+    appLogger.warn(
+      "[Migration] Starting filesize processing for existing documents..."
+    );
+
+    const fileManagerService = new FileManagerService();
+    const batchSize = 500;
+    let offset = 0;
+    let totalProcessed = 0;
+    let totalUpdated = 0;
+    let totalNotFound = 0;
+
+    while (true) {
+      // Récupérer un batch de documents sans fileSize avec jointure pour avoir structure.uuid
+      const documents = await usagerDocsRepository
+        .createQueryBuilder("doc")
+        .leftJoin("structure", "structure", "structure.id = doc.structureId")
+        .leftJoin("usager", "usager", "usager.uuid = doc.usagerUUID")
+        .select([
+          "doc.uuid as uuid",
+          "doc.path as path",
+          "doc.label as label",
+          "structure.uuid as structure_uuid",
+          "usager.uuid as usager_uuid",
+        ])
+        .where("doc.filesize IS NULL")
+        .orderBy("doc.createdAt", "ASC")
+        .limit(batchSize)
+        .offset(offset)
+        .getRawMany();
+
+      if (documents.length === 0) {
+        break;
+      }
+
+      appLogger.warn(
+        `[Migration] Processing batch ${
+          Math.floor(offset / batchSize) + 1
+        }, documents ${offset + 1} to ${offset + documents.length}`
+      );
+
+      // Traiter chaque document du batch
+      for (const doc of documents) {
+        try {
+          // Reconstituer le chemin complet du fichier comme dans votre logique existante
+          const filePath = join(
+            "usager-documents",
+            cleanPath(doc.structure_uuid),
+            cleanPath(doc.usager_uuid),
+            `${doc.path}.sfe`
+          );
+
+          const filesize = await fileManagerService.getFileSize(filePath);
+          await usagerDocsRepository.update({ uuid: doc.uuid }, { filesize });
+
+          if (filesize > 0) {
+            totalUpdated++;
+            appLogger.debug(
+              `[Migration] Updated document ${doc.uuid} (${doc.label}) with size ${filesize} bytes, path: ${filePath}`
+            );
+          } else if (filesize === -1) {
+            totalNotFound++;
+            appLogger.warn(
+              `[Migration] File not found for document ${doc.uuid} (${doc.label}), path: ${filePath}`
+            );
+          } else {
+            console.log(`🟥 ${filePath} - ${filesize} `);
+          }
+        } catch (error) {
+          appLogger.error(
+            `[Migration] Error processing document ${doc.uuid}: ${error.message}`
+          );
+        }
+      }
+
+      totalProcessed += documents.length;
+      offset += batchSize;
+
+      // Log de progression tous les 10 batches
+      if (Math.floor(offset / batchSize) % 10 === 0) {
+        appLogger.warn(
+          `[Migration] Progress: ${totalProcessed} documents processed, ${totalUpdated} updated, ${totalNotFound} not found`
+        );
+      }
+
+      // Petite pause pour ne pas surcharger S3
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+
+    appLogger.warn(`[Migration] FileSize processing completed:`);
+    appLogger.warn(
+      `[Migration] - Total documents processed: ${totalProcessed}`
+    );
+    appLogger.warn(`[Migration] - Successfully updated: ${totalUpdated}`);
+    appLogger.warn(`[Migration] - Files not found in S3: ${totalNotFound}`);
+    appLogger.warn(
+      `[Migration] - Errors: ${totalProcessed - totalUpdated - totalNotFound}`
+    );
+  }
+}
diff --git a/packages/backend/src/_migrations/_init-db/domifa_test_schema.sql b/packages/backend/src/_migrations/_init-db/domifa_test_schema.sql
@@ -328,7 +328,8 @@ CREATE TABLE public.usager_docs (
     "createdBy" text NOT NULL,
     "encryptionContext" text,
     "encryptionVersion" integer,
-    shared boolean DEFAULT false NOT NULL
+    shared boolean DEFAULT false NOT NULL,
+    filesize integer
 );
 CREATE TABLE public.usager_entretien (
     uuid uuid DEFAULT public.uuid_generate_v4() NOT NULL,
diff --git a/packages/backend/src/database/entities/usager/UsagerDocsTable.typeorm.ts b/packages/backend/src/database/entities/usager/UsagerDocsTable.typeorm.ts
@@ -37,6 +37,9 @@ export class UsagerDocsTable
   @Column({ type: "text", nullable: false })
   public filetype: string;
 
+  @Column({ type: "integer", nullable: true })
+  public filesize: number;
+
   @Column({ type: "text", nullable: false })
   public createdBy: string;
 
diff --git a/packages/backend/src/util/file-manager/file-manager.service.ts b/packages/backend/src/util/file-manager/file-manager.service.ts
@@ -284,4 +284,29 @@ export class FileManagerService {
       `Unsupported input type for toNodeReadable: ${typeof input}`
     );
   }
+
+  public async getFileSize(filePath: string): Promise<number> {
+    try {
+      const headResult = await this.s3.send(
+        new HeadObjectCommand({
+          Bucket: domifaConfig().upload.bucketName,
+          Key: `${domifaConfig().upload.bucketRootDir}/${filePath}`,
+        })
+      );
+
+      return headResult.ContentLength || 0;
+    } catch (error) {
+      if (
+        error.name === "NotFound" ||
+        error.$metadata?.httpStatusCode === 404
+      ) {
+        return -1;
+      } else {
+        appLogger.error(
+          `Error getting file size for ${filePath}: ${error.message}`
+        );
+        return -2;
+      }
+    }
+  }
 }
diff --git a/packages/common/src/_core/interfaces/CommonDoc.interface.ts b/packages/common/src/_core/interfaces/CommonDoc.interface.ts
@@ -8,4 +8,5 @@ export interface CommonDoc extends AppEntity {
   structureId: number;
   encryptionContext?: string;
   encryptionVersion?: number;
+  filesize?: number;
 }

Original file line number	Diff line number	Diff line change
`@@ -8,4 +8,5 @@ export interface CommonDoc extends AppEntity {`
`8`	`8`	`structureId: number;`
`9`	`9`	`encryptionContext?: string;`
`10`	`10`	`encryptionVersion?: number;`
	`11`	`+ filesize?: number;`
`11`	`12`	`}`