fix(backend): update data from open data every week

pYassine · pYassine · commit d65581c4132f · 2024-11-28T21:29:54.000+01:00
diff --git a/_scripts/db/dumps/domifa_test.postgres.custom.gz b/_scripts/db/dumps/domifa_test.postgres.custom.gz
diff --git a/packages/backend/src/_migrations/1731349672897-manual-migration.ts b/packages/backend/src/_migrations/1731349672897-manual-migration.ts
@@ -4,7 +4,7 @@ import { domifaConfig } from "../config";
 import { appLogger } from "../util";
 import { normalizeString } from "@domifa/common";
 
-const batchSize = 5000;
+const batchSize = 2000;
 export class ManualMigration1731349672897 implements MigrationInterface {
   public async up(queryRunner: QueryRunner): Promise<void> {
     if (
@@ -51,7 +51,6 @@ export class ManualMigration1731349672897 implements MigrationInterface {
             }
           }
 
-          // Valide la transaction
           await queryRunner.commitTransaction();
 
           processedRecords += usagers.length;
@@ -71,9 +70,6 @@ export class ManualMigration1731349672897 implements MigrationInterface {
           );
           throw error;
         }
-
-        // Petite pause entre les lots
-        await new Promise((resolve) => setTimeout(resolve, 100));
       }
     }
   }
diff --git a/packages/backend/src/app.module.ts b/packages/backend/src/app.module.ts
@@ -18,6 +18,7 @@ import { ContactSupportModule } from "./modules/contact-support/contact-support.
 import { FileManagerService } from "./util/file-manager/file-manager.service";
 import { InteractionsService } from "./modules/interactions/services";
 import { SmsModule } from "./modules/sms/sms.module";
+import { OpenDataPlacesModule } from "./modules/open-data-places/open-data-places.module";
 
 @Module({
   controllers: [HealthController],
@@ -37,6 +38,7 @@ import { SmsModule } from "./modules/sms/sms.module";
     PortailAdminModule,
     AppLogsModule,
     ContactSupportModule,
+    OpenDataPlacesModule,
   ],
   providers: [PostgresHealthIndicator, FileManagerService, InteractionsService],
 })
diff --git a/packages/backend/src/modules/open-data-places/load-data-inclusion.ts b/packages/backend/src/modules/open-data-places/load-data-inclusion.ts
@@ -93,12 +93,9 @@ const getFromDataInclusion = async (structureType: "CCAS" | "CIAS") => {
 
     if (datInclusionData?.length >= 500) {
       appLogger.warn(
-        "Import 'data-inclusion' data N°" +
-          page +
-          " : " +
-          datInclusionData.length * page +
-          "/" +
-          nbResults
+        `Import 'data-inclusion' data N°${page} : ${
+          datInclusionData.length * page
+        }/${nbResults}`
       );
       page++;
       await getFromDataInclusion(structureType);
diff --git a/packages/backend/src/modules/open-data-places/open-data-places.module.ts b/packages/backend/src/modules/open-data-places/open-data-places.module.ts
@@ -0,0 +1,7 @@
+import { Module } from "@nestjs/common";
+import { UpdateOpenDataPlacesService } from "./services/update-open-data-places/update-open-data-places.service";
+
+@Module({
+  providers: [UpdateOpenDataPlacesService],
+})
+export class OpenDataPlacesModule {}
diff --git a/packages/backend/src/modules/open-data-places/services/update-open-data-places/update-open-data-places.service.spec.ts b/packages/backend/src/modules/open-data-places/services/update-open-data-places/update-open-data-places.service.spec.ts
@@ -0,0 +1,20 @@
+import { Test, TestingModule } from "@nestjs/testing";
+import { UpdateOpenDataPlacesService } from "./update-open-data-places.service";
+
+describe("UpdateOpenDataPlacesService", () => {
+  let service: UpdateOpenDataPlacesService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [UpdateOpenDataPlacesService],
+    }).compile();
+
+    service = module.get<UpdateOpenDataPlacesService>(
+      UpdateOpenDataPlacesService
+    );
+  });
+
+  it("should be defined", () => {
+    expect(service).toBeDefined();
+  });
+});
diff --git a/packages/backend/src/modules/open-data-places/services/update-open-data-places/update-open-data-places.service.ts b/packages/backend/src/modules/open-data-places/services/update-open-data-places/update-open-data-places.service.ts
@@ -0,0 +1,26 @@
+import { Injectable, OnModuleInit } from "@nestjs/common";
+import { Cron } from "@nestjs/schedule";
+import { domifaConfig } from "../../../../config";
+import { isCronEnabled } from "../../../../config/services/isCronEnabled.service";
+import { loadDomifaData } from "../../load-domifa";
+import { loadMssData } from "../../load-mss";
+import { loadSoliguideData } from "../../load-soliguide";
+
+@Injectable()
+export class UpdateOpenDataPlacesService implements OnModuleInit {
+  async onModuleInit() {
+    if (domifaConfig().envId === "prod" && isCronEnabled()) {
+      await this.loadOpenDataPlaces();
+    }
+  }
+
+  @Cron("0 06 * * 2", {
+    timeZone: "Europe/Paris",
+    disabled: domifaConfig().envId !== "prod" && !isCronEnabled(),
+  })
+  protected async loadOpenDataPlaces() {
+    await loadDomifaData();
+    await loadSoliguideData();
+    await loadMssData();
+  }
+}
diff --git a/packages/backend/src/run-app.ts b/packages/backend/src/run-app.ts
@@ -3,11 +3,6 @@ import { bootstrapApplication, tearDownApplication } from "./app.bootstrap";
 
 import { appLogger } from "./util";
 
-import { domifaConfig } from "./config";
-import { loadDomifaData } from "./modules/open-data-places/load-domifa";
-import { loadMssData } from "./modules/open-data-places/load-mss";
-import { loadSoliguideData } from "./modules/open-data-places/load-soliguide";
-
 (async () => {
   appLogger.warn(`[${__filename}] Starting app...`);
 
@@ -18,15 +13,6 @@ import { loadSoliguideData } from "./modules/open-data-places/load-soliguide";
       const server = await app.listen(3000);
       server.setTimeout(1000 * 60 * 5);
       appLogger.warn(`[${__filename}] Application listening on port 3000`);
-
-      if (
-        (domifaConfig().envId === "prod" || domifaConfig().envId === "local") &&
-        domifaConfig().cron.enable
-      ) {
-        await loadDomifaData();
-        await loadSoliguideData();
-        await loadMssData();
-      }
     } catch (error) {
       const err = error as Error;
       appLogger.error(`[${__filename}] Error running application`, {
diff --git a/packages/backend/src/stats/services/publicStats.service.ts b/packages/backend/src/stats/services/publicStats.service.ts
@@ -29,19 +29,19 @@ import { In } from "typeorm";
 export class PublicStatsService implements OnModuleInit {
   constructor(private readonly structuresService: StructuresService) {}
 
-  onModuleInit() {
+  async onModuleInit() {
     if (domifaConfig().envId === "local" && isCronEnabled()) {
-      this.updateAllStatsCache();
+      await this.updateAllStatsCache();
     }
   }
 
   @Cron(CronExpression.EVERY_DAY_AT_2AM, {
     timeZone: "Europe/Paris",
-    disabled: isCronEnabled(),
+    disabled: !isCronEnabled(),
   })
   public async updateAllStatsCache(): Promise<void> {
     for (const regionId of Object.keys(REGIONS_LISTE)) {
-      appLogger.info("[CACHE] Update public stats for region " + regionId);
+      appLogger.info(`[CACHE] Update public stats for region ${regionId}`);
       await this.generatePublicStats({ updateCache: true, regionId });
     }
 
@@ -58,7 +58,7 @@ export class PublicStatsService implements OnModuleInit {
     updateCache?: boolean;
     regionId?: string;
   }): Promise<PublicStats> {
-    const key = regionId ? "public-stats-" + regionId : "public-stats";
+    const key = regionId ? `public-stats-${regionId}` : "public-stats";
 
     const value = await publicStatsCacheRepository
       .createQueryBuilder("public_stats_cache")
diff --git a/packages/backend/src/usagers/controllers/usagers-decision.controller.ts b/packages/backend/src/usagers/controllers/usagers-decision.controller.ts
@@ -87,7 +87,7 @@ export class UsagersDecisionController {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     @Param("usagerRef", new ParseIntPipe()) _usagerRef: number
   ): Promise<Usager> {
-    return this.usagersService.renouvellement(usager, user);
+    return await this.usagersService.renouvellement(usager, user);
   }
 
   @UseGuards(UsagerAccessGuard)
diff --git a/packages/backend/src/usagers/services/custom-docs/buildCustomDoc.service.ts b/packages/backend/src/usagers/services/custom-docs/buildCustomDoc.service.ts
@@ -46,8 +46,9 @@ export function buildCustomDoc({
   let adresseStructure = ucFirst(structure.adresse);
 
   if (!isNil(structure.complementAdresse)) {
-    adresseStructure =
-      adresseStructure + ", " + ucFirst(structure.complementAdresse);
+    adresseStructure = `${adresseStructure}, ${ucFirst(
+      structure.complementAdresse
+    )}`;
   }
 
   // Procu & transfert
diff --git a/packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/renderStructureUsagersExcel.ts b/packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/renderStructureUsagersExcel.ts
@@ -6,6 +6,22 @@ import { StructureUsagerExport } from "./StructureUsagerExport.type";
 import { StructureCustomDocTags } from "../../../_common/model";
 import { isValid, parse } from "date-fns";
 
+const applyDateFormat = (
+  worksheet: StructureCustomDocTags[],
+  elements: Array<keyof StructureCustomDocTags>
+): void => {
+  worksheet.forEach((ws: StructureCustomDocTags) => {
+    elements.forEach((element: keyof StructureCustomDocTags) => {
+      if (ws[element]) {
+        const value = ws[element] as string;
+        ws[element] = isValid(parse(value, "dd/MM/yyyy", new Date()))
+          ? parse(value, "dd/MM/yyyy", new Date())
+          : value;
+      }
+    });
+  });
+};
+
 export const renderStructureUsagersExcel = (
   usagers: StructureUsagerExport[],
   structure: StructureCommon
@@ -52,19 +68,3 @@ export const renderStructureUsagersExcel = (
     type: "buffer",
   });
 };
-
-const applyDateFormat = (
-  worksheet: StructureCustomDocTags[],
-  elements: Array<keyof StructureCustomDocTags>
-): void => {
-  worksheet.forEach((ws: StructureCustomDocTags) => {
-    elements.forEach((element: keyof StructureCustomDocTags) => {
-      if (ws[element]) {
-        const value = ws[element] as string;
-        ws[element] = isValid(parse(value, "dd/MM/yyyy", new Date()))
-          ? parse(value, "dd/MM/yyyy", new Date())
-          : value;
-      }
-    });
-  });
-};
diff --git a/packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/renderStructureUsagersRows.ts b/packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/renderStructureUsagersRows.ts
@@ -104,18 +104,18 @@ export const renderStructureUsagersRows = (
   ];
 
   for (let index = 0; index < 8; index++) {
-    set(usagersListHeader, "AD_NOM_" + index, `Nom ayant-droit ${index + 1}`);
+    set(usagersListHeader, `AD_NOM_${index}`, `Nom ayant-droit ${index + 1}`);
     set(
       usagersListHeader,
-      "AD_PRENOM_" + index,
+      `AD_PRENOM_${index}`,
       `Prénom ayant-droit ${index + 1}`
     );
     set(
       usagersListHeader,
-      "AD_DATE_NAISSANCE_" + index,
+      `AD_DATE_NAISSANCE_${index}`,
       `Date naissance ayant-droit ${index + 1}`
     );
-    set(usagersListHeader, "AD_LIEN_" + index, `Lien de parenté ${index + 1}`);
+    set(usagersListHeader, `AD_LIEN_${index}`, `Lien de parenté ${index + 1}`);
   }
 
   for (const usagerToExport of usagers) {
@@ -143,14 +143,14 @@ export const renderStructureUsagersRows = (
       let index = 0;
 
       usagerToExport.ayantsDroits.forEach((ad: UsagerAyantDroit) => {
-        set(usager, "AD_NOM_" + index, ad.nom);
-        set(usager, "AD_PRENOM_" + index, ad.prenom);
+        set(usager, `AD_NOM_${index}`, ad.nom);
+        set(usager, `AD_PRENOM_${index}`, ad.prenom);
         set(
           usager,
-          "AD_DATE_NAISSANCE_" + index,
+          `AD_DATE_NAISSANCE_${index}`,
           format(new Date(ad.dateNaissance), DATE_FORMAT.JOUR)
         );
-        set(usager, "AD_LIEN_" + index, ad.lien);
+        set(usager, `AD_LIEN_${index}`, ad.lien);
         index++;
       });
 
@@ -221,6 +221,7 @@ export const renderSecondSheetData = (
     ENTRETIEN_ORIENTATION_DETAIL: usager.ENTRETIEN_ORIENTATION_DETAIL,
     ENTRETIEN_DOMICILIATION_EXISTANTE: usager.ENTRETIEN_DOMICILIATION_EXISTANTE,
     ENTRETIEN_REVENUS: usager.ENTRETIEN_REVENUS,
+    ENTRETIEN_REVENUS_DETAIL: usager.ENTRETIEN_REVENUS_DETAIL,
     ENTRETIEN_LIEN_COMMUNE: usager.ENTRETIEN_LIEN_COMMUNE,
     ENTRETIEN_COMPOSITION_MENAGE: usager.ENTRETIEN_COMPOSITION_MENAGE,
     ENTRETIEN_SITUATION_RESIDENTIELLE: usager.ENTRETIEN_SITUATION_RESIDENTIELLE,
diff --git a/packages/frontend/src/app/modules/manage-usagers/components/manage-usagers-table/manage-usagers-table.html b/packages/frontend/src/app/modules/manage-usagers/components/manage-usagers-table/manage-usagers-table.html
@@ -141,7 +141,7 @@
     >
       <button
         type="button"
-        [ngClass]="filters.sortKey === 'ECHEANCE' ? 'fw-bold text-primary' : ''"
+        [ngClass]="filters.sortKey === 'PASSAGE' ? 'fw-bold text-primary' : ''"
         [attr.aria-sort]="filters.sortKey ==='PASSAGE' ? ARIA_SORT[filters.sortValue] : 'none'"
         (click)="updateFilters.emit({element: 'sortKey', value: 'PASSAGE'})"
       >

Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@ import { domifaConfig } from "../config";`
`4`	`4`	`import { appLogger } from "../util";`
`5`	`5`	`import { normalizeString } from "@domifa/common";`
`6`	`6`
`7`		`-const batchSize = 5000;`
	`7`	`+const batchSize = 2000;`
`8`	`8`	`export class ManualMigration1731349672897 implements MigrationInterface {`
`9`	`9`	`public async up(queryRunner: QueryRunner): Promise<void> {`
`10`	`10`	`if (`
`@@ -51,7 +51,6 @@ export class ManualMigration1731349672897 implements MigrationInterface {`
`51`	`51`	`}`
`52`	`52`	`}`
`53`	`53`
`54`		`- // Valide la transaction`
`55`	`54`	`await queryRunner.commitTransaction();`
`56`	`55`
`57`	`56`	`processedRecords += usagers.length;`
`@@ -71,9 +70,6 @@ export class ManualMigration1731349672897 implements MigrationInterface {`
`71`	`70`	`);`
`72`	`71`	`throw error;`
`73`	`72`	`}`
`74`		`-`
`75`		`- // Petite pause entre les lots`
`76`		`- await new Promise((resolve) => setTimeout(resolve, 100));`
`77`	`73`	`}`
`78`	`74`	`}`
`79`	`75`	`}`
Original file line number	Diff line number	Diff line change
`@@ -87,7 +87,7 @@ export class UsagersDecisionController {`
`87`	`87`	`// eslint-disable-next-line @typescript-eslint/no-unused-vars`
`88`	`88`	`@Param("usagerRef", new ParseIntPipe()) _usagerRef: number`
`89`	`89`	`): Promise<Usager> {`
`90`		`- return this.usagersService.renouvellement(usager, user);`
	`90`	`+ return await this.usagersService.renouvellement(usager, user);`
`91`	`91`	`}`
`92`	`92`
`93`	`93`	`@UseGuards(UsagerAccessGuard)`