🚑 poll funfacts worker do not loop over simulations every time

Author: paulsouche

scripts/NGC-1784-20250331.ts

@@ -17,7 +17,7 @@ const main = async () => {
     let updatedPolls = 0
 
     for await (const poll of batchPolls) {
-      await updatePollFunFacts(poll.id, { session: prisma })
+      await updatePollFunFacts({ pollId: poll.id }, { session: prisma })
 
       updatedPolls++
 

src/adapters/redis/constant.ts

@@ -3,6 +3,7 @@ export const CHANNELS = {
 }
 
 export const KEYS = {
+  pollsFunFactsResults: 'pollsFunFactsResults',
   geolocationSortedIps: 'geolocationSortedIps',
   geolocationCountries: 'geolocationCountries',
 }

src/features/organisations/organisations.service.ts

@@ -430,19 +430,26 @@ export const fetchPublicPoll = async ({
 }
 
 export const updatePollFunFacts = async (
-  pollId: string,
+  { pollId, simulationId }: { pollId: string; simulationId?: string },
   { session }: { session?: Session } = {}
 ) => {
   return transaction(async (session) => {
-    const funFacts = await getPollFunFacts({ id: pollId }, { session })
+    const funFacts = await getPollFunFacts(
+      { id: pollId, simulationId },
+      { session }
+    )
 
     await setPollFunFacts(pollId, funFacts, { session })
   }, session)
 }
 
-export const updatePollFunFactsAfterSimulationChange = (
+export const updatePollFunFactsAfterSimulationChange = ({
+  simulationId,
+  created,
+}: {
   simulationId: string
-) => {
+  created: boolean
+}) => {
   return transaction(async (session) => {
     const simulationPoll = await findSimulationPoll(
       { simulationId },
@@ -455,6 +462,9 @@ export const updatePollFunFactsAfterSimulationChange = (
 
     const { pollId } = simulationPoll
 
-    return updatePollFunFacts(pollId, { session })
+    return updatePollFunFacts(
+      { pollId, ...(created ? { simulationId } : {}) },
+      { session }
+    )
   }, prisma)
 }

src/features/simulations/__tests__/create-poll-simulation.spec.ts

@@ -33,6 +33,7 @@ describe('Given a NGC user', () => {
   const { computedResults, nom, situation } = getRandomTestCase()
 
   afterEach(async () => {
+    await EventBus.flush()
     await Promise.all([
       prisma.organisationAdministrator.deleteMany(),
       prisma.simulationPoll.deleteMany(),

src/features/simulations/handlers/compute-poll-fun-facts.ts

@@ -5,7 +5,8 @@ import type { SimulationUpsertedAsyncEvent } from '../events/SimulationUpserted.
 export const computePollFunFacts: Handler<SimulationUpsertedAsyncEvent> = ({
   attributes: {
     simulation: { id },
+    created,
   },
 }) => {
-  return updatePollFunFactsAfterSimulationChange(id)
+  return updatePollFunFactsAfterSimulationChange({ simulationId: id, created })
 }

src/features/simulations/simulations.repository.ts

@@ -15,7 +15,6 @@ import type { UserParams } from '../users/users.validator'
 import type {
   SimulationCreateDto,
   SimulationParticipantCreateDto,
-  UserSimulationParams,
 } from './simulations.validator'
 
 export const createUserSimulation = async (
@@ -151,17 +150,13 @@ export const fetchUserSimulations = (
   })
 }
 
-export const fetchUserSimulation = (
-  {
-    simulationId,
-    // userId,
-  }: UserSimulationParams,
+export const fetchSimulationById = (
+  { simulationId }: { simulationId: string },
   { session }: { session: Session }
 ) => {
   return session.simulation.findUniqueOrThrow({
     where: {
       id: simulationId,
-      // userId,
     },
     select: defaultSimulationSelection,
   })

src/features/simulations/simulations.service.ts

@@ -11,6 +11,8 @@ import type Engine from 'publicodes'
 import { prisma } from '../../adapters/prisma/client'
 import type { Session } from '../../adapters/prisma/transaction'
 import { transaction } from '../../adapters/prisma/transaction'
+import { redis } from '../../adapters/redis/client'
+import { KEYS } from '../../adapters/redis/constant'
 import { EntityNotFoundException } from '../../core/errors/EntityNotFoundException'
 import { ForbiddenException } from '../../core/errors/ForbiddenException'
 import { EventBus } from '../../core/event-bus/event-bus'
@@ -26,7 +28,7 @@ import {
   createPollUserSimulation,
   createUserSimulation,
   fetchPollSimulations,
-  fetchUserSimulation,
+  fetchSimulationById,
   fetchUserSimulations,
 } from './simulations.repository'
 import type {
@@ -48,7 +50,7 @@ const simulationToDto = (
     polls,
     user,
     ...rest
-  }: Partial<Awaited<ReturnType<typeof fetchUserSimulation>>>,
+  }: Partial<Awaited<ReturnType<typeof fetchSimulationById>>>,
   connectedUser: string
 ) => ({
   ...rest,
@@ -105,7 +107,7 @@ export const fetchSimulations = async (params: UserParams) => {
 export const fetchSimulation = async (params: UserSimulationParams) => {
   try {
     const simulation = await transaction(
-      (session) => fetchUserSimulation(params, { session }),
+      (session) => fetchSimulationById(params, { session }),
       prisma
     )
 
@@ -237,7 +239,7 @@ const isValidSimulation = <T>(
   ].every((v) => v <= MAX_VALUE)
 }
 
-const getFunFactValues = async (
+const computeAllFunFactValues = async (
   { id, engine }: { id: string; engine?: Engine },
   { session }: { session: Session }
 ) => {
@@ -271,14 +273,70 @@ const getFunFactValues = async (
     }, funFactValues)
   }
 
-  return {
-    simulationCount,
-    funFactValues,
+  return { simulationCount, funFactValues }
+}
+
+const getFunFactValues = async (
+  {
+    id,
+    simulationId,
+    engine,
+  }: { id: string; simulationId?: string; engine?: Engine },
+  { session }: { session: Session }
+) => {
+  const redisKey = `${KEYS.pollsFunFactsResults}:${id}`
+
+  let result:
+    | {
+        simulationCount: number
+        funFactValues: { [key in DottedName]?: number }
+      }
+    | undefined
+  if (simulationId) {
+    const rawPreviousFunFactValues = await redis.get(redisKey)
+    if (rawPreviousFunFactValues) {
+      const result = JSON.parse(rawPreviousFunFactValues)
+      const simulation = await fetchSimulationById(
+        { simulationId },
+        { session }
+      )
+
+      if (isValidSimulation(simulation)) {
+        const { situation } = simulation
+        Object.values(funFactsRules).reduce((acc, dottedName) => {
+          if (dottedName in frRules) {
+            acc[dottedName] =
+              (acc[dottedName] || 0) +
+              (engine
+                ? getSituationDottedNameValueWithEngine({
+                    dottedName,
+                    situation,
+                    engine,
+                  })
+                : getSituationDottedNameValue({
+                    dottedName,
+                    situation,
+                    rules: frRules,
+                  }))
+          }
+          return acc
+        }, result.funFactValues)
+      }
+    }
+  }
+
+  if (!result) {
+    result = await computeAllFunFactValues({ id, engine }, { session })
   }
+
+  await redis.set(redisKey, JSON.stringify(result))
+  await redis.expire(redisKey, 60 * 60)
+
+  return result
 }
 
 export const getPollFunFacts = async (
-  params: { id: string; engine?: Engine },
+  params: { id: string; simulationId?: string; engine?: Engine },
   session: { session: Session }
 ) => {
   const { funFactValues, simulationCount } = await getFunFactValues(

src/worker.ts

@@ -30,6 +30,7 @@ redis.on('message', async (_, message) => {
     await EventBus.once(event)
     logger.info('Handled event', { event })
   } catch (err) {
+    console.error(err.reasons[0].errors[0].error)
     logger.error('Redis api event failure', err)
   }
 })