👔 change /api/stats to be non-cumulative

src/features/stats/__tests__/fetch-northstar-stats.spec.ts

@@ -4,13 +4,86 @@ import dayjs from 'dayjs'
 import { StatusCodes } from 'http-status-codes'
 import supertest from 'supertest'
 import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'
+import isoWeek from 'dayjs/plugin/isoWeek.js'
+import utc from 'dayjs/plugin/utc.js'
 import { prisma } from '../../../adapters/prisma/client.js'
 import app from '../../../app.js'
 import logger from '../../../logger.js'
 import { PERIODS } from '../stats.constant.js'
-import type { NorthstarStat } from '../stats.repository.js'
 import { upsertStat } from '../stats.repository.js'
 
+dayjs.extend(isoWeek)
+dayjs.extend(utc)
+
+// to be sure week starts on monday
+function isoWeekStartMsUTC(d: dayjs.Dayjs) {
+  const du = d.utc()
+  const dow = du.day()
+  const offset = (dow + 6) % 7
+  return du.startOf('day').subtract(offset, 'day').valueOf()
+}
+
+function startOfPeriodUTC(
+  d: dayjs.Dayjs,
+  unit: 'day' | 'week' | 'month' | 'year'
+) {
+  if (unit === 'week') return isoWeekStartMsUTC(d)
+  return d.utc().startOf(unit).valueOf()
+}
+
+/**
+ * Construit l'attendu "zéro-rempli"
+ * - periodicity: 'day'|'week'|'month'|'year'
+ * - since: nombre de dernières périodes (si null => depuis la première période ayant des données)
+ * - points: Map<timestamp_ms, value> des données réellement insérées (début de période en UTC)
+ */
+function buildExpectedZeroFilled(
+  periodicity: 'day' | 'week' | 'month' | 'year',
+  since: number | null,
+  points: Map<number, number>
+) {
+  const endMs = startOfPeriodUTC(dayjs(), periodicity)
+
+  let startMs: number
+  if (since != null) {
+    // fenêtre calendaire: N dernières périodes, comme en SQL
+    startMs = startOfPeriodUTC(dayjs(), periodicity)
+    // recule (since - 1) périodes
+    startMs = dayjs
+      .utc(startMs)
+      .subtract(since - 1, periodicity)
+      .valueOf()
+  } else {
+    // depuis la première période où l'on a des données
+    const minPoint = [...points.keys()].sort((a, b) => a - b)[0]
+    // si aucune donnée => renvoie []
+    if (minPoint == null) return []
+    startMs = dayjs.utc(minPoint).valueOf()
+  }
+
+  // boucle et remplissage
+  const out: { date: number; value: number }[] = []
+  let cursor = dayjs.utc(startMs)
+  const step = { day: 'day', week: 'week', month: 'month', year: 'year' }[
+    periodicity
+  ] as dayjs.ManipulateType
+
+  // borne de sécurité pour éviter boucles infinies en cas de bug
+  const maxSteps = 20000
+  let steps = 0
+
+  while (cursor.valueOf() <= endMs && steps < maxSteps) {
+    const bucketStart = startOfPeriodUTC(cursor, periodicity)
+    out.push({
+      date: bucketStart,
+      value: points.get(bucketStart) ?? 0,
+    })
+    cursor = cursor.add(1, step)
+    steps++
+  }
+  return out
+}
+
 describe('Given a NGC user', () => {
   const agent = supertest(app)
   const url = '/api/stats'
@@ -65,47 +138,35 @@ describe('Given a redirected NGC user', () => {
     describe.each(
       Object.values(PERIODS).map((periodicity) => ({ periodicity }))
     )('And $periodicity stats', ({ periodicity }) => {
-      let stats: NorthstarStat[]
+      let insertedPoints: Map<number, number> // Map<ms, value>
 
       beforeEach(async () => {
         const fivePeriodsAgo = dayjs()
-          .locale('fr', {
-            weekStart: 1,
-          })
+          .locale('fr', { weekStart: 1 })
           .endOf(periodicity)
           .subtract(4, periodicity)
           .startOf(periodicity)
+
         const lastFivePeriods = new Array(5)
           .fill(null)
           .map((_, i) => fivePeriodsAgo.add(i, periodicity))
 
-        let accumulator = 0
-
-        stats = []
+        insertedPoints = new Map()
 
         for (const period of lastFivePeriods) {
           const finishedSimulations = faker.number.int({
             min: 1000,
             max: 999999,
           })
-
           const firstAnswer =
-            finishedSimulations +
-            faker.number.int({
-              min: 100,
-              max: 9999,
-            })
-
+            finishedSimulations + faker.number.int({ min: 100, max: 9999 })
           const visits =
-            finishedSimulations *
-            faker.number.int({
-              min: 3,
-              max: 15,
-            })
+            finishedSimulations * faker.number.int({ min: 3, max: 15 })
 
-          const stat = await upsertStat(
+          const isoStart = `${period.format('YYYY-MM-DD')}T00:00:00.000Z`
+          await upsertStat(
             {
-              date: new Date(`${period.format('YYYY-MM-DD')}T00:00:00.000Z`),
+              date: new Date(isoStart),
               device: MatomoStatsDevice.all,
               iframe: false,
               source: MatomoStatsSource.beta,
@@ -117,12 +178,8 @@ describe('Given a redirected NGC user', () => {
             { session: prisma }
           )
 
-          accumulator += stat.finishedSimulations
-
-          stats.push({
-            date: period.format('YYYY-MM-DD'),
-            value: accumulator,
-          })
+          const bucketMs = new Date(isoStart).getTime()
+          insertedPoints.set(bucketMs, finishedSimulations) // valeur par période
         }
       })
 
@@ -132,39 +189,59 @@ describe('Given a redirected NGC user', () => {
           .query({ periodicity })
           .expect(StatusCodes.OK)
 
+        const expected = buildExpectedZeroFilled(
+          periodicity,
+          null, // since absent
+          insertedPoints
+        )
+
         expect(body).toEqual({
-          stats,
+          stats: expected,
           description: 'Nombre de simulations réalisées',
         })
       })
 
       describe(`And since query`, () => {
         test(`Then it returns a ${StatusCodes.OK} response with stats since the given periods`, async () => {
-          const since =
-            stats.length - faker.number.int({ min: 1, max: stats.length - 1 })
+          // since strict: on construit la série calendaire N dernières périodes (zéro-remplie)
+          const since = faker.number.int({ min: 1, max: 10 }) // libre: pas besoin de le borner à 1..stats.length maintenant
 
           const { body } = await agent
             .get(url)
             .query({ periodicity, since })
             .expect(StatusCodes.OK)
 
+          const expected = buildExpectedZeroFilled(
+            periodicity,
+            since,
+            insertedPoints
+          )
+
           expect(body).toEqual({
-            stats: stats.slice(stats.length - since),
+            stats: expected,
             description: 'Nombre de simulations réalisées',
           })
         })
 
         test(`Then it returns a ${StatusCodes.OK} response with all stats if since is greater than the number of stats`, async () => {
+          // Ici, on garde l'esprit du test: since très grand => on attend toute la série depuis le min(data) jusqu'à now (zéro-remplie)
           const since =
-            stats.length + faker.number.int({ min: 1, max: stats.length })
+            [...insertedPoints.keys()].length +
+            faker.number.int({ min: 1, max: 5 })
 
           const { body } = await agent
             .get(url)
             .query({ periodicity, since })
             .expect(StatusCodes.OK)
 
+          const expected = buildExpectedZeroFilled(
+            periodicity,
+            since,
+            insertedPoints
+          )
+
           expect(body).toEqual({
-            stats,
+            stats: expected,
             description: 'Nombre de simulations réalisées',
           })
         })

src/features/stats/stats.repository.ts

@@ -80,39 +80,83 @@ export const createNewsLetterStats = (
 }
 
 export type NorthstarStat = {
-  date: string
+  /**
+   * Timestamp (in ms) of the start of the period (UTC)
+   */
+  date: number
   value: number
 }
 
 export const getNorthstarStats = (
-  { periodicity }: { periodicity: PERIODS },
+  { periodicity, since }: { periodicity: PERIODS; since: number | null },
   { session }: { session: Session }
 ) => {
+  const unit = Prisma.raw(`'${periodicity}'`)
+  const oneUnit = Prisma.raw(`INTERVAL '1 ${periodicity}'`)
+
   return session.$queryRaw<NorthstarStat[]>(
     Prisma.sql`
       WITH
-      period_data AS (
-        SELECT TO_CHAR(DATE_TRUNC(${periodicity}, date), 'YYYY-MM-DD') AS start_period, SUM("finishedSimulations") AS value
-        FROM "ngc"."MatomoStats"
-        WHERE "source" = cast(${MatomoStatsSource.beta} as ngc."MatomoStatsSource") AND "kind" = cast(${StatsKind.all} as ngc."StatsKind") AND "referrer" = 'all' AND "device" = cast(${MatomoStatsDevice.all} as ngc."MatomoStatsDevice") AND "iframe" = false
-        GROUP BY start_period
+      filters AS (
+        SELECT
+          cast(${MatomoStatsSource.beta} as ngc."MatomoStatsSource") AS src,
+          cast(${StatsKind.all} as ngc."StatsKind")                  AS knd,
+          cast(${MatomoStatsDevice.all} as ngc."MatomoStatsDevice")  AS dev
+      ),
+      min_trunc AS (
+        SELECT date_trunc(
+                ${unit},
+                (MIN(m.date)::timestamp AT TIME ZONE 'UTC')
+              ) AS first_period
+        FROM "ngc"."MatomoStats" m, filters f
+        WHERE m."source" = f.src
+          AND m."kind"   = f.knd
+          AND m."referrer" = 'all'
+          AND m."device" = f.dev
+          AND m."iframe" = false
+      ),
+      bounds AS (
+        SELECT
+          CASE
+            WHEN ${since}::int IS NOT NULL
+              THEN date_trunc(${unit}, now()) - ((${since}::int - 1) * ${oneUnit})
+            ELSE (SELECT first_period FROM min_trunc)
+          END AS start_at,
+          CASE
+            WHEN ${since}::int IS NOT NULL
+              THEN date_trunc(${unit}, now())
+            ELSE CASE WHEN (SELECT first_period FROM min_trunc) IS NOT NULL
+              THEN date_trunc(${unit}, now())
+              ELSE NULL
+            END
+          END AS end_at
       ),
       period_intervals AS (
-        SELECT TO_CHAR(DATE_TRUNC(${periodicity}, interval_start), 'YYYY-MM-DD') as start_period
-        FROM generate_series(DATE_TRUNC(${periodicity},(
-          SELECT MIN(date)
-          FROM "ngc"."MatomoStats"
-          WHERE "source" = cast(${MatomoStatsSource.beta} as ngc."MatomoStatsSource") AND "kind" = cast(${StatsKind.all} as ngc."StatsKind") AND "referrer" = 'all' AND "device" = cast(${MatomoStatsDevice.all} as ngc."MatomoStatsDevice") AND "iframe" = false
-        )), now(), ('1 ' || ${periodicity})::interval) AS interval_start
+        SELECT gs AS start_period_utc
+        FROM bounds b
+        CROSS JOIN LATERAL generate_series(b.start_at, b.end_at, ${oneUnit}) AS gs
+        WHERE b.start_at IS NOT NULL AND b.end_at IS NOT NULL
+      ),
+      period_data AS (
+        SELECT
+          date_trunc(${unit}, (m.date::timestamp AT TIME ZONE 'UTC')) AS start_period_utc,
+          SUM(m."finishedSimulations")::int AS value
+        FROM "ngc"."MatomoStats" m, filters f, bounds b
+        WHERE m."source" = f.src
+          AND m."kind"   = f.knd
+          AND m."referrer" = 'all'
+          AND m."device" = f.dev
+          AND m."iframe" = false
+          AND date_trunc(${unit}, (m.date::timestamp AT TIME ZONE 'UTC')) >= b.start_at
+          AND date_trunc(${unit}, (m.date::timestamp AT TIME ZONE 'UTC')) <= b.end_at
+        GROUP BY 1
       )
       SELECT
-        period_intervals.start_period as "date",
-        (SUM(coalesce(period_data.value, 0)) OVER (ORDER BY period_intervals.start_period))::integer AS "value"
-      FROM
-        period_data
-      RIGHT OUTER JOIN period_intervals on period_intervals.start_period = period_data.start_period
-      ORDER BY
-        period_intervals.start_period;
+        (EXTRACT(EPOCH FROM pi.start_period_utc) * 1000)::double precision AS "date",
+        COALESCE(pd.value, 0) AS "value"
+      FROM period_intervals pi
+      LEFT JOIN period_data pd USING (start_period_utc)
+      ORDER BY pi.start_period_utc;
     `
   )
 }

src/features/stats/stats.service.ts

@@ -266,6 +266,6 @@ export const fetchNorthstarStats = async (query: NorthstarStatsFetchQuery) => {
 
   return {
     description: 'Nombre de simulations réalisées',
-    stats: stats.slice(Math.max(stats.length - query.since, 0)),
+    stats,
   }
 }

src/features/stats/stats.validator.ts

@@ -4,7 +4,7 @@ import { PERIODS } from './stats.constant.js'
 export const NorthstarStatsFetchQuery = z
   .object({
     periodicity: z.nativeEnum(PERIODS).default(PERIODS.month),
-    since: z.coerce.number().int().positive().default(Number.MAX_VALUE),
+    since: z.coerce.number().int().positive().nullable().default(null),
   })
   .strict()