From 6eca7d56fbd5964b8daf687e6459e5a21c527a0b Mon Sep 17 00:00:00 2001 From: Lilian Saget-Lethias Date: Wed, 10 Sep 2025 13:32:35 +0200 Subject: [PATCH] :necktie: change /api/stats to be non-cumulative --- .../__tests__/fetch-northstar-stats.spec.ts | 145 ++++++++++++++---- src/features/stats/stats.repository.ts | 84 +++++++--- src/features/stats/stats.service.ts | 2 +- src/features/stats/stats.validator.ts | 2 +- 4 files changed, 177 insertions(+), 56 deletions(-) diff --git a/src/features/stats/__tests__/fetch-northstar-stats.spec.ts b/src/features/stats/__tests__/fetch-northstar-stats.spec.ts index 95b03a89..ca1eea9a 100644 --- a/src/features/stats/__tests__/fetch-northstar-stats.spec.ts +++ b/src/features/stats/__tests__/fetch-northstar-stats.spec.ts @@ -4,13 +4,86 @@ import dayjs from 'dayjs' import { StatusCodes } from 'http-status-codes' import supertest from 'supertest' import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' +import isoWeek from 'dayjs/plugin/isoWeek.js' +import utc from 'dayjs/plugin/utc.js' import { prisma } from '../../../adapters/prisma/client.js' import app from '../../../app.js' import logger from '../../../logger.js' import { PERIODS } from '../stats.constant.js' -import type { NorthstarStat } from '../stats.repository.js' import { upsertStat } from '../stats.repository.js' +dayjs.extend(isoWeek) +dayjs.extend(utc) + +// to be sure week starts on monday +function isoWeekStartMsUTC(d: dayjs.Dayjs) { + const du = d.utc() + const dow = du.day() + const offset = (dow + 6) % 7 + return du.startOf('day').subtract(offset, 'day').valueOf() +} + +function startOfPeriodUTC( + d: dayjs.Dayjs, + unit: 'day' | 'week' | 'month' | 'year' +) { + if (unit === 'week') return isoWeekStartMsUTC(d) + return d.utc().startOf(unit).valueOf() +} + +/** + * Construit l'attendu "zéro-rempli" + * - periodicity: 'day'|'week'|'month'|'year' + * - since: nombre de dernières périodes (si null => depuis la première période ayant des données) + * - points: Map des données réellement insérées (début de période en UTC) + */ +function buildExpectedZeroFilled( + periodicity: 'day' | 'week' | 'month' | 'year', + since: number | null, + points: Map +) { + const endMs = startOfPeriodUTC(dayjs(), periodicity) + + let startMs: number + if (since != null) { + // fenêtre calendaire: N dernières périodes, comme en SQL + startMs = startOfPeriodUTC(dayjs(), periodicity) + // recule (since - 1) périodes + startMs = dayjs + .utc(startMs) + .subtract(since - 1, periodicity) + .valueOf() + } else { + // depuis la première période où l'on a des données + const minPoint = [...points.keys()].sort((a, b) => a - b)[0] + // si aucune donnée => renvoie [] + if (minPoint == null) return [] + startMs = dayjs.utc(minPoint).valueOf() + } + + // boucle et remplissage + const out: { date: number; value: number }[] = [] + let cursor = dayjs.utc(startMs) + const step = { day: 'day', week: 'week', month: 'month', year: 'year' }[ + periodicity + ] as dayjs.ManipulateType + + // borne de sécurité pour éviter boucles infinies en cas de bug + const maxSteps = 20000 + let steps = 0 + + while (cursor.valueOf() <= endMs && steps < maxSteps) { + const bucketStart = startOfPeriodUTC(cursor, periodicity) + out.push({ + date: bucketStart, + value: points.get(bucketStart) ?? 0, + }) + cursor = cursor.add(1, step) + steps++ + } + return out +} + describe('Given a NGC user', () => { const agent = supertest(app) const url = '/api/stats' @@ -65,47 +138,35 @@ describe('Given a redirected NGC user', () => { describe.each( Object.values(PERIODS).map((periodicity) => ({ periodicity })) )('And $periodicity stats', ({ periodicity }) => { - let stats: NorthstarStat[] + let insertedPoints: Map // Map beforeEach(async () => { const fivePeriodsAgo = dayjs() - .locale('fr', { - weekStart: 1, - }) + .locale('fr', { weekStart: 1 }) .endOf(periodicity) .subtract(4, periodicity) .startOf(periodicity) + const lastFivePeriods = new Array(5) .fill(null) .map((_, i) => fivePeriodsAgo.add(i, periodicity)) - let accumulator = 0 - - stats = [] + insertedPoints = new Map() for (const period of lastFivePeriods) { const finishedSimulations = faker.number.int({ min: 1000, max: 999999, }) - const firstAnswer = - finishedSimulations + - faker.number.int({ - min: 100, - max: 9999, - }) - + finishedSimulations + faker.number.int({ min: 100, max: 9999 }) const visits = - finishedSimulations * - faker.number.int({ - min: 3, - max: 15, - }) + finishedSimulations * faker.number.int({ min: 3, max: 15 }) - const stat = await upsertStat( + const isoStart = `${period.format('YYYY-MM-DD')}T00:00:00.000Z` + await upsertStat( { - date: new Date(`${period.format('YYYY-MM-DD')}T00:00:00.000Z`), + date: new Date(isoStart), device: MatomoStatsDevice.all, iframe: false, source: MatomoStatsSource.beta, @@ -117,12 +178,8 @@ describe('Given a redirected NGC user', () => { { session: prisma } ) - accumulator += stat.finishedSimulations - - stats.push({ - date: period.format('YYYY-MM-DD'), - value: accumulator, - }) + const bucketMs = new Date(isoStart).getTime() + insertedPoints.set(bucketMs, finishedSimulations) // valeur par période } }) @@ -132,39 +189,59 @@ describe('Given a redirected NGC user', () => { .query({ periodicity }) .expect(StatusCodes.OK) + const expected = buildExpectedZeroFilled( + periodicity, + null, // since absent + insertedPoints + ) + expect(body).toEqual({ - stats, + stats: expected, description: 'Nombre de simulations réalisées', }) }) describe(`And since query`, () => { test(`Then it returns a ${StatusCodes.OK} response with stats since the given periods`, async () => { - const since = - stats.length - faker.number.int({ min: 1, max: stats.length - 1 }) + // since strict: on construit la série calendaire N dernières périodes (zéro-remplie) + const since = faker.number.int({ min: 1, max: 10 }) // libre: pas besoin de le borner à 1..stats.length maintenant const { body } = await agent .get(url) .query({ periodicity, since }) .expect(StatusCodes.OK) + const expected = buildExpectedZeroFilled( + periodicity, + since, + insertedPoints + ) + expect(body).toEqual({ - stats: stats.slice(stats.length - since), + stats: expected, description: 'Nombre de simulations réalisées', }) }) test(`Then it returns a ${StatusCodes.OK} response with all stats if since is greater than the number of stats`, async () => { + // Ici, on garde l'esprit du test: since très grand => on attend toute la série depuis le min(data) jusqu'à now (zéro-remplie) const since = - stats.length + faker.number.int({ min: 1, max: stats.length }) + [...insertedPoints.keys()].length + + faker.number.int({ min: 1, max: 5 }) const { body } = await agent .get(url) .query({ periodicity, since }) .expect(StatusCodes.OK) + const expected = buildExpectedZeroFilled( + periodicity, + since, + insertedPoints + ) + expect(body).toEqual({ - stats, + stats: expected, description: 'Nombre de simulations réalisées', }) }) diff --git a/src/features/stats/stats.repository.ts b/src/features/stats/stats.repository.ts index fb1131ee..da674455 100644 --- a/src/features/stats/stats.repository.ts +++ b/src/features/stats/stats.repository.ts @@ -80,39 +80,83 @@ export const createNewsLetterStats = ( } export type NorthstarStat = { - date: string + /** + * Timestamp (in ms) of the start of the period (UTC) + */ + date: number value: number } export const getNorthstarStats = ( - { periodicity }: { periodicity: PERIODS }, + { periodicity, since }: { periodicity: PERIODS; since: number | null }, { session }: { session: Session } ) => { + const unit = Prisma.raw(`'${periodicity}'`) + const oneUnit = Prisma.raw(`INTERVAL '1 ${periodicity}'`) + return session.$queryRaw( Prisma.sql` WITH - period_data AS ( - SELECT TO_CHAR(DATE_TRUNC(${periodicity}, date), 'YYYY-MM-DD') AS start_period, SUM("finishedSimulations") AS value - FROM "ngc"."MatomoStats" - WHERE "source" = cast(${MatomoStatsSource.beta} as ngc."MatomoStatsSource") AND "kind" = cast(${StatsKind.all} as ngc."StatsKind") AND "referrer" = 'all' AND "device" = cast(${MatomoStatsDevice.all} as ngc."MatomoStatsDevice") AND "iframe" = false - GROUP BY start_period + filters AS ( + SELECT + cast(${MatomoStatsSource.beta} as ngc."MatomoStatsSource") AS src, + cast(${StatsKind.all} as ngc."StatsKind") AS knd, + cast(${MatomoStatsDevice.all} as ngc."MatomoStatsDevice") AS dev + ), + min_trunc AS ( + SELECT date_trunc( + ${unit}, + (MIN(m.date)::timestamp AT TIME ZONE 'UTC') + ) AS first_period + FROM "ngc"."MatomoStats" m, filters f + WHERE m."source" = f.src + AND m."kind" = f.knd + AND m."referrer" = 'all' + AND m."device" = f.dev + AND m."iframe" = false + ), + bounds AS ( + SELECT + CASE + WHEN ${since}::int IS NOT NULL + THEN date_trunc(${unit}, now()) - ((${since}::int - 1) * ${oneUnit}) + ELSE (SELECT first_period FROM min_trunc) + END AS start_at, + CASE + WHEN ${since}::int IS NOT NULL + THEN date_trunc(${unit}, now()) + ELSE CASE WHEN (SELECT first_period FROM min_trunc) IS NOT NULL + THEN date_trunc(${unit}, now()) + ELSE NULL + END + END AS end_at ), period_intervals AS ( - SELECT TO_CHAR(DATE_TRUNC(${periodicity}, interval_start), 'YYYY-MM-DD') as start_period - FROM generate_series(DATE_TRUNC(${periodicity},( - SELECT MIN(date) - FROM "ngc"."MatomoStats" - WHERE "source" = cast(${MatomoStatsSource.beta} as ngc."MatomoStatsSource") AND "kind" = cast(${StatsKind.all} as ngc."StatsKind") AND "referrer" = 'all' AND "device" = cast(${MatomoStatsDevice.all} as ngc."MatomoStatsDevice") AND "iframe" = false - )), now(), ('1 ' || ${periodicity})::interval) AS interval_start + SELECT gs AS start_period_utc + FROM bounds b + CROSS JOIN LATERAL generate_series(b.start_at, b.end_at, ${oneUnit}) AS gs + WHERE b.start_at IS NOT NULL AND b.end_at IS NOT NULL + ), + period_data AS ( + SELECT + date_trunc(${unit}, (m.date::timestamp AT TIME ZONE 'UTC')) AS start_period_utc, + SUM(m."finishedSimulations")::int AS value + FROM "ngc"."MatomoStats" m, filters f, bounds b + WHERE m."source" = f.src + AND m."kind" = f.knd + AND m."referrer" = 'all' + AND m."device" = f.dev + AND m."iframe" = false + AND date_trunc(${unit}, (m.date::timestamp AT TIME ZONE 'UTC')) >= b.start_at + AND date_trunc(${unit}, (m.date::timestamp AT TIME ZONE 'UTC')) <= b.end_at + GROUP BY 1 ) SELECT - period_intervals.start_period as "date", - (SUM(coalesce(period_data.value, 0)) OVER (ORDER BY period_intervals.start_period))::integer AS "value" - FROM - period_data - RIGHT OUTER JOIN period_intervals on period_intervals.start_period = period_data.start_period - ORDER BY - period_intervals.start_period; + (EXTRACT(EPOCH FROM pi.start_period_utc) * 1000)::double precision AS "date", + COALESCE(pd.value, 0) AS "value" + FROM period_intervals pi + LEFT JOIN period_data pd USING (start_period_utc) + ORDER BY pi.start_period_utc; ` ) } diff --git a/src/features/stats/stats.service.ts b/src/features/stats/stats.service.ts index 7a0aeb2f..5856da03 100644 --- a/src/features/stats/stats.service.ts +++ b/src/features/stats/stats.service.ts @@ -266,6 +266,6 @@ export const fetchNorthstarStats = async (query: NorthstarStatsFetchQuery) => { return { description: 'Nombre de simulations réalisées', - stats: stats.slice(Math.max(stats.length - query.since, 0)), + stats, } } diff --git a/src/features/stats/stats.validator.ts b/src/features/stats/stats.validator.ts index dfa73c97..91eb0910 100644 --- a/src/features/stats/stats.validator.ts +++ b/src/features/stats/stats.validator.ts @@ -4,7 +4,7 @@ import { PERIODS } from './stats.constant.js' export const NorthstarStatsFetchQuery = z .object({ periodicity: z.nativeEnum(PERIODS).default(PERIODS.month), - since: z.coerce.number().int().positive().default(Number.MAX_VALUE), + since: z.coerce.number().int().positive().nullable().default(null), }) .strict()