From 5228029879a413089555b83ac846aaf9aeb5a370 Mon Sep 17 00:00:00 2001 From: Julian Gruber Date: Fri, 6 Dec 2024 13:59:53 +0100 Subject: [PATCH 1/6] wip --- stats/lib/platform-routes.js | 5 +++- stats/lib/platform-stats-fetchers.js | 15 ++++++++++++ stats/test/platform-routes.test.js | 36 ++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/stats/lib/platform-routes.js b/stats/lib/platform-routes.js index 3bb17a0..574240e 100644 --- a/stats/lib/platform-routes.js +++ b/stats/lib/platform-routes.js @@ -7,7 +7,8 @@ import { fetchTopEarningParticipants, fetchParticipantsWithTopMeasurements, fetchDailyStationMeasurementCounts, - fetchParticipantsSummary + fetchParticipantsSummary, + fetchAccumulativeDailyParticipantCount } from './platform-stats-fetchers.js' const createRespondWithFetchFn = (pathname, searchParams, res) => (pgPool, fetchFn) => { @@ -40,6 +41,8 @@ export const handlePlatformRoutes = async (req, res, pgPools) => { await respond(pgPools.stats, fetchTopEarningParticipants) } else if (req.method === 'GET' && url === '/participants/summary') { await respondWithParticipantsSummary(res, pgPools) + } else if (req.method === 'GET' && url === '/participants/accumulative/daily') { + await respond(pgPools.evaluate, fetchAccumulativeDailyParticipantCount) } else if (req.method === 'GET' && url === '/transfers/daily') { await respond(pgPools.stats, fetchDailyRewardTransfers) } else { diff --git a/stats/lib/platform-stats-fetchers.js b/stats/lib/platform-stats-fetchers.js index 2efa565..366aee0 100644 --- a/stats/lib/platform-stats-fetchers.js +++ b/stats/lib/platform-stats-fetchers.js @@ -77,6 +77,21 @@ export const fetchDailyRewardTransfers = async (pgPool, filter) => { return rows } +/** + * @param {Queryable} pgPool + * @param {import('./typings.js').DateRangeFilter} filter + */ +export const fetchAccumulativeDailyParticipantCount = async (pgPool, filter) => { + const { rows } = await pgPool.query(` + SELECT day::TEXT, COUNT(DISTINCT participant_id) as participants + FROM daily_participants + WHERE day >= $1 AND day <= $2 + GROUP BY day + ORDER BY day + `, [filter.from, filter.to]) + return rows +} + /** * @param {Queryable} pgPool * @param {import('./typings.js').DateRangeFilter} filter diff --git a/stats/test/platform-routes.test.js b/stats/test/platform-routes.test.js index 4fba293..0c7410e 100644 --- a/stats/test/platform-routes.test.js +++ b/stats/test/platform-routes.test.js @@ -362,6 +362,42 @@ describe('Platform Routes HTTP request handler', () => { ) }) }) + + describe('GET /participants/daily', () => { + it('counts daily participants', async () => { + await givenDailyParticipants( + pgPools.evaluate, + '2000-01-01', + ['0x1', '0x2', '0x3'] + ) + await givenDailyParticipants( + pgPools.evaluate, + '2000-01-03', + ['0x1'] + ) + await givenDailyParticipants( + pgPools.evaluate, + '2000-01-04', + ['0x1'] + ) + + const res = await fetch( + new URL('/participants/daily?from=2000-01-01&to=2000-01-03', baseUrl), { + redirect: 'manual' + } + ) + await assertResponseStatus(res, 200) + const daily = await res.json() + assert.deepStrictEqual(daily, [ + { day: '2000-01-01', participants: 3 }, + { day: '2000-01-03', participants: 1 } + ]) + assert.strictEqual( + res.headers.get('cache-control'), + 'public, max-age=86400' + ) + }) + }) }) const givenDailyMeasurementsSummary = async (pgPoolEvaluate, summaryData) => { From aba87e9c280de61b7ceace016d564fbf4789a52f Mon Sep 17 00:00:00 2001 From: Julian Gruber Date: Tue, 10 Dec 2024 13:35:57 +0100 Subject: [PATCH 2/6] test passes --- stats/lib/platform-stats-fetchers.js | 25 ++++++++++++++++++++++--- stats/test/platform-routes.test.js | 23 ++++++++++++++++------- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/stats/lib/platform-stats-fetchers.js b/stats/lib/platform-stats-fetchers.js index 366aee0..c43ee18 100644 --- a/stats/lib/platform-stats-fetchers.js +++ b/stats/lib/platform-stats-fetchers.js @@ -83,10 +83,29 @@ export const fetchDailyRewardTransfers = async (pgPool, filter) => { */ export const fetchAccumulativeDailyParticipantCount = async (pgPool, filter) => { const { rows } = await pgPool.query(` - SELECT day::TEXT, COUNT(DISTINCT participant_id) as participants - FROM daily_participants + WITH daily_participants_count AS ( + SELECT + day, + participant_id, + ROW_NUMBER() OVER (PARTITION BY participant_id ORDER BY day) AS rn + FROM daily_participants + ), + first_appearance AS ( + SELECT day, participant_id + FROM daily_participants_count + WHERE rn = 1 + ), + cumulative_participants AS ( + SELECT + day, + COUNT(participant_id) OVER (ORDER BY day) AS cumulative_participants + FROM first_appearance + ) + SELECT + DISTINCT(day::TEXT), + cumulative_participants::INT as participants + FROM cumulative_participants WHERE day >= $1 AND day <= $2 - GROUP BY day ORDER BY day `, [filter.from, filter.to]) return rows diff --git a/stats/test/platform-routes.test.js b/stats/test/platform-routes.test.js index 0c7410e..8e130fc 100644 --- a/stats/test/platform-routes.test.js +++ b/stats/test/platform-routes.test.js @@ -363,26 +363,35 @@ describe('Platform Routes HTTP request handler', () => { }) }) - describe('GET /participants/daily', () => { - it('counts daily participants', async () => { + describe('GET /participants/accumulative/daily', () => { + it('counts accumulative daily participants', async () => { + // 3 new participants await givenDailyParticipants( pgPools.evaluate, '2000-01-01', ['0x1', '0x2', '0x3'] ) + // 0 new participants, 2 old participants + await givenDailyParticipants( + pgPools.evaluate, + '2000-01-02', + ['0x1', '0x2'] + ) + // 1 new participant, 1 old participant await givenDailyParticipants( pgPools.evaluate, '2000-01-03', - ['0x1'] + ['0x1', '0x4'] ) + // out of range await givenDailyParticipants( pgPools.evaluate, '2000-01-04', - ['0x1'] + ['0x5'] ) const res = await fetch( - new URL('/participants/daily?from=2000-01-01&to=2000-01-03', baseUrl), { + new URL('/participants/accumulative/daily?from=2000-01-01&to=2000-01-03', baseUrl), { redirect: 'manual' } ) @@ -390,11 +399,11 @@ describe('Platform Routes HTTP request handler', () => { const daily = await res.json() assert.deepStrictEqual(daily, [ { day: '2000-01-01', participants: 3 }, - { day: '2000-01-03', participants: 1 } + { day: '2000-01-03', participants: 4 } ]) assert.strictEqual( res.headers.get('cache-control'), - 'public, max-age=86400' + 'public, max-age=31536000, immutable' ) }) }) From 7732e9812cf5672f6c8fc262195e4045cd2cb211 Mon Sep 17 00:00:00 2001 From: Julian Gruber Date: Tue, 10 Dec 2024 13:45:39 +0100 Subject: [PATCH 3/6] more test --- stats/test/platform-routes.test.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/stats/test/platform-routes.test.js b/stats/test/platform-routes.test.js index 8e130fc..0592eb6 100644 --- a/stats/test/platform-routes.test.js +++ b/stats/test/platform-routes.test.js @@ -365,7 +365,13 @@ describe('Platform Routes HTTP request handler', () => { describe('GET /participants/accumulative/daily', () => { it('counts accumulative daily participants', async () => { - // 3 new participants + // 3 new participants, out of range + await givenDailyParticipants( + pgPools.evaluate, + '1999-01-01', + ['0x10', '0x20', '0x30'] + ) + // 3 new participants -> 6 await givenDailyParticipants( pgPools.evaluate, '2000-01-01', @@ -377,13 +383,13 @@ describe('Platform Routes HTTP request handler', () => { '2000-01-02', ['0x1', '0x2'] ) - // 1 new participant, 1 old participant + // 1 new participant, 1 old participant -> 7 await givenDailyParticipants( pgPools.evaluate, '2000-01-03', ['0x1', '0x4'] ) - // out of range + // 1 new participant, out of range await givenDailyParticipants( pgPools.evaluate, '2000-01-04', @@ -398,8 +404,8 @@ describe('Platform Routes HTTP request handler', () => { await assertResponseStatus(res, 200) const daily = await res.json() assert.deepStrictEqual(daily, [ - { day: '2000-01-01', participants: 3 }, - { day: '2000-01-03', participants: 4 } + { day: '2000-01-01', participants: 6 }, + { day: '2000-01-03', participants: 7 } ]) assert.strictEqual( res.headers.get('cache-control'), From 8ac04b8a2c10ac896420f41b48a05b7aace4c3a9 Mon Sep 17 00:00:00 2001 From: Julian Gruber Date: Tue, 10 Dec 2024 13:46:16 +0100 Subject: [PATCH 4/6] more tests --- stats/test/platform-routes.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stats/test/platform-routes.test.js b/stats/test/platform-routes.test.js index 0592eb6..227085e 100644 --- a/stats/test/platform-routes.test.js +++ b/stats/test/platform-routes.test.js @@ -371,11 +371,11 @@ describe('Platform Routes HTTP request handler', () => { '1999-01-01', ['0x10', '0x20', '0x30'] ) - // 3 new participants -> 6 + // 3 new participants, 1 old participant -> 6 await givenDailyParticipants( pgPools.evaluate, '2000-01-01', - ['0x1', '0x2', '0x3'] + ['0x1', '0x2', '0x3', '0x10'] ) // 0 new participants, 2 old participants await givenDailyParticipants( From ad52c7e551fec401806fac1cdad158fd430bca23 Mon Sep 17 00:00:00 2001 From: Julian Gruber Date: Tue, 10 Dec 2024 13:51:20 +0100 Subject: [PATCH 5/6] docs --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 1bf403b..b897f34 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,10 @@ Base URL: http://stats.filspark.com/ http://stats.filspark.com/participants/monthly +- `GET /participants/accumulative/daily?from=&to=` + + http://stats.filspark.com/participants/accumulative/daily + - `GET /participants/change-rates?from=&to=` http://stats.filspark.com/participants/change-rates From 280f63a5b1962ab922c32ca500e978ad71a2b977 Mon Sep 17 00:00:00 2001 From: Julian Gruber Date: Mon, 16 Dec 2024 15:41:33 +0100 Subject: [PATCH 6/6] simplify query --- stats/lib/platform-stats-fetchers.js | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/stats/lib/platform-stats-fetchers.js b/stats/lib/platform-stats-fetchers.js index c43ee18..e42a60b 100644 --- a/stats/lib/platform-stats-fetchers.js +++ b/stats/lib/platform-stats-fetchers.js @@ -83,17 +83,10 @@ export const fetchDailyRewardTransfers = async (pgPool, filter) => { */ export const fetchAccumulativeDailyParticipantCount = async (pgPool, filter) => { const { rows } = await pgPool.query(` - WITH daily_participants_count AS ( - SELECT - day, - participant_id, - ROW_NUMBER() OVER (PARTITION BY participant_id ORDER BY day) AS rn + WITH first_appearance AS ( + SELECT participant_id, MIN(day) as day FROM daily_participants - ), - first_appearance AS ( - SELECT day, participant_id - FROM daily_participants_count - WHERE rn = 1 + GROUP BY participant_id ), cumulative_participants AS ( SELECT