From 24c6e00639e0fe0c5918980fad26898e804fbca5 Mon Sep 17 00:00:00 2001 From: Prospector <6166773+Prospector@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:24:42 -0700 Subject: [PATCH] Revert "Analytics backend V2 (#4408)" This reverts commit 6919c8dea9418d48b2f428c9c58a0e3fac72fa41. --- CLAUDE.md | 28 - Cargo.lock | 177 +-- Cargo.toml | 1 - ...5b67df45f8c07bb70e3c4f090d943feafdaf3.json | 37 + ...3611289b509ade9e1996d711598cd14c7f8fa.json | 37 - ...7a224ee4710e202cf1c8f3f35e49b54d6f2f9.json | 37 + apps/labrinth/Cargo.toml | 4 +- apps/labrinth/src/routes/v3/analytics_get.rs | 1387 +++++++---------- apps/labrinth/tests/analytics.rs | 287 ++-- apps/labrinth/tests/common/api_v3/project.rs | 85 +- 10 files changed, 905 insertions(+), 1175 deletions(-) create mode 100644 apps/labrinth/.sqlx/query-4198ea701f956dd65cab1a8e60b5b67df45f8c07bb70e3c4f090d943feafdaf3.json delete mode 100644 apps/labrinth/.sqlx/query-82b4d6e555dd727d31cca036b923611289b509ade9e1996d711598cd14c7f8fa.json create mode 100644 apps/labrinth/.sqlx/query-dfb4bd3db0d1cc2b2f811c267547a224ee4710e202cf1c8f3f35e49b54d6f2f9.json diff --git a/CLAUDE.md b/CLAUDE.md index 2563436225..172f4d8d58 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,33 +1,5 @@ # Architecture -## Frontend - -There are two similar frontends in the Modrinth monorepo, the website (apps/frontend) and the app frontend (apps/app-frontend). - -Both use Tailwind v3, and their respective configs can be seen at `tailwind.config.ts` and `tailwind.config.js` respectively. - -Both utilize shared and common components from `@modrinth/ui` which can be found at `packages/ui`, and stylings from `@modrinth/assets` which can be found at `packages/assets`. - -Both can utilize icons from `@modrinth/assets`, which are automatically generated based on what's available within the `icons` folder of the `packages/assets` directory. You can see the generated icons list in `generated-icons.ts`. - -Both have access to our dependency injection framework, examples as seen in `packages/ui/src/providers/`. Ideally any state which is shared between a page and it's subpages should be shared using this dependency injection framework. - -### Website (apps/frontend) - -Before a pull request can be opened for the website, `pnpm web:fix` and `pnpm web:intl:extract` must be run, otherwise CI will fail. - -To run a development version of the frontend, you must first copy over the relevant `.env` template file (prod, staging or local, usually prod) within the `apps/frontend` folder into `apps/frontend/.env`. Then you can run the frontend by running `pnpm web:dev` in the root folder. - -### App Frontend (apps/app-frontend) - -Before a pull request can be opened for the website, you must CD into the `app-frontend` folder; `pnpm fix` and `pnpm intl:extract` must be run, otherwise CI will fail. - -To run a development version of the app frontend, you must first copy over the relevant `.env` template file (prod, staging or local, usually prod) within `packages/app-lib` into `packages/app-lib/.env`. Then you must run the app itself by running `pnpm app:dev` in the root folder. - -### Localization - -Refer to `.github/instructions/i18n-convert.instructions.md` if the user asks you to perform any i18n conversion work on a component, set of components, pages or sets of pages. - ## Labrinth Labrinth is the backend API service for Modrinth. diff --git a/Cargo.lock b/Cargo.lock index 3c32cf100f..877c3bb52c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,9 +59,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.11.2" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7926860314cbe2fb5d1f13731e387ab43bd32bca224e82e6e2db85de0a3dba49" +checksum = "44cceded2fb55f3c4b67068fa64962e2ca59614edc5b03167de9ff82ae803da0" dependencies = [ "actix-codec", "actix-rt", @@ -603,7 +603,7 @@ dependencies = [ "polling", "rustix 1.1.2", "slab", - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -661,7 +661,7 @@ dependencies = [ "rustix 1.1.2", "signal-hook-registry", "slab", - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -941,7 +941,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -1364,7 +1364,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -1579,8 +1579,8 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.2", - "windows-sys 0.61.2", + "unicode-width 0.2.1", + "windows-sys 0.61.1", ] [[package]] @@ -1648,26 +1648,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "const_format" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" -dependencies = [ - "const_format_proc_macros", -] - -[[package]] -name = "const_format_proc_macros" -version = "0.2.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - [[package]] name = "constant_time_eq" version = "0.3.1" @@ -2264,7 +2244,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -2648,7 +2628,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -2821,9 +2801,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.4" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -3866,7 +3846,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.62.1", ] [[package]] @@ -4084,7 +4064,7 @@ checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" dependencies = [ "console", "portable-atomic", - "unicode-width 0.2.2", + "unicode-width 0.2.1", "unit-prefix", "web-time", ] @@ -4497,7 +4477,6 @@ dependencies = [ "color-eyre", "color-thief", "console-subscriber", - "const_format", "dashmap", "deadpool-redis", "dotenv-build", @@ -4676,7 +4655,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -5052,9 +5031,9 @@ dependencies = [ [[package]] name = "moxcms" -version = "0.7.6" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cc7d85f3d741164e8972ad355e26ac6e51b20fcae5f911c7da8f2d8bbbb3f33" +checksum = "ddd32fa8935aeadb8a8a6b6b351e40225570a37c43de67690383d87ef170cd08" dependencies = [ "num-traits", "pxfm", @@ -5867,7 +5846,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -6221,7 +6200,7 @@ dependencies = [ "hermit-abi", "pin-project-lite", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -7324,7 +7303,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -7481,7 +7460,7 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -9216,7 +9195,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -10079,9 +10058,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unicode-xid" @@ -10707,7 +10686,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.61.1", ] [[package]] @@ -10768,15 +10747,15 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.62.2" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +checksum = "6844ee5416b285084d3d3fffd743b925a6c9385455f64f6d4fa3031c4c2749a9" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", + "windows-link 0.2.0", + "windows-result 0.4.0", + "windows-strings 0.5.0", ] [[package]] @@ -10792,9 +10771,9 @@ dependencies = [ [[package]] name = "windows-implement" -version = "0.60.2" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +checksum = "edb307e42a74fb6de9bf3a02d9712678b22399c87e6fa869d6dfcd8c1b7754e0" dependencies = [ "proc-macro2", "quote", @@ -10803,9 +10782,9 @@ dependencies = [ [[package]] name = "windows-interface" -version = "0.59.3" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +checksum = "c0abd1ddbc6964ac14db11c7213d6532ef34bd9aa042c2e5935f59d7908b46a5" dependencies = [ "proc-macro2", "quote", @@ -10820,9 +10799,9 @@ checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" [[package]] name = "windows-numerics" @@ -10856,11 +10835,11 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" dependencies = [ - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -10874,11 +10853,11 @@ dependencies = [ [[package]] name = "windows-strings" -version = "0.5.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" dependencies = [ - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -10923,16 +10902,16 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.5", + "windows-targets 0.53.4", ] [[package]] name = "windows-sys" -version = "0.61.2" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +checksum = "6f109e41dd4a3c848907eb83d5a42ea98b3769495597450cf6d153507b166f0f" dependencies = [ - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -10983,19 +10962,19 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.5" +version = "0.53.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +checksum = "2d42b7b7f66d2a06854650af09cfdf8713e427a439c97ad65a6375318033ac4b" dependencies = [ - "windows-link 0.2.1", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows-link 0.2.0", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -11009,11 +10988,11 @@ dependencies = [ [[package]] name = "windows-version" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4060a1da109b9d0326b7262c8e12c84df67cc0dbc9e33cf49e01ccc2eb63631" +checksum = "700dad7c058606087f6fdc1f88da5841e06da40334413c6cd4367b25ef26d24e" dependencies = [ - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -11036,9 +11015,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" [[package]] name = "windows_aarch64_msvc" @@ -11060,9 +11039,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" [[package]] name = "windows_i686_gnu" @@ -11084,9 +11063,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" [[package]] name = "windows_i686_gnullvm" @@ -11096,9 +11075,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" [[package]] name = "windows_i686_msvc" @@ -11120,9 +11099,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" [[package]] name = "windows_x86_64_gnu" @@ -11144,9 +11123,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" [[package]] name = "windows_x86_64_gnullvm" @@ -11168,9 +11147,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" [[package]] name = "windows_x86_64_msvc" @@ -11192,9 +11171,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" @@ -11264,9 +11243,9 @@ checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "wry" -version = "0.53.4" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d78ec082b80fa088569a970d043bb3050abaabf4454101d44514ee8d9a8c9f6" +checksum = "31f0e9642a0d061f6236c54ccae64c2722a7879ad4ec7dff59bd376d446d8e90" dependencies = [ "base64 0.22.1", "block2 0.6.2", diff --git a/Cargo.toml b/Cargo.toml index 060de4bf42..5f0e7bb6b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,6 @@ async-stripe = { version = "0.41.0", default-features = false, features = [ async-trait = "0.1.88" async-tungstenite = { version = "0.30.0", default-features = false, features = ["futures-03-sink"] } async-walkdir = "2.1.0" -const_format = "0.2.34" base64 = "0.22.1" bitflags = "2.9.1" bytemuck = "1.23.1" diff --git a/apps/labrinth/.sqlx/query-4198ea701f956dd65cab1a8e60b5b67df45f8c07bb70e3c4f090d943feafdaf3.json b/apps/labrinth/.sqlx/query-4198ea701f956dd65cab1a8e60b5b67df45f8c07bb70e3c4f090d943feafdaf3.json new file mode 100644 index 0000000000..be792ea9aa --- /dev/null +++ b/apps/labrinth/.sqlx/query-4198ea701f956dd65cab1a8e60b5b67df45f8c07bb70e3c4f090d943feafdaf3.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT mod_id, SUM(amount) amount_sum, DATE_BIN($4::interval, created, TIMESTAMP '2001-01-01') AS interval_start\n FROM payouts_values\n WHERE mod_id = ANY($1) AND created BETWEEN $2 AND $3\n GROUP by mod_id, interval_start ORDER BY interval_start\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mod_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "amount_sum", + "type_info": "Numeric" + }, + { + "ordinal": 2, + "name": "interval_start", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8Array", + "Timestamptz", + "Timestamptz", + "Interval" + ] + }, + "nullable": [ + true, + null, + null + ] + }, + "hash": "4198ea701f956dd65cab1a8e60b5b67df45f8c07bb70e3c4f090d943feafdaf3" +} diff --git a/apps/labrinth/.sqlx/query-82b4d6e555dd727d31cca036b923611289b509ade9e1996d711598cd14c7f8fa.json b/apps/labrinth/.sqlx/query-82b4d6e555dd727d31cca036b923611289b509ade9e1996d711598cd14c7f8fa.json deleted file mode 100644 index 607f4aa33f..0000000000 --- a/apps/labrinth/.sqlx/query-82b4d6e555dd727d31cca036b923611289b509ade9e1996d711598cd14c7f8fa.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n WIDTH_BUCKET(\n EXTRACT(EPOCH FROM created)::bigint,\n EXTRACT(EPOCH FROM $1::timestamp with time zone AT TIME ZONE 'UTC')::bigint,\n EXTRACT(EPOCH FROM $2::timestamp with time zone AT TIME ZONE 'UTC')::bigint,\n $3::integer\n ) AS bucket,\n COALESCE(mod_id, 0) AS mod_id,\n SUM(amount) amount_sum\n FROM payouts_values\n WHERE\n user_id = $4\n AND created BETWEEN $1 AND $2\n GROUP BY bucket, mod_id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "bucket", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "mod_id", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "amount_sum", - "type_info": "Numeric" - } - ], - "parameters": { - "Left": [ - "Timestamptz", - "Timestamptz", - "Int4", - "Int8" - ] - }, - "nullable": [ - null, - null, - null - ] - }, - "hash": "82b4d6e555dd727d31cca036b923611289b509ade9e1996d711598cd14c7f8fa" -} diff --git a/apps/labrinth/.sqlx/query-dfb4bd3db0d1cc2b2f811c267547a224ee4710e202cf1c8f3f35e49b54d6f2f9.json b/apps/labrinth/.sqlx/query-dfb4bd3db0d1cc2b2f811c267547a224ee4710e202cf1c8f3f35e49b54d6f2f9.json new file mode 100644 index 0000000000..2515dfe43e --- /dev/null +++ b/apps/labrinth/.sqlx/query-dfb4bd3db0d1cc2b2f811c267547a224ee4710e202cf1c8f3f35e49b54d6f2f9.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT mod_id, SUM(amount) amount_sum, DATE_BIN($4::interval, created, TIMESTAMP '2001-01-01') AS interval_start\n FROM payouts_values\n WHERE user_id = $1 AND created BETWEEN $2 AND $3\n GROUP by mod_id, interval_start ORDER BY interval_start\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mod_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "amount_sum", + "type_info": "Numeric" + }, + { + "ordinal": 2, + "name": "interval_start", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8", + "Timestamptz", + "Timestamptz", + "Interval" + ] + }, + "nullable": [ + true, + null, + null + ] + }, + "hash": "dfb4bd3db0d1cc2b2f811c267547a224ee4710e202cf1c8f3f35e49b54d6f2f9" +} diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 2945eda5a4..e8758e98b0 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -58,7 +58,7 @@ sha2.workspace = true hmac.workspace = true argon2.workspace = true murmur2.workspace = true -bitflags = { workspace = true, features = ["serde"] } +bitflags.workspace = true hex.workspace = true zxcvbn.workspace = true totp-rs = { workspace = true, features = ["gen_secret"] } @@ -137,8 +137,6 @@ path-util.workspace = true clap = { workspace = true, features = ["derive"] } -const_format.workspace = true - [target.'cfg(target_os = "linux")'.dependencies] tikv-jemallocator = { workspace = true, features = [ "profiling", diff --git a/apps/labrinth/src/routes/v3/analytics_get.rs b/apps/labrinth/src/routes/v3/analytics_get.rs index 628416df12..582a85ac5d 100644 --- a/apps/labrinth/src/routes/v3/analytics_get.rs +++ b/apps/labrinth/src/routes/v3/analytics_get.rs @@ -1,876 +1,673 @@ -//! # Design rationale -//! -//! - different metrics require different scopes -//! - views, downloads, playtime requires `Scopes::ANALYTICS` -//! - revenue requires `Scopes::PAYOUTS_READ` -//! - each request returns an array of N elements; if you have to make multiple -//! requests, you have to zip together M arrays of N elements -//! - this makes it inconvenient to have separate endpoints - -use std::num::NonZeroU64; - -use actix_web::{HttpRequest, web}; -use chrono::{DateTime, TimeDelta, Utc}; -use futures::StreamExt; -use rust_decimal::Decimal; -use serde::{Deserialize, Serialize}; -use sqlx::PgPool; - +use super::ApiError; +use crate::database; +use crate::database::redis::RedisPool; +use crate::models::teams::ProjectPermissions; use crate::{ - auth::{AuthenticationError, get_user_from_headers}, - database::{ - self, DBProject, - models::{DBProjectId, DBUser, DBUserId, DBVersionId}, - redis::RedisPool, - }, + auth::get_user_from_headers, + database::models::user_item, models::{ ids::{ProjectId, VersionId}, pats::Scopes, - teams::ProjectPermissions, }, queue::session::AuthQueue, - routes::ApiError, }; +use actix_web::{HttpRequest, HttpResponse, web}; +use ariadne::ids::base62_impl::to_base62; +use chrono::{DateTime, Duration, Utc}; +use eyre::eyre; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; +use sqlx::postgres::types::PgInterval; +use std::collections::HashMap; +use std::convert::TryInto; +use std::num::NonZeroU32; pub fn config(cfg: &mut web::ServiceConfig) { - cfg.service(web::scope("analytics").route("", web::post().to(get))); -} - -// request - -/// Requests analytics data, aggregating over all possible analytics sources -/// like projects and affiliate codes, returning the data in a list of time -/// slices. -#[derive(Debug, Serialize, Deserialize)] -pub struct GetRequest { - /// What time range to return statistics for. - pub time_range: TimeRange, - /// What analytics metrics to return data for. - pub return_metrics: ReturnMetrics, -} - -/// Time range for fetching analytics. -#[derive(Debug, Serialize, Deserialize)] -pub struct TimeRange { - /// When to start including data. - pub start: DateTime, - /// When to stop including data. - pub end: DateTime, - /// Determines how many time slices between the start and end will be - /// included, and how fine-grained those time slices will be. - /// - /// This must fall within the bounds of [`MIN_RESOLUTION`] and - /// [`MAX_TIME_SLICES`]. - pub resolution: TimeRangeResolution, -} - -/// Determines how many time slices between the start and end will be -/// included, and how fine-grained those time slices will be. -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TimeRangeResolution { - /// Use a set number of time slices, with the resolution being determined - /// automatically. - Slices(NonZeroU64), - /// Each time slice will be a set number of minutes long, and the number of - /// slices is determined automatically. - Minutes(NonZeroU64), + cfg.service( + web::scope("analytics") + .route("playtime", web::get().to(playtimes_get)) + .route("views", web::get().to(views_get)) + .route("downloads", web::get().to(downloads_get)) + .route("revenue", web::get().to(revenue_get)) + .route( + "countries/downloads", + web::get().to(countries_downloads_get), + ) + .route("countries/views", web::get().to(countries_views_get)), + ); } -/// What metrics the caller would like to receive from this analytics get -/// request. -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct ReturnMetrics { - /// How many times a project page has been viewed. - pub project_views: Option>, - /// How many times a project has been downloaded. - pub project_downloads: Option>, - /// How long users have been playing a project. - pub project_playtime: Option>, - /// How much payout revenue a project has generated. - pub project_revenue: Option>, -} +/// The json data to be passed to fetch analytic data +/// Either a list of project_ids or version_ids can be used, but not both. Unauthorized projects/versions will be filtered out. +/// start_date and end_date are optional, and default to two weeks ago, and the maximum date respectively. +/// resolution_minutes is optional. This refers to the window by which we are looking (every day, every minute, etc) and defaults to 1440 (1 day) +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GetData { + // only one of project_ids or version_ids should be used + // if neither are provided, all projects the user has access to will be used + pub project_ids: Option, -/// See [`ReturnMetrics`]. -#[derive(Debug, Serialize, Deserialize)] -pub struct Metrics { - /// When collecting metrics, what fields do we want to group the results by? - /// - /// For example, if we have two views entries: - /// - `{ "project_id": "abcdefgh", "domain": "youtube.com", "count": 5 }` - /// - `{ "project_id": "abcdefgh", "domain": "discord.com", "count": 3 }` - /// - /// If we bucket by `domain`, then we will get two results: - /// - `{ "project_id": "abcdefgh", "domain": "youtube.com", "count": 5 }` - /// - `{ "project_id": "abcdefgh", "domain": "discord.com", "count": 3 }` - /// - /// If we do not bucket by `domain`, we will only get one, which is an - /// aggregate of the two rows: - /// - `{ "project_id": "abcdefgh", "count": 8 }` - #[serde(default = "Vec::default")] - pub bucket_by: Vec, -} + pub start_date: Option>, // defaults to 2 weeks ago + pub end_date: Option>, // defaults to now -/// Fields for [`ReturnMetrics::project_views`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProjectViewsField { - /// Project ID. - ProjectId, - /// Referrer domain which linked to this project. - Domain, - /// Modrinth site path which was visited, e.g. `/mod/foo`. - SitePath, - /// Whether these views were monetized or not. - Monetized, - /// What country these views came from. - /// - /// To anonymize the data, the country may be reported as `XX`. - Country, + pub resolution_minutes: Option, // defaults to 1 day. Ignored in routes that do not aggregate over a resolution (eg: /countries) } -/// Fields for [`ReturnMetrics::project_downloads`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProjectDownloadsField { - /// Project ID. - ProjectId, - /// Version ID of this project. - VersionId, - /// Referrer domain which linked to this project. - Domain, - /// Modrinth site path which was visited, e.g. `/mod/foo`. - SitePath, - /// What country these views came from. - /// - /// To anonymize the data, the country may be reported as `XX`. - Country, +/// Get playtime data for a set of projects or versions +/// Data is returned as a hashmap of project/version ids to a hashmap of days to playtime data +/// eg: +/// { +/// "4N1tEhnO": { +/// "20230824": 23 +/// } +///} +/// Either a list of project_ids or version_ids can be used, but not both. Unauthorized projects/versions will be filtered out. +#[derive(Serialize, Deserialize, Clone)] +pub struct FetchedPlaytime { + pub time: u64, + pub total_seconds: u64, + pub loader_seconds: HashMap, + pub game_version_seconds: HashMap, + pub parent_seconds: HashMap, } +pub async fn playtimes_get( + req: HttpRequest, + clickhouse: web::Data, + data: web::Query, + session_queue: web::Data, + pool: web::Data, + redis: web::Data, +) -> Result { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::ANALYTICS, + ) + .await + .map(|x| x.1)?; + + let project_ids = data + .project_ids + .as_ref() + .map(|ids| serde_json::from_str::>(ids)) + .transpose()?; + + let start_date = data.start_date.unwrap_or(Utc::now() - Duration::weeks(2)); + let end_date = data.end_date.unwrap_or(Utc::now()); + let resolution_minutes = data + .resolution_minutes + .map_or(60 * 24, |minutes| minutes.get()); + + // Convert String list to list of ProjectIds or VersionIds + // - Filter out unauthorized projects/versions + // - If no project_ids or version_ids are provided, we default to all projects the user has access to + let project_ids = + filter_allowed_ids(project_ids, user, &pool, &redis, None).await?; + + // Get the views + let playtimes = crate::clickhouse::fetch_playtimes( + project_ids.unwrap_or_default(), + start_date, + end_date, + resolution_minutes, + clickhouse.into_inner(), + ) + .await?; -/// Fields for [`ReturnMetrics::project_playtime`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProjectPlaytimeField { - /// Project ID. - ProjectId, - /// Version ID of this project. - VersionId, - /// Game mod loader which was used to count this playtime, e.g. Fabric. - Loader, - /// Game version which this project was played on. - GameVersion, -} + let mut hm = HashMap::new(); + for playtime in playtimes { + let id_string = to_base62(playtime.id); + if !hm.contains_key(&id_string) { + hm.insert(id_string.clone(), HashMap::new()); + } + if let Some(hm) = hm.get_mut(&id_string) { + hm.insert(playtime.time, playtime.total); + } + } -/// Minimum width of a [`TimeSlice`], controlled by [`TimeRange::resolution`]. -pub const MIN_RESOLUTION: TimeDelta = TimeDelta::minutes(60); - -/// Maximum number of [`TimeSlice`]s in a [`GetResponse`], controlled by -/// [`TimeRange::resolution`]. -pub const MAX_TIME_SLICES: usize = 1024; - -// response - -/// Response for a [`GetRequest`]. -/// -/// This is a list of N [`TimeSlice`]s, where each slice represents an equal -/// time interval of metrics collection. The number of slices is determined -/// by [`GetRequest::time_range`]. -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct GetResponse(pub Vec); - -/// Single time interval of metrics collection. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct TimeSlice(pub Vec); - -/// Metrics collected in a single [`TimeSlice`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] // the presence of `source_project`, `source_affiliate_code` determines the kind -pub enum AnalyticsData { - /// Project metrics. - Project(ProjectAnalytics), - // AffiliateCode(AffiliateCodeAnalytics), + Ok(HttpResponse::Ok().json(hm)) } -/// Project metrics. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ProjectAnalytics { - /// What project these metrics are for. - source_project: ProjectId, - /// Metrics collected. - #[serde(flatten)] - metrics: ProjectMetrics, -} +/// Get view data for a set of projects or versions +/// Data is returned as a hashmap of project/version ids to a hashmap of days to views +/// eg: +/// { +/// "4N1tEhnO": { +/// "20230824": 1090 +/// } +///} +/// Either a list of project_ids or version_ids can be used, but not both. Unauthorized projects/versions will be filtered out. +pub async fn views_get( + req: HttpRequest, + clickhouse: web::Data, + data: web::Query, + session_queue: web::Data, + pool: web::Data, + redis: web::Data, +) -> Result { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::ANALYTICS, + ) + .await + .map(|x| x.1)?; + + let project_ids = data + .project_ids + .as_ref() + .map(|ids| serde_json::from_str::>(ids)) + .transpose()?; + + let start_date = data.start_date.unwrap_or(Utc::now() - Duration::weeks(2)); + let end_date = data.end_date.unwrap_or(Utc::now()); + let resolution_minutes = data + .resolution_minutes + .map_or(60 * 24, |minutes| minutes.get()); + + // Convert String list to list of ProjectIds or VersionIds + // - Filter out unauthorized projects/versions + // - If no project_ids or version_ids are provided, we default to all projects the user has access to + let project_ids = + filter_allowed_ids(project_ids, user, &pool, &redis, None).await?; + + // Get the views + let views = crate::clickhouse::fetch_views( + project_ids.unwrap_or_default(), + start_date, + end_date, + resolution_minutes, + clickhouse.into_inner(), + ) + .await?; -impl ProjectAnalytics { - /// Get the project ID for these analytics. - pub fn project_id(&self) -> &ProjectId { - &self.source_project + let mut hm = HashMap::new(); + for views in views { + let id_string = to_base62(views.id); + if !hm.contains_key(&id_string) { + hm.insert(id_string.clone(), HashMap::new()); + } + if let Some(hm) = hm.get_mut(&id_string) { + hm.insert(views.time, views.total); + } } -} -/// Project metrics of a specific kind. -/// -/// If a field is not included in [`Metrics::bucket_by`], it will be [`None`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case", tag = "metric_kind")] -pub enum ProjectMetrics { - /// [`ReturnMetrics::project_views`]. - Views(ProjectViews), - /// [`ReturnMetrics::project_downloads`]. - Downloads(ProjectDownloads), - /// [`ReturnMetrics::project_playtime`]. - Playtime(ProjectPlaytime), - /// [`ReturnMetrics::project_revenue`]. - Revenue(ProjectRevenue), + Ok(HttpResponse::Ok().json(hm)) } -/// [`ReturnMetrics::project_views`]. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ProjectViews { - /// [`ProjectViewsField::Domain`]. - #[serde(skip_serializing_if = "Option::is_none")] - pub domain: Option, - /// [`ProjectViewsField::SitePath`]. - #[serde(skip_serializing_if = "Option::is_none")] - pub site_path: Option, - /// [`ProjectViewsField::Monetized`]. - #[serde(skip_serializing_if = "Option::is_none")] - pub monetized: Option, - /// [`ProjectViewsField::Country`]. - #[serde(skip_serializing_if = "Option::is_none")] - pub country: Option, - /// Total number of views for this bucket. - pub views: u64, -} +/// Get download data for a set of projects or versions +/// Data is returned as a hashmap of project/version ids to a hashmap of days to downloads +/// eg: +/// { +/// "4N1tEhnO": { +/// "20230824": 32 +/// } +///} +/// Either a list of project_ids or version_ids can be used, but not both. Unauthorized projects/versions will be filtered out. +pub async fn downloads_get( + req: HttpRequest, + clickhouse: web::Data, + data: web::Query, + session_queue: web::Data, + pool: web::Data, + redis: web::Data, +) -> Result { + let user_option = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::ANALYTICS, + ) + .await + .map(|x| x.1)?; + + let project_ids = data + .project_ids + .as_ref() + .map(|ids| serde_json::from_str::>(ids)) + .transpose()?; + + let start_date = data.start_date.unwrap_or(Utc::now() - Duration::weeks(2)); + let end_date = data.end_date.unwrap_or(Utc::now()); + let resolution_minutes = data + .resolution_minutes + .map_or(60 * 24, |minutes| minutes.get()); + + // Convert String list to list of ProjectIds or VersionIds + // - Filter out unauthorized projects/versions + // - If no project_ids or version_ids are provided, we default to all projects the user has access to + let project_ids = + filter_allowed_ids(project_ids, user_option, &pool, &redis, None) + .await?; + + // Get the downloads + let downloads = crate::clickhouse::fetch_downloads( + project_ids.unwrap_or_default(), + start_date, + end_date, + resolution_minutes, + clickhouse.into_inner(), + ) + .await?; -/// [`ReturnMetrics::project_downloads`]. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ProjectDownloads { - /// [`ProjectDownloadsField::Domain`]. - #[serde(skip_serializing_if = "Option::is_none")] - domain: Option, - /// [`ProjectDownloadsField::SitePath`]. - #[serde(skip_serializing_if = "Option::is_none")] - site_path: Option, - /// [`ProjectDownloadsField::VersionId`]. - #[serde(skip_serializing_if = "Option::is_none")] - version_id: Option, - /// [`ProjectDownloadsField::Country`]. - #[serde(skip_serializing_if = "Option::is_none")] - country: Option, - /// Total number of downloads for this bucket. - downloads: u64, -} + let mut hm = HashMap::new(); + for downloads in downloads { + let id_string = to_base62(downloads.id); + if !hm.contains_key(&id_string) { + hm.insert(id_string.clone(), HashMap::new()); + } + if let Some(hm) = hm.get_mut(&id_string) { + hm.insert(downloads.time, downloads.total); + } + } -/// [`ReturnMetrics::project_playtime`]. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ProjectPlaytime { - /// [`ProjectPlaytimeField::VersionId`]. - #[serde(skip_serializing_if = "Option::is_none")] - version_id: Option, - /// [`ProjectPlaytimeField::Loader`]. - #[serde(skip_serializing_if = "Option::is_none")] - loader: Option, - /// [`ProjectPlaytimeField::GameVersion`]. - #[serde(skip_serializing_if = "Option::is_none")] - game_version: Option, - /// Total number of seconds of playtime for this bucket. - seconds: u64, + Ok(HttpResponse::Ok().json(hm)) } -/// [`ReturnMetrics::project_revenue`]. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ProjectRevenue { - /// Total revenue for this bucket. - revenue: Decimal, -} +/// Get payout data for a set of projects +/// Data is returned as a hashmap of project ids to a hashmap of days to amount earned per day +/// eg: +/// { +/// "4N1tEhnO": { +/// "20230824": 0.001 +/// } +///} +/// ONLY project IDs can be used. Unauthorized projects will be filtered out. +pub async fn revenue_get( + req: HttpRequest, + data: web::Query, + session_queue: web::Data, + pool: web::Data, + redis: web::Data, +) -> Result { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::PAYOUTS_READ, + ) + .await + .map(|x| x.1)?; + + let project_ids = data + .project_ids + .as_ref() + .map(|ids| serde_json::from_str::>(ids)) + .transpose()?; + + let start_date = data.start_date.unwrap_or(Utc::now() - Duration::weeks(2)); + let end_date = data.end_date.unwrap_or(Utc::now()); + let resolution_minutes = data + .resolution_minutes + .map_or(60 * 24, |minutes| minutes.get()); + + // Round up/down to nearest duration as we are using pgadmin, does not have rounding in the fetch command + // Round start_date down to nearest resolution + let diff = start_date.timestamp() % (resolution_minutes as i64 * 60); + let start_date = start_date - Duration::seconds(diff); + + // Round end_date up to nearest resolution + let diff = end_date.timestamp() % (resolution_minutes as i64 * 60); + let end_date = + end_date + Duration::seconds((resolution_minutes as i64 * 60) - diff); + + // Convert String list to list of ProjectIds or VersionIds + // - Filter out unauthorized projects/versions + // - If no project_ids or version_ids are provided, we default to all projects the user has access to + let project_ids = filter_allowed_ids( + project_ids, + user.clone(), + &pool, + &redis, + Some(true), + ) + .await?; -// logic - -/// Clickhouse queries - separate from [`sqlx`] queries. -mod query { - use crate::database::models::{DBProjectId, DBVersionId}; - use const_format::formatcp; - - const TIME_RANGE_START: &str = "{time_range_start: UInt64}"; - const TIME_RANGE_END: &str = "{time_range_end: UInt64}"; - const TIME_SLICES: &str = "{time_slices: UInt64}"; - const PROJECT_IDS: &str = "{project_ids: Array(UInt64)}"; - - #[derive(Debug, clickhouse::Row, serde::Deserialize)] - pub struct ViewRow { - pub bucket: u64, - pub project_id: DBProjectId, - pub domain: String, - pub site_path: String, - pub monetized: i8, - pub country: String, - pub views: u64, + let duration: PgInterval = Duration::minutes(resolution_minutes as i64) + .try_into() + .map_err(|_| { + ApiError::Request(eyre!("Invalid `resolution_minutes`")) + })?; + // Get the revenue data + let project_ids = project_ids.unwrap_or_default(); + + struct PayoutValue { + mod_id: Option, + amount_sum: Option, + interval_start: Option>, } - pub const VIEWS: &str = { - const USE_PROJECT_ID: &str = "{use_project_id: Bool}"; - const USE_DOMAIN: &str = "{use_domain: Bool}"; - const USE_SITE_PATH: &str = "{use_site_path: Bool}"; - const USE_MONETIZED: &str = "{use_monetized: Bool}"; - const USE_COUNTRY: &str = "{use_country: Bool}"; - - formatcp!( - "SELECT - widthBucket(toUnixTimestamp(recorded), {TIME_RANGE_START}, {TIME_RANGE_END}, {TIME_SLICES}) AS bucket, - if({USE_PROJECT_ID}, project_id, 0) AS project_id, - if({USE_DOMAIN}, domain, '') AS domain, - if({USE_SITE_PATH}, site_path, '') AS site_path, - if({USE_MONETIZED}, CAST(monetized AS Int8), -1) AS monetized, - if({USE_COUNTRY}, country, '') AS country, - COUNT(*) AS views - FROM views - WHERE - recorded BETWEEN {TIME_RANGE_START} AND {TIME_RANGE_END} - -- make sure that the REAL project id is included, - -- not the possibly-zero one, - -- by using `views.project_id` instead of `project_id` - AND views.project_id IN {PROJECT_IDS} - GROUP BY - bucket, project_id, domain, site_path, monetized, country" + let payouts_values = if project_ids.is_empty() { + sqlx::query!( + " + SELECT mod_id, SUM(amount) amount_sum, DATE_BIN($4::interval, created, TIMESTAMP '2001-01-01') AS interval_start + FROM payouts_values + WHERE user_id = $1 AND created BETWEEN $2 AND $3 + GROUP by mod_id, interval_start ORDER BY interval_start + ", + user.id.0 as i64, + start_date, + end_date, + duration, ) - }; - - #[derive(Debug, clickhouse::Row, serde::Deserialize)] - pub struct DownloadRow { - pub bucket: u64, - pub project_id: DBProjectId, - pub domain: String, - pub site_path: String, - pub version_id: DBVersionId, - pub country: String, - pub downloads: u64, - } - - pub const DOWNLOADS: &str = { - const USE_PROJECT_ID: &str = "{use_project_id: Bool}"; - const USE_DOMAIN: &str = "{use_domain: Bool}"; - const USE_SITE_PATH: &str = "{use_site_path: Bool}"; - const USE_VERSION_ID: &str = "{use_version_id: Bool}"; - const USE_COUNTRY: &str = "{use_country: Bool}"; - - formatcp!( - "SELECT - widthBucket(toUnixTimestamp(recorded), {TIME_RANGE_START}, {TIME_RANGE_END}, {TIME_SLICES}) AS bucket, - if({USE_PROJECT_ID}, project_id, 0) AS project_id, - if({USE_DOMAIN}, domain, '') AS domain, - if({USE_SITE_PATH}, site_path, '') AS site_path, - if({USE_VERSION_ID}, version_id, 0) AS version_id, - if({USE_COUNTRY}, country, '') AS country, - COUNT(*) AS downloads - FROM downloads - WHERE - recorded BETWEEN {TIME_RANGE_START} AND {TIME_RANGE_END} - -- make sure that the REAL project id is included, - -- not the possibly-zero one, - -- by using `downloads.project_id` instead of `project_id` - AND downloads.project_id IN {PROJECT_IDS} - GROUP BY - bucket, project_id, domain, site_path, version_id, country" + .fetch_all(&**pool) + .await?.into_iter().map(|x| PayoutValue { + mod_id: x.mod_id, + amount_sum: x.amount_sum, + interval_start: x.interval_start, + }).collect::>() + } else { + sqlx::query!( + " + SELECT mod_id, SUM(amount) amount_sum, DATE_BIN($4::interval, created, TIMESTAMP '2001-01-01') AS interval_start + FROM payouts_values + WHERE mod_id = ANY($1) AND created BETWEEN $2 AND $3 + GROUP by mod_id, interval_start ORDER BY interval_start + ", + &project_ids.iter().map(|x| x.0 as i64).collect::>(), + start_date, + end_date, + duration, ) + .fetch_all(&**pool) + .await?.into_iter().map(|x| PayoutValue { + mod_id: x.mod_id, + amount_sum: x.amount_sum, + interval_start: x.interval_start, + }).collect::>() }; - #[derive(Debug, clickhouse::Row, serde::Deserialize)] - pub struct PlaytimeRow { - pub bucket: u64, - pub project_id: DBProjectId, - pub version_id: DBVersionId, - pub loader: String, - pub game_version: String, - pub seconds: u64, + let mut hm: HashMap<_, _> = project_ids + .into_iter() + .map(|x| (x.to_string(), HashMap::new())) + .collect::>(); + for value in payouts_values { + if let Some(mod_id) = value.mod_id + && let Some(amount) = value.amount_sum + && let Some(interval_start) = value.interval_start + { + let id_string = to_base62(mod_id as u64); + if !hm.contains_key(&id_string) { + hm.insert(id_string.clone(), HashMap::new()); + } + if let Some(hm) = hm.get_mut(&id_string) { + hm.insert(interval_start.timestamp(), amount); + } + } } - pub const PLAYTIME: &str = { - const USE_PROJECT_ID: &str = "{use_project_id: Bool}"; - const USE_VERSION_ID: &str = "{use_version_id: Bool}"; - const USE_LOADER: &str = "{use_loader: Bool}"; - const USE_GAME_VERSION: &str = "{use_game_version: Bool}"; - - formatcp!( - "SELECT - widthBucket(toUnixTimestamp(recorded), {TIME_RANGE_START}, {TIME_RANGE_END}, {TIME_SLICES}) AS bucket, - if({USE_PROJECT_ID}, project_id, 0) AS project_id, - if({USE_VERSION_ID}, version_id, 0) AS version_id, - if({USE_LOADER}, loader, '') AS loader, - if({USE_GAME_VERSION}, game_version, '') AS game_version, - SUM(seconds) AS seconds - FROM playtime - WHERE - recorded BETWEEN {TIME_RANGE_START} AND {TIME_RANGE_END} - -- make sure that the REAL project id is included, - -- not the possibly-zero one, - -- by using `playtime.project_id` instead of `project_id` - AND playtime.project_id IN {PROJECT_IDS} - GROUP BY - bucket, project_id, version_id, loader, game_version" - ) - }; + Ok(HttpResponse::Ok().json(hm)) } -async fn get( - http_req: HttpRequest, - req: web::Json, +/// Get country data for a set of projects or versions +/// Data is returned as a hashmap of project/version ids to a hashmap of coutnry to downloads. +/// Unknown countries are labeled "". +/// This is usuable to see significant performing countries per project +/// eg: +/// { +/// "4N1tEhnO": { +/// "CAN": 22 +/// } +///} +/// Either a list of project_ids or version_ids can be used, but not both. Unauthorized projects/versions will be filtered out. +/// For this endpoint, provided dates are a range to aggregate over, not specific days to fetch +pub async fn countries_downloads_get( + req: HttpRequest, + clickhouse: web::Data, + data: web::Query, + session_queue: web::Data, pool: web::Data, redis: web::Data, - session_queue: web::Data, - clickhouse: web::Data, -) -> Result, ApiError> { - let (scopes, user) = get_user_from_headers( - &http_req, +) -> Result { + let user = get_user_from_headers( + &req, &**pool, &redis, &session_queue, Scopes::ANALYTICS, ) - .await?; + .await + .map(|x| x.1)?; - let full_time_range = req.time_range.end - req.time_range.start; - if full_time_range < TimeDelta::zero() { - return Err(ApiError::InvalidInput( - "End date must be after start date".into(), - )); - } + let project_ids = data + .project_ids + .as_ref() + .map(|ids| serde_json::from_str::>(ids)) + .transpose()?; - let (num_time_slices, resolution) = match req.time_range.resolution { - TimeRangeResolution::Slices(slices) => { - let slices = i32::try_from(slices.get()).map_err(|_| { - ApiError::InvalidInput( - "Number of slices must fit into an `i32`".into(), - ) - })?; - let resolution = full_time_range / slices; - (slices as usize, resolution) + let start_date = data.start_date.unwrap_or(Utc::now() - Duration::weeks(2)); + let end_date = data.end_date.unwrap_or(Utc::now()); + + // Convert String list to list of ProjectIds or VersionIds + // - Filter out unauthorized projects/versions + // - If no project_ids or version_ids are provided, we default to all projects the user has access to + let project_ids = + filter_allowed_ids(project_ids, user, &pool, &redis, None).await?; + + // Get the countries + let countries = crate::clickhouse::fetch_countries_downloads( + project_ids.unwrap_or_default(), + start_date, + end_date, + clickhouse.into_inner(), + ) + .await?; + + let mut hm = HashMap::new(); + for views in countries { + let id_string = to_base62(views.id); + if !hm.contains_key(&id_string) { + hm.insert(id_string.clone(), HashMap::new()); } - TimeRangeResolution::Minutes(resolution_minutes) => { - let resolution_minutes = i64::try_from(resolution_minutes.get()) - .map_err(|_| { - ApiError::InvalidInput( - "Resolution must fit into a `i64`".into(), - ) - })?; - let resolution = TimeDelta::try_minutes(resolution_minutes) - .ok_or_else(|| { - ApiError::InvalidInput("Resolution overflow".into()) - })?; - - let num_slices = - full_time_range.as_seconds_f64() / resolution.as_seconds_f64(); - - (num_slices as usize, resolution) + if let Some(hm) = hm.get_mut(&id_string) { + hm.insert(views.country, views.total); } - }; - - if num_time_slices > MAX_TIME_SLICES { - return Err(ApiError::InvalidInput(format!( - "Resolution is too fine or range is too large - maximum of {MAX_TIME_SLICES} time slices, was {num_time_slices}" - ))); - } - if resolution < MIN_RESOLUTION { - return Err(ApiError::InvalidInput(format!( - "Resolution must be at least {MIN_RESOLUTION}, was {resolution}", - ))); } - let mut time_slices = vec![TimeSlice::default(); num_time_slices]; - - // TODO fetch from req - let project_ids = - DBUser::get_projects(user.id.into(), &**pool, &redis).await?; + let hm: HashMap> = hm + .into_iter() + .map(|(key, value)| (key, condense_countries(value))) + .collect(); - let project_ids = - filter_allowed_project_ids(&project_ids, &user, &pool, &redis).await?; + Ok(HttpResponse::Ok().json(hm)) +} - let mut query_clickhouse_cx = QueryClickhouseContext { - clickhouse: &clickhouse, - req: &req, - time_slices: &mut time_slices, - project_ids: &project_ids, - }; +/// Get country data for a set of projects or versions +/// Data is returned as a hashmap of project/version ids to a hashmap of coutnry to views. +/// Unknown countries are labeled "". +/// This is usuable to see significant performing countries per project +/// eg: +/// { +/// "4N1tEhnO": { +/// "CAN": 56165 +/// } +///} +/// Either a list of project_ids or version_ids can be used, but not both. Unauthorized projects/versions will be filtered out. +/// For this endpoint, provided dates are a range to aggregate over, not specific days to fetch +pub async fn countries_views_get( + req: HttpRequest, + clickhouse: web::Data, + data: web::Query, + session_queue: web::Data, + pool: web::Data, + redis: web::Data, +) -> Result { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::ANALYTICS, + ) + .await + .map(|x| x.1)?; - if let Some(metrics) = &req.return_metrics.project_views { - use ProjectViewsField as F; - let uses = |field| metrics.bucket_by.contains(&field); - - query_clickhouse::( - &mut query_clickhouse_cx, - query::VIEWS, - &[ - ("use_project_id", uses(F::ProjectId)), - ("use_domain", uses(F::Domain)), - ("use_site_path", uses(F::SitePath)), - ("use_monetized", uses(F::Monetized)), - ("use_country", uses(F::Country)), - ], - |row| row.bucket, - |row| { - let country = if uses(F::Country) { - Some(condense_country(row.country, row.views)) - } else { - None - }; - AnalyticsData::Project(ProjectAnalytics { - source_project: row.project_id.into(), - metrics: ProjectMetrics::Views(ProjectViews { - domain: none_if_empty(row.domain), - site_path: none_if_empty(row.site_path), - monetized: match row.monetized { - 0 => Some(false), - 1 => Some(true), - _ => None, - }, - country, - views: row.views, - }), - }) - }, - ) - .await?; - } + let project_ids = data + .project_ids + .as_ref() + .map(|ids| serde_json::from_str::>(ids)) + .transpose()?; - if let Some(metrics) = &req.return_metrics.project_downloads { - use ProjectDownloadsField as F; - let uses = |field| metrics.bucket_by.contains(&field); - - query_clickhouse::( - &mut query_clickhouse_cx, - query::DOWNLOADS, - &[ - ("use_project_id", uses(F::ProjectId)), - ("use_domain", uses(F::Domain)), - ("use_site_path", uses(F::SitePath)), - ("use_version_id", uses(F::VersionId)), - ("use_country", uses(F::Country)), - ], - |row| row.bucket, - |row| { - let country = if uses(F::Country) { - Some(condense_country(row.country, row.downloads)) - } else { - None - }; - AnalyticsData::Project(ProjectAnalytics { - source_project: row.project_id.into(), - metrics: ProjectMetrics::Downloads(ProjectDownloads { - domain: none_if_empty(row.domain), - site_path: none_if_empty(row.site_path), - version_id: none_if_zero_version_id(row.version_id), - country, - downloads: row.downloads, - }), - }) - }, - ) - .await?; - } + let start_date = data.start_date.unwrap_or(Utc::now() - Duration::weeks(2)); + let end_date = data.end_date.unwrap_or(Utc::now()); - if let Some(metrics) = &req.return_metrics.project_playtime { - use ProjectPlaytimeField as F; - let uses = |field| metrics.bucket_by.contains(&field); - - query_clickhouse::( - &mut query_clickhouse_cx, - query::PLAYTIME, - &[ - ("use_project_id", uses(F::ProjectId)), - ("use_version_id", uses(F::VersionId)), - ("use_loader", uses(F::Loader)), - ("use_game_version", uses(F::GameVersion)), - ], - |row| row.bucket, - |row| { - AnalyticsData::Project(ProjectAnalytics { - source_project: row.project_id.into(), - metrics: ProjectMetrics::Playtime(ProjectPlaytime { - version_id: none_if_zero_version_id(row.version_id), - loader: none_if_empty(row.loader), - game_version: none_if_empty(row.game_version), - seconds: row.seconds, - }), - }) - }, - ) - .await?; - } + // Convert String list to list of ProjectIds or VersionIds + // - Filter out unauthorized projects/versions + // - If no project_ids or version_ids are provided, we default to all projects the user has access to + let project_ids = + filter_allowed_ids(project_ids, user, &pool, &redis, None).await?; + + // Get the countries + let countries = crate::clickhouse::fetch_countries_views( + project_ids.unwrap_or_default(), + start_date, + end_date, + clickhouse.into_inner(), + ) + .await?; - if req.return_metrics.project_revenue.is_some() { - if !scopes.contains(Scopes::PAYOUTS_READ) { - return Err(AuthenticationError::InvalidCredentials.into()); + let mut hm = HashMap::new(); + for views in countries { + let id_string = to_base62(views.id); + if !hm.contains_key(&id_string) { + hm.insert(id_string.clone(), HashMap::new()); } - - let mut rows = sqlx::query!( - "SELECT - WIDTH_BUCKET( - EXTRACT(EPOCH FROM created)::bigint, - EXTRACT(EPOCH FROM $1::timestamp with time zone AT TIME ZONE 'UTC')::bigint, - EXTRACT(EPOCH FROM $2::timestamp with time zone AT TIME ZONE 'UTC')::bigint, - $3::integer - ) AS bucket, - COALESCE(mod_id, 0) AS mod_id, - SUM(amount) amount_sum - FROM payouts_values - WHERE - user_id = $4 - AND created BETWEEN $1 AND $2 - GROUP BY bucket, mod_id", - req.time_range.start, - req.time_range.end, - num_time_slices as i64, - DBUserId::from(user.id) as DBUserId, - ) - .fetch(&**pool); - while let Some(row) = rows.next().await.transpose()? { - let bucket = row.bucket.ok_or_else(|| { - ApiError::InvalidInput( - "bucket should be non-null - query bug!".into(), - ) - })?; - let bucket = usize::try_from(bucket).map_err(|_| { - ApiError::InvalidInput( - "bucket value {bucket} does not fit into `usize` - query bug!".into(), - ) - })?; - - if let Some(source_project) = - row.mod_id.map(DBProjectId).map(ProjectId::from) - && let Some(revenue) = row.amount_sum - { - add_to_time_slice( - &mut time_slices, - bucket, - AnalyticsData::Project(ProjectAnalytics { - source_project, - metrics: ProjectMetrics::Revenue(ProjectRevenue { - revenue, - }), - }), - )?; - } + if let Some(hm) = hm.get_mut(&id_string) { + hm.insert(views.country, views.total); } } - Ok(web::Json(GetResponse(time_slices))) -} - -fn none_if_empty(s: String) -> Option { - if s.is_empty() { None } else { Some(s) } -} + let hm: HashMap> = hm + .into_iter() + .map(|(key, value)| (key, condense_countries(value))) + .collect(); -fn none_if_zero_version_id(v: DBVersionId) -> Option { - if v.0 == 0 { None } else { Some(v.into()) } + Ok(HttpResponse::Ok().json(hm)) } -fn condense_country(country: String, count: u64) -> String { - // Every country under '50' (view or downloads) should be condensed into 'XX' - if count < 50 { - "XX".to_string() - } else { - country +fn condense_countries(countries: HashMap) -> HashMap { + // Every country under '15' (view or downloads) should be condensed into 'XX' + let mut hm = HashMap::new(); + for (mut country, count) in countries { + if count < 50 { + country = "XX".to_string(); + } + if !hm.contains_key(&country) { + hm.insert(country.to_string(), 0); + } + if let Some(hm) = hm.get_mut(&country) { + *hm += count; + } } + hm } -struct QueryClickhouseContext<'a> { - clickhouse: &'a clickhouse::Client, - req: &'a GetRequest, - time_slices: &'a mut [TimeSlice], - project_ids: &'a [DBProjectId], -} - -async fn query_clickhouse( - cx: &mut QueryClickhouseContext<'_>, - query: &str, - use_columns: &[(&str, bool)], - row_get_bucket: impl Fn(&Row) -> u64, - row_to_analytics: impl Fn(Row) -> AnalyticsData, -) -> Result<(), ApiError> -where - Row: clickhouse::Row + serde::de::DeserializeOwned + std::fmt::Debug, -{ - let mut query = cx - .clickhouse - .query(query) - .param("time_range_start", cx.req.time_range.start.timestamp()) - .param("time_range_end", cx.req.time_range.end.timestamp()) - .param("time_slices", cx.time_slices.len()) - .param("project_ids", cx.project_ids); - for (param_name, used) in use_columns { - query = query.param(param_name, used) - } - let mut cursor = query.fetch::()?; - - while let Some(row) = cursor.next().await? { - let bucket = row_get_bucket(&row) as usize; - add_to_time_slice(cx.time_slices, bucket, row_to_analytics(row))?; +async fn filter_allowed_ids( + mut project_ids: Option>, + user: crate::models::users::User, + pool: &web::Data, + redis: &RedisPool, + remove_defaults: Option, +) -> Result>, ApiError> { + // If no project_ids or version_ids are provided, we default to all projects the user has *public* access to + if project_ids.is_none() && !remove_defaults.unwrap_or(false) { + project_ids = Some( + user_item::DBUser::get_projects(user.id.into(), &***pool, redis) + .await? + .into_iter() + .map(|x| ProjectId::from(x).to_string()) + .collect(), + ); } - Ok(()) -} - -fn add_to_time_slice( - time_slices: &mut [TimeSlice], - bucket: usize, - data: AnalyticsData, -) -> Result<(), ApiError> { - // row.recorded < time_range_start => bucket = 0 - // row.recorded >= time_range_end => bucket = num_time_slices - // (note: this is out of range of `time_slices`!) - let Some(bucket) = bucket.checked_sub(1) else { - return Ok(()); - }; - - let num_time_slices = time_slices.len(); - let slice = time_slices.get_mut(bucket).ok_or_else(|| { - ApiError::InvalidInput( - format!("bucket {bucket} returned by query out of range for {num_time_slices} - query bug!") + // Convert String list to list of ProjectIds or VersionIds + // - Filter out unauthorized projects/versions + let project_ids = if let Some(project_strings) = project_ids { + let projects_data = database::models::DBProject::get_many( + &project_strings, + &***pool, + redis, ) - })?; - - slice.0.push(data); - Ok(()) -} - -async fn filter_allowed_project_ids( - project_ids: &[DBProjectId], - user: &crate::models::users::User, - pool: &PgPool, - redis: &RedisPool, -) -> Result, ApiError> { - let projects = DBProject::get_many_ids(project_ids, pool, redis).await?; - - let team_ids = projects - .iter() - .map(|x| x.inner.team_id) - .collect::>(); - let team_members = database::models::DBTeamMember::get_from_team_full_many( - &team_ids, pool, redis, - ) - .await?; - - let organization_ids = projects - .iter() - .filter_map(|x| x.inner.organization_id) - .collect::>(); - let organizations = database::models::DBOrganization::get_many_ids( - &organization_ids, - pool, - redis, - ) - .await?; + .await?; - let organization_team_ids = organizations - .iter() - .map(|x| x.team_id) - .collect::>(); - let organization_team_members = - database::models::DBTeamMember::get_from_team_full_many( - &organization_team_ids, - pool, + let team_ids = projects_data + .iter() + .map(|x| x.inner.team_id) + .collect::>(); + let team_members = + database::models::DBTeamMember::get_from_team_full_many( + &team_ids, &***pool, redis, + ) + .await?; + + let organization_ids = projects_data + .iter() + .filter_map(|x| x.inner.organization_id) + .collect::>(); + let organizations = database::models::DBOrganization::get_many_ids( + &organization_ids, + &***pool, redis, ) .await?; - Ok(projects - .into_iter() - .filter(|project| { - let team_member = team_members.iter().find(|x| { - x.team_id == project.inner.team_id - && x.user_id == user.id.into() - }); - - let organization = project - .inner - .organization_id - .and_then(|oid| organizations.iter().find(|x| x.id == oid)); - - let organization_team_member = - if let Some(organization) = organization { - organization_team_members.iter().find(|x| { - x.team_id == organization.team_id - && x.user_id == user.id.into() - }) - } else { - None - }; - - let permissions = ProjectPermissions::get_permissions_by_role( - &user.role, - &team_member.cloned(), - &organization_team_member.cloned(), + let organization_team_ids = organizations + .iter() + .map(|x| x.team_id) + .collect::>(); + let organization_team_members = + database::models::DBTeamMember::get_from_team_full_many( + &organization_team_ids, + &***pool, + redis, ) - .unwrap_or_default(); + .await?; + + let ids = projects_data + .into_iter() + .filter(|project| { + let team_member = team_members.iter().find(|x| { + x.team_id == project.inner.team_id + && x.user_id == user.id.into() + }); + + let organization = project + .inner + .organization_id + .and_then(|oid| organizations.iter().find(|x| x.id == oid)); + + let organization_team_member = + if let Some(organization) = organization { + organization_team_members.iter().find(|x| { + x.team_id == organization.team_id + && x.user_id == user.id.into() + }) + } else { + None + }; + + let permissions = ProjectPermissions::get_permissions_by_role( + &user.role, + &team_member.cloned(), + &organization_team_member.cloned(), + ) + .unwrap_or_default(); - permissions.contains(ProjectPermissions::VIEW_ANALYTICS) - }) - .map(|project| project.inner.id) - .collect::>()) -} + permissions.contains(ProjectPermissions::VIEW_ANALYTICS) + }) + .map(|x| x.inner.id.into()) + .collect::>(); -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - #[test] - fn response_format() { - let test_project_1 = ProjectId(123); - let test_project_2 = ProjectId(456); - let test_project_3 = ProjectId(789); - - let src = GetResponse(vec![ - TimeSlice(vec![ - AnalyticsData::Project(ProjectAnalytics { - source_project: test_project_1, - metrics: ProjectMetrics::Views(ProjectViews { - domain: Some("youtube.com".into()), - views: 100, - ..Default::default() - }), - }), - AnalyticsData::Project(ProjectAnalytics { - source_project: test_project_2, - metrics: ProjectMetrics::Downloads(ProjectDownloads { - domain: Some("discord.com".into()), - downloads: 150, - ..Default::default() - }), - }), - ]), - TimeSlice(vec![AnalyticsData::Project(ProjectAnalytics { - source_project: test_project_3, - metrics: ProjectMetrics::Revenue(ProjectRevenue { - revenue: Decimal::new(20000, 2), - }), - })]), - ]); - let target = json!([ - [ - { - "source_project": test_project_1.to_string(), - "metric_kind": "views", - "domain": "youtube.com", - "views": 100, - }, - { - "source_project": test_project_2.to_string(), - "metric_kind": "downloads", - "domain": "discord.com", - "downloads": 150, - } - ], - [ - { - "source_project": test_project_3.to_string(), - "metric_kind": "revenue", - "revenue": "200.00", - } - ] - ]); - - assert_eq!(serde_json::to_value(src).unwrap(), target); - } + Some(ids) + } else { + None + }; + // Only one of project_ids or version_ids will be Some + Ok(project_ids) } diff --git a/apps/labrinth/tests/analytics.rs b/apps/labrinth/tests/analytics.rs index 1380e346f3..217f0e7b15 100644 --- a/apps/labrinth/tests/analytics.rs +++ b/apps/labrinth/tests/analytics.rs @@ -1,23 +1,16 @@ -use actix_web::test; use ariadne::ids::base62_impl::parse_base62; use chrono::{DateTime, Duration, Utc}; use common::permissions::PermissionsTest; use common::permissions::PermissionsTestContext; use common::{ - api_common::{Api, AppendsOptionalPat}, api_v3::ApiV3, database::*, environment::{TestEnvironment, with_test_environment}, }; +use itertools::Itertools; use labrinth::models::teams::ProjectPermissions; use labrinth::queue::payouts; - -use labrinth::routes::v3::analytics_get::{ - AnalyticsData, GetRequest, Metrics, ReturnMetrics, TimeRange, - TimeRangeResolution, -}; -use rust_decimal::Decimal; -use std::num::NonZeroU64; +use rust_decimal::{Decimal, prelude::ToPrimitive}; pub mod common; @@ -78,123 +71,88 @@ pub async fn analytics_revenue() { .unwrap(); transaction.commit().await.unwrap(); + let day = 86400; + // Test analytics endpoint with default values // - all time points in the last 2 weeks // - 1 day resolution - let time_range = TimeRange { - start: Utc::now() - Duration::days(14), - end: Utc::now(), - resolution: TimeRangeResolution::Slices( - NonZeroU64::new(14).unwrap(), - ), - }; - - let return_metrics = ReturnMetrics { - project_revenue: Some(Metrics { bucket_by: vec![] }), - ..Default::default() - }; - - let request = GetRequest { - time_range, - return_metrics: ReturnMetrics { - project_revenue: Some(Metrics { bucket_by: vec![] }), - ..Default::default() - }, - }; - - let response = - api.get_analytics_revenue_new(request, USER_USER_PAT).await; - - // GetResponse is a Vec, each TimeSlice contains Vec - // For now, just check that we get some response - assert!(!response.0.is_empty()); - - // Find our project in the response - for time_slice in &response.0 { - if let Some(analytics_data) = time_slice.0.first() { - let AnalyticsData::Project(_project_analytics) = - analytics_data; - break; - } - } - - // GetResponse is a Vec, each TimeSlice contains Vec - // For now, just check that we get some response - assert!(!response.0.is_empty()); - - // Check that we have some project data (not specific to our project) - let mut found_any_project = false; - for time_slice in &response.0 { - if let Some(analytics_data) = time_slice.0.first() { - let AnalyticsData::Project(_project_analytics) = - analytics_data; - found_any_project = true; - break; - } - if found_any_project { - break; - } - } - assert!( - found_any_project, - "Should find some project in the analytics response" + let analytics = api + .get_analytics_revenue_deserialized( + vec![&alpha_project_id], + false, + None, + None, + None, + USER_USER_PAT, + ) + .await; + assert_eq!(analytics.len(), 1); // 1 project + let project_analytics = &analytics[&alpha_project_id]; + assert_eq!(project_analytics.len(), 8); // 1 days cut off, and 2 points take place on the same day. note that the day exactly 14 days ago is included + // sorted_by_key, values in the order of smallest to largest key + let (sorted_keys, sorted_by_key): (Vec, Vec) = + project_analytics + .iter() + .sorted_by_key(|(k, _)| *k) + .rev() + .unzip(); + assert_eq!( + vec![100.1, 101.0, 200.0, 311.0, 400.0, 526.0, 633.0, 800.0], + to_f64_vec_rounded_up(sorted_by_key) ); + // Ensure that the keys are in multiples of 1 day + for k in sorted_keys { + assert_eq!(k % day, 0); + } // Test analytics with last 900 days to include all data // keep resolution at default - let time_range = TimeRange { - start: Utc::now() - Duration::days(801), - end: Utc::now(), - resolution: TimeRangeResolution::Slices( - NonZeroU64::new(900).unwrap(), - ), - }; - - let request = GetRequest { - time_range, - return_metrics, - }; - - let response = - api.get_analytics_revenue_new(request, USER_USER_PAT).await; - - // Again, just check that we get some response - assert!(!response.0.is_empty()); - - // Find our project in the response - for time_slice in &response.0 { - if let Some(analytics_data) = time_slice.0.first() { - let AnalyticsData::Project(_project_analytics) = - analytics_data; - break; - } - } - - // Again, just check that we get some response - assert!(!response.0.is_empty()); - - // Check that we have some project data (not specific to our project) - let mut found_any_project = false; - for time_slice in &response.0 { - if let Some(analytics_data) = time_slice.0.first() { - let AnalyticsData::Project(_project_analytics) = - analytics_data; - found_any_project = true; - break; - } - if found_any_project { - break; - } - } - assert!( - found_any_project, - "Should find some project in the analytics response" + let analytics = api + .get_analytics_revenue_deserialized( + vec![&alpha_project_id], + false, + Some(Utc::now() - Duration::days(801)), + None, + None, + USER_USER_PAT, + ) + .await; + let project_analytics = &analytics[&alpha_project_id]; + assert_eq!(project_analytics.len(), 9); // and 2 points take place on the same day + let (sorted_keys, sorted_by_key): (Vec, Vec) = + project_analytics + .iter() + .sorted_by_key(|(k, _)| *k) + .rev() + .unzip(); + assert_eq!( + vec![ + 100.1, 101.0, 200.0, 311.0, 400.0, 526.0, 633.0, 800.0, + 800.0 + ], + to_f64_vec_rounded_up(sorted_by_key) ); + for k in sorted_keys { + assert_eq!(k % day, 0); + } }, ) .await; } +fn to_f64_rounded_up(d: Decimal) -> f64 { + d.round_dp_with_strategy( + 1, + rust_decimal::RoundingStrategy::MidpointAwayFromZero, + ) + .to_f64() + .unwrap() +} + +fn to_f64_vec_rounded_up(d: Vec) -> Vec { + d.into_iter().map(to_f64_rounded_up).collect_vec() +} + #[actix_rt::test] pub async fn permissions_analytics_revenue() { with_test_environment( @@ -212,48 +170,31 @@ pub async fn permissions_analytics_revenue() { // first, do check with a project let req_gen = |ctx: PermissionsTestContext| async move { - // TODO: when we add filters, make sure this only returns the - // projects with this ID - let _project_id = ctx.project_id.unwrap(); - let time_range = TimeRange { - start: Utc::now() - Duration::days(14), - end: Utc::now(), - resolution: TimeRangeResolution::Slices( - NonZeroU64::new(14).unwrap(), - ), - }; - let return_metrics = ReturnMetrics { - project_revenue: Some(Metrics { bucket_by: vec![] }), - ..Default::default() - }; - let request = GetRequest { - time_range, - return_metrics, - }; - // Return a ServiceResponse for the permissions test - let req = test::TestRequest::post() - .uri("/v3/analytics") - .set_json(request) - .append_pat(ctx.test_pat.as_deref()) - .to_request(); - api.call(req).await + let project_id = ctx.project_id.unwrap(); + let ids_or_slugs = vec![project_id.as_str()]; + api.get_analytics_revenue( + ids_or_slugs, + false, + None, + None, + Some(5), + ctx.test_pat.as_deref(), + ) + .await }; PermissionsTest::new(&test_env) .with_failure_codes(vec![200, 401]) .with_200_json_checks( // On failure, should have 0 projects returned - |_value: &serde_json::Value| { - // TODO: when we add filters, make sure this is empty - // but for now since we don't filter on project IDs, - // just check that it's a non-error - // let value = value.as_array().unwrap(); - // assert_eq!(value.len(), 0); + |value: &serde_json::Value| { + let value = value.as_object().unwrap(); + assert_eq!(value.len(), 0); }, // On success, should have 1 project returned |value: &serde_json::Value| { - let value = value.as_array().unwrap(); - assert!(!value.is_empty()); + let value = value.as_object().unwrap(); + assert_eq!(value.len(), 1); }, ) .simple_project_permissions_test(view_analytics, req_gen) @@ -263,32 +204,18 @@ pub async fn permissions_analytics_revenue() { // Now with a version // Need to use alpha let req_gen = |ctx: PermissionsTestContext| { - // TODO: when we add filters, make sure this only returns the - // projects with this ID - let _alpha_version_id = alpha_version_id.clone(); + let alpha_version_id = alpha_version_id.clone(); async move { - let time_range = TimeRange { - start: Utc::now() - Duration::days(14), - end: Utc::now(), - resolution: TimeRangeResolution::Slices( - NonZeroU64::new(14).unwrap(), - ), - }; - let return_metrics = ReturnMetrics { - project_revenue: Some(Metrics { bucket_by: vec![] }), - ..Default::default() - }; - let request = GetRequest { - time_range, - return_metrics, - }; - // Return a ServiceResponse for the permissions test - let req = test::TestRequest::post() - .uri("/v3/analytics") - .set_json(request) - .append_pat(ctx.test_pat.as_deref()) - .to_request(); - api.call(req).await + let ids_or_slugs = vec![alpha_version_id.as_str()]; + api.get_analytics_revenue( + ids_or_slugs, + true, + None, + None, + Some(5), + ctx.test_pat.as_deref(), + ) + .await } }; @@ -298,20 +225,14 @@ pub async fn permissions_analytics_revenue() { .with_user(FRIEND_USER_ID, FRIEND_USER_PAT, true) .with_200_json_checks( // On failure, should have 0 versions returned - |_value: &serde_json::Value| { - // TODO: when we add filters, make sure this is empty - // but for now since we don't filter on project IDs, - // just check that it's a non-error - // let value = value.as_array().unwrap(); - // assert_eq!(value.len(), 0); + |value: &serde_json::Value| { + let value = value.as_object().unwrap(); + assert_eq!(value.len(), 0); }, // On success, should have 1 versions returned - |_value: &serde_json::Value| { - // TODO: when we add filters, make sure this is empty - // but for now since we don't filter on project IDs, - // just check that it's a non-error - // let value = value.as_array().unwrap(); - // assert_eq!(value.len(), 0); + |value: &serde_json::Value| { + let value = value.as_object().unwrap(); + assert_eq!(value.len(), 0); }, ) .simple_project_permissions_test(view_analytics, req_gen) diff --git a/apps/labrinth/tests/common/api_v3/project.rs b/apps/labrinth/tests/common/api_v3/project.rs index 56e758c994..0513206f8d 100644 --- a/apps/labrinth/tests/common/api_v3/project.rs +++ b/apps/labrinth/tests/common/api_v3/project.rs @@ -7,14 +7,13 @@ use actix_web::{ }; use async_trait::async_trait; use bytes::Bytes; +use chrono::{DateTime, Utc}; use labrinth::{ models::{organizations::Organization, projects::Project}, - routes::v3::analytics_get::{ - GetRequest, GetResponse, Metrics, ReturnMetrics, TimeRange, - }, search::SearchResults, util::actix::AppendsMultipart, }; +use rust_decimal::Decimal; use serde_json::json; use crate::{ @@ -571,42 +570,70 @@ impl ApiV3 { pub async fn get_analytics_revenue( &self, - time_range: TimeRange, + id_or_slugs: Vec<&str>, + ids_are_version_ids: bool, + start_date: Option>, + end_date: Option>, + resolution_minutes: Option, pat: Option<&str>, - ) -> GetResponse { - let req = GetRequest { - time_range, - return_metrics: ReturnMetrics { - project_revenue: Some(Metrics { - bucket_by: Vec::new(), - }), - ..Default::default() - }, + ) -> ServiceResponse { + let pv_string = if ids_are_version_ids { + let version_string: String = + serde_json::to_string(&id_or_slugs).unwrap(); + let version_string = urlencoding::encode(&version_string); + format!("version_ids={version_string}") + } else { + let projects_string: String = + serde_json::to_string(&id_or_slugs).unwrap(); + let projects_string = urlencoding::encode(&projects_string); + format!("project_ids={projects_string}") }; - let req = test::TestRequest::post() - .uri("/v3/analytics") - .set_json(req) + let mut extra_args = String::new(); + if let Some(start_date) = start_date { + let start_date = start_date.to_rfc3339(); + // let start_date = serde_json::to_string(&start_date).unwrap(); + let start_date = urlencoding::encode(&start_date); + write!(&mut extra_args, "&start_date={start_date}").unwrap(); + } + if let Some(end_date) = end_date { + let end_date = end_date.to_rfc3339(); + // let end_date = serde_json::to_string(&end_date).unwrap(); + let end_date = urlencoding::encode(&end_date); + write!(&mut extra_args, "&end_date={end_date}").unwrap(); + } + if let Some(resolution_minutes) = resolution_minutes { + write!(&mut extra_args, "&resolution_minutes={resolution_minutes}") + .unwrap(); + } + + let req = test::TestRequest::get() + .uri(&format!("/v3/analytics/revenue?{pv_string}{extra_args}",)) .append_pat(pat) .to_request(); - let resp = self.call(req).await; - assert_status!(&resp, StatusCode::OK); - test::read_body_json(resp).await + self.call(req).await } - pub async fn get_analytics_revenue_new( + pub async fn get_analytics_revenue_deserialized( &self, - request: GetRequest, + id_or_slugs: Vec<&str>, + ids_are_version_ids: bool, + start_date: Option>, + end_date: Option>, + resolution_minutes: Option, pat: Option<&str>, - ) -> GetResponse { - let req = test::TestRequest::post() - .uri("/v3/analytics") - .set_json(request) - .append_pat(pat) - .to_request(); - - let resp = self.call(req).await; + ) -> HashMap> { + let resp = self + .get_analytics_revenue( + id_or_slugs, + ids_are_version_ids, + start_date, + end_date, + resolution_minutes, + pat, + ) + .await; assert_status!(&resp, StatusCode::OK); test::read_body_json(resp).await }