Skip unnecessary steps when downloading an unchanged mod list

In SplashMixin, skip downloading the chunks and updating them into the
IndexedDB if the index file indicates the packages haven't changed. If
downloading the index file itself fails, try to load earlier mod list
from the IndexedDB like it did earlier.

In UtilityMixin, skip the whole process if the index file indicates the
packages haven't changed, as at this point we already have the mod list
loaded in to Vuex. If loading the index file itself fails, the error is
caught by the caller, which handles retries as it did earlier.

Author: anttimaki

src/components/mixins/SplashMixin.vue

@@ -4,7 +4,7 @@ import Component from 'vue-class-component';
 
 import R2Error from '../../model/errors/R2Error';
 import RequestItem from '../../model/requests/RequestItem';
-import type { PackageListChunks } from '../../store/modules/TsModsModule';
+import type { PackageListChunks, PackageListIndex } from '../../store/modules/TsModsModule';
 
 
 @Component
@@ -51,16 +51,19 @@ export default class SplashMixin extends Vue {
 
     // Get the list of Thunderstore mods from API or local cache.
     async getThunderstoreMods() {
-        const packageListChunks = await this.fetchPackageListChunksIfUpdated();
+        const packageListIndex = await this.fetchPackageListIndex();
+        const packageListChunks = await this.fetchPackageListChunksIfUpdated(packageListIndex);
         this.getRequestItem('ThunderstoreDownload').setProgress(100);
-        await this.writeModsToPersistentCacheIfUpdated(packageListChunks);
+        await this.writeModsToPersistentCacheIfUpdated(packageListIndex, packageListChunks);
         const isModListLoaded = await this.readModsToVuex();
 
         // To proceed, the loading of the mod list should result in a non-empty list.
         // Empty list is allowed if that's actually what the API returned.
+        // API wasn't queried at all if we already had the latest index chunk.
         const modListHasMods = this.$store.state.tsMods.mods.length;
         const apiReturnedEmptyList = packageListChunks && packageListChunks[0].length === 0;
-        if (isModListLoaded && (modListHasMods || apiReturnedEmptyList)) {
+        const apiWasNotQueried = packageListIndex && packageListIndex.isLatest;
+        if (isModListLoaded && (modListHasMods || apiReturnedEmptyList || apiWasNotQueried)) {
             await this.moveToNextScreen();
         } else {
             this.heroTitle = 'Failed to get the list of online mods';
@@ -70,18 +73,41 @@ export default class SplashMixin extends Vue {
     }
 
     /***
-     * Load the package list in chunks.
+     * Query Thunderstore API for the URLs pointing to parts of the package list.
      * Fails silently to fallback reading the old values from the IndexedDB cache.
      */
-    async fetchPackageListChunksIfUpdated(): Promise<PackageListChunks|undefined> {
+    async fetchPackageListIndex(): Promise<PackageListIndex|undefined> {
+        this.loadingText = 'Checking for mod list updates from Thunderstore';
 
-        const showProgress = (progress: number) => {
+        try {
+            return await this.$store.dispatch('tsMods/fetchPackageListIndex');
+        } catch (e) {
+            console.error('SplashMixin failed to fetch mod list index from API.', e);
+            return undefined;
+        }
+    }
+
+    /***
+     * Load the package list in chunks pointed out by the packageListIndex.
+     * This step is skipped if we can't or don't need to load the chunks.
+     * Fails silently to fallback reading the old values from the IndexedDB cache.
+     */
+    async fetchPackageListChunksIfUpdated(packageListIndex?: PackageListIndex): Promise<PackageListChunks|undefined> {
+        // Skip loading chunks if loading index failed, or if we already have the latest data.
+        if (!packageListIndex || packageListIndex.isLatest) {
+            return undefined;
+        }
+
+        const progressCallback = (progress: number) => {
             this.loadingText = 'Loading latest mod list from Thunderstore';
             this.getRequestItem('ThunderstoreDownload').setProgress(progress);
         };
 
         try {
-            return await this.$store.dispatch('tsMods/fetchPackageListChunks', showProgress);
+            return await this.$store.dispatch(
+                'tsMods/fetchPackageListChunks',
+                {chunkUrls: packageListIndex.content, progressCallback}
+            );
         } catch (e) {
             console.error('SplashMixin failed to fetch mod list from API.', e);
             return undefined;
@@ -90,15 +116,18 @@ export default class SplashMixin extends Vue {
 
     /***
      * Update a fresh package list to the IndexedDB cache.
-     * Done only if the package list was loaded successfully.
+     * Done only if there was a fresh list to load and it was loaded successfully.
      * Fails silently to fallback reading the old values from the IndexedDB cache.
      */
-    async writeModsToPersistentCacheIfUpdated(packageListChunks?: PackageListChunks) {
-        if (packageListChunks) {
+    async writeModsToPersistentCacheIfUpdated(packageListIndex?: PackageListIndex, packageListChunks?: PackageListChunks) {
+        if (packageListIndex && packageListChunks) {
             this.loadingText = 'Storing the mod list into local cache';
 
             try {
-                await this.$store.dispatch('tsMods/updatePersistentCache', packageListChunks)
+                await this.$store.dispatch(
+                    'tsMods/updatePersistentCache',
+                    {indexHash: packageListIndex.hash, chunks: packageListChunks}
+                );
             } catch (e) {
                 console.error('SplashMixin failed to cache mod list locally.', e);
             }
@@ -117,12 +146,12 @@ export default class SplashMixin extends Vue {
      * queried from the API successfully or not. This also handles the type
      * casting, since mod manager expects the data to be formatted into objects.
      *
+     * Failure at this point is no longer silently ignored, instead an error
+     * modal is shown.
+     *
      * Return value is used to tell whether Vuex might contain an empty list
      * after calling this because there was an error, or because the package
      * list is actually empty.
-     *
-     * Failure at this point is no longer silently ignored, instead an error
-     * modal is shown.
      */
     async readModsToVuex(): Promise<boolean> {
         let isModListLoaded = false;

src/components/mixins/UtilityMixin.vue

@@ -4,6 +4,7 @@ import Component from 'vue-class-component';
 
 import R2Error from '../../model/errors/R2Error';
 import CdnProvider from '../../providers/generic/connection/CdnProvider';
+import { PackageListIndex } from '../../store/modules/TsModsModule';
 
 @Component
 export default class UtilityMixin extends Vue {
@@ -15,8 +16,20 @@ export default class UtilityMixin extends Vue {
     }
 
     async refreshThunderstoreModList() {
-        const packageListChunks = await this.$store.dispatch('tsMods/fetchPackageListChunks');
-        await this.$store.dispatch("tsMods/updatePersistentCache", packageListChunks);
+        const packageListIndex: PackageListIndex = await this.$store.dispatch("tsMods/fetchPackageListIndex");
+
+        if (packageListIndex.isLatest) {
+            return;
+        }
+
+        const packageListChunks = await this.$store.dispatch(
+            "tsMods/fetchPackageListChunks",
+            {chunkUrls: packageListIndex.content},
+        );
+        await this.$store.dispatch(
+            "tsMods/updatePersistentCache",
+            {chunks: packageListChunks, indexHash: packageListIndex.hash},
+        );
         await this.$store.dispatch("tsMods/updateMods");
         await this.$store.dispatch("profile/tryLoadModListFromDisk");
         await this.$store.dispatch("tsMods/prewarmCache");

src/store/modules/TsModsModule.ts

@@ -29,6 +29,7 @@ interface State {
 type ProgressCallback = (progress: number) => void;
 type PackageListChunk = {full_name: string}[];
 export type PackageListChunks = PackageListChunk[];
+export type PackageListIndex = {content: string[], hash: string, isLatest: boolean};
 
 function isPackageListChunk(value: unknown): value is PackageListChunk {
     return Array.isArray(value) && (
@@ -161,7 +162,7 @@ export const TsModsModule = {
     },
 
     actions: <ActionTree<State, RootState>>{
-        async _fetchPackageListIndex({rootState}): Promise<string[]> {
+        async fetchPackageListIndex({rootState}): Promise<PackageListIndex> {
             const indexUrl = rootState.activeGame.thunderstoreUrl;
             const index = await retry(() => fetchAndProcessBlobFile(indexUrl));
 
@@ -172,28 +173,29 @@ export const TsModsModule = {
                 throw new Error('Received empty chunk index from API');
             }
 
-            return index.content;
+            const community = rootState.activeGame.internalFolderName;
+            const isLatest = await PackageDb.isLatestPackageListIndex(community, index.hash);
+
+            return {...index, isLatest};
         },
 
         async fetchPackageListChunks(
-            {dispatch},
-            progressCallback?: ProgressCallback
+            {},
+            {chunkUrls, progressCallback}: {chunkUrls: string[], progressCallback?: ProgressCallback},
         ): Promise<PackageListChunks> {
-            const chunkIndex: string[] = await dispatch('_fetchPackageListIndex');
-
             // Count index as a chunk for progress bar purposes.
-            const chunkCount = chunkIndex.length + 1;
+            const chunkCount = chunkUrls.length + 1;
             let completed = 1;
             const updateProgress = () => progressCallback && progressCallback((completed / chunkCount) * 100);
             updateProgress();
 
             // Download chunks serially to avoid slow connections timing
             // out due to concurrent requests competing for the bandwidth.
             const chunks = [];
-            for (const [i, chunkUrl] of chunkIndex.entries()) {
+            for (const [i, chunkUrl] of chunkUrls.entries()) {
                 const {content: chunk} = await retry(() => fetchAndProcessBlobFile(chunkUrl))
 
-                if (chunkIndex.length > 1 && isEmptyArray(chunkIndex)) {
+                if (chunkUrls.length > 1 && isEmptyArray(chunk)) {
                     throw new Error(`Chunk #${i} in multichunk response was empty`);
                 } else if (!isPackageListChunk(chunk)) {
                     throw new Error(`Chunk #${i} was invalid format`);
@@ -232,17 +234,18 @@ export const TsModsModule = {
         /*** Save a mod list received from the Thunderstore API to IndexedDB */
         async updatePersistentCache(
             {dispatch, rootState, state},
-            packages: PackageListChunks
+            {chunks, indexHash}: {chunks: PackageListChunks, indexHash: string}
         ) {
             if (state.exclusions === undefined) {
                 await dispatch('updateExclusions');
             }
 
-            const filtered = packages.map((chunk) => chunk.filter(
+            const filtered = chunks.map((chunk) => chunk.filter(
                 (pkg) => !state.exclusions!.includes(pkg.full_name)
             ));
             const community = rootState.activeGame.internalFolderName;
             await PackageDb.updateFromApiResponse(community, filtered);
+            await PackageDb.setLatestPackageListIndex(community, indexHash);
         }
     }
 }