use IDBFS to persist data (#75)

Author: eagleoflqj

README.md

@@ -6,7 +6,7 @@ The project provides an npm package `fcitx5-js.tgz`, which powers derivative app
 
 Derivative | Note
 -|-
-[Fcitx5 Online](https://github.com/fcitx-contrib/fcitx5-online) | [Preview](https://fcitx-contrib.github.io/online/)
+[Fcitx5 Online](https://github.com/fcitx/fcitx5-online) | [Preview](https://fcitx-contrib.github.io/online/)
 [Fcitx5 Chrome](https://github.com/fcitx-contrib/fcitx5-chrome) |
 
 ## Build

page/Fcitx5.d.ts

@@ -39,14 +39,17 @@ interface FS {
   lstat: (path: string) => { mode: number }
   mkdir: (path: string) => void
   mkdirTree: (path: string) => void
+  mount: (type: any, opts: { autoPersist?: boolean }, mountpoint: string) => void
   readFile: {
     (path: string): Uint8Array
     (path: string, options: { encoding: 'utf8' }): string
   }
   readdir: (path: string) => string[]
   rmdir: (path: string) => void
+  symlink: (target: string, path: string) => void
+  syncfs: (populate: boolean, callback: (err: any) => void) => void
   unlink: (path: string) => void
-  writeFile: (path: string, data: Uint8Array) => void
+  writeFile: (path: string, data: Uint8Array | string) => void
 }
 
 type WASM_TYPE = 'void' | 'bool' | 'number' | 'string'
@@ -56,6 +59,7 @@ export interface EM_MODULE {
   locateFile: (file: string) => string
   onRuntimeInitialized: () => void
   FS: FS
+  IDBFS: any
 }
 
 export type SyncCallback = (path: string) => void
@@ -106,6 +110,7 @@ export interface FCITX {
   utf8Index2JS: (text: string, index: number) => number
   setNotificationCallback: (callback: NotificationCallback) => void
   notify: NotificationCallback
+  reset: () => Promise<any>
   Module: EM_MODULE
 }
 

page/fs.ts

@@ -1,12 +1,15 @@
 import type { AsyncCallback, SyncCallback } from './Fcitx5'
 import Module from './module'
 
+export const USR_MOUNT_POINT = '/backup/usr'
+export const HOME_MOUNT_POINT = '/home/web_user'
+
 export function lsDir(path: string) {
   const names = Module.FS.readdir(path)
   return names.filter(name => name !== '.' && name !== '..')
 }
 
-function traverseSync(preDirCallback: SyncCallback | undefined, fileCallback: SyncCallback, postDirCallback: SyncCallback | undefined) {
+export function traverseSync(preDirCallback: SyncCallback | undefined, fileCallback: SyncCallback, postDirCallback: SyncCallback | undefined) {
   async function closure(path: string) {
     const { mode } = Module.FS.lstat(path)
     if (Module.FS.isDir(mode)) {
@@ -49,3 +52,19 @@ export function traverseAsync(preDirCallback: AsyncCallback | undefined, fileCal
   }
   return closure
 }
+
+export function mount() {
+  // Don't mount /usr directly as it stores unnecessary files and is too slow.
+  Module.FS.mkdirTree(USR_MOUNT_POINT)
+  Module.FS.mount(Module.IDBFS, { autoPersist: true }, USR_MOUNT_POINT)
+  Module.FS.mount(Module.IDBFS, { autoPersist: true }, HOME_MOUNT_POINT)
+}
+
+export function reset() {
+  const { promise, resolve } = Promise.withResolvers<void>()
+  rmR(USR_MOUNT_POINT)
+  rmR(HOME_MOUNT_POINT)
+  // Manually trigger syncfs to ensure data is cleared.
+  Module.FS.syncfs(false, () => resolve())
+  return promise
+}

page/index.ts

@@ -6,21 +6,17 @@ import { getAddons, getConfig, setConfig } from './config'
 import { SERVICE_WORKER, WEB, WEB_WORKER } from './constant'
 import { hasTouch, isFirefox } from './context'
 import { blur, clickPanel, focus, isInputElement, redrawCaretAndPreeditUnderline } from './focus'
-import { rmR, traverseAsync } from './fs'
+import { mount, reset, rmR, traverseAsync } from './fs'
 import { currentInputMethod, getAllInputMethods, getInputMethods, getLanguageName, setCurrentInputMethod, setInputMethods } from './input-method'
 import { createKeyboard, sendEventToKeyboard } from './keyboard'
 import { jsKeyToFcitxString, keyEvent } from './keycode'
 import { getLocale } from './locale'
 import Module from './module'
-import { getInstalledPlugins, installPlugin, unzip } from './plugin'
+import { getInstalledPlugins, installPlugin, restorePlugins, unzip } from './plugin'
 import { utf8Index2JS } from './unicode'
 import { deployRimeInWorker } from './workerAPI'
 
-let res: (value: any) => void
-
-const fcitxReady = new Promise((resolve) => {
-  res = resolve
-})
+const { promise: fcitxReady, resolve } = Promise.withResolvers()
 
 let inputMethodsCallback = () => {}
 let statusAreaCallback = () => {}
@@ -148,6 +144,7 @@ globalThis.fcitx = {
   rmR,
   traverseAsync,
   deployRimeInWorker,
+  reset,
   // Private field that indicates whether spawn a worker in current environment.
   // On f5o main thread set true to enable worker. On worker thread this is always false.
   useWorker: false,
@@ -171,7 +168,13 @@ for (const api of apis) {
   globalThis.fcitx[name] = (...args: any[]) => Module.ccall('web_action', 'void', ['string', 'string'], [name, JSON.stringify(args)])
 }
 
-Module.onRuntimeInitialized = () => res(null)
+Module.onRuntimeInitialized = () => {
+  mount()
+  Module.FS.syncfs(true, () => {
+    restorePlugins()
+    resolve(null)
+  })
+}
 
 export {
   fcitxReady,

page/plugin.ts

@@ -1,6 +1,6 @@
 import UZIP from 'uzip'
 import { hasTouch } from './context'
-import { lsDir } from './fs'
+import { lsDir, traverseSync, USR_MOUNT_POINT } from './fs'
 import { getLocale } from './locale'
 import Module from './module'
 
@@ -28,6 +28,21 @@ function distributeFiles(manifest: UZIP.UZIPFiles, dir: string) {
   })
 }
 
+function symlink(manifest: UZIP.UZIPFiles) {
+  Object.entries(manifest).forEach(([path]) => {
+    const absolutePath = `/usr/${path}`
+    if (path.endsWith('/')) {
+      Module.FS.mkdirTree(absolutePath)
+    }
+    else {
+      try {
+        Module.FS.symlink(`${USR_MOUNT_POINT}/${path}`, absolutePath)
+      }
+      catch {}
+    }
+  })
+}
+
 export function unzip(buffer: ArrayBuffer, dir: string) {
   // UZIP hangs on empty zip.
   if (!buffer.byteLength) {
@@ -43,7 +58,7 @@ export function installPlugin(buffer: ArrayBuffer) {
     throw new Error('Invalid plugin')
   }
   const manifest = UZIP.parse(buffer)
-  const names = Object.keys(manifest).map(path => (path.match(/^plugin\/(\S+)\.json$/) ?? { 1: undefined })[1]).filter(name => name !== undefined)
+  const names = Object.keys(manifest).flatMap(path => (path.match(/^plugin\/(\S+)\.json$/) ?? { 1: [] })[1])
   if (names.length !== 1) {
     throw new Error('Invalid plugin')
   }
@@ -52,12 +67,29 @@ export function installPlugin(buffer: ArrayBuffer) {
   if (!byteArray) {
     throw new Error('Invalid plugin')
   }
-  distributeFiles(manifest, '/usr')
+  // Extract plugin to /backup/usr so that they are stored in IDBFS.
+  distributeFiles(manifest, USR_MOUNT_POINT)
+  // Symlink one plugin's files to /usr.
+  symlink(manifest)
   reload()
   addDefaultIMs(byteArray)
   return name
 }
 
+// Symlink all plugins' files from /backup/usr to /usr.
+export function restorePlugins() {
+  traverseSync((backupPath) => {
+    const path = `/usr${backupPath.slice(USR_MOUNT_POINT.length)}`
+    Module.FS.mkdirTree(path)
+  }, (backupPath) => {
+    const path = `/usr${backupPath.slice(USR_MOUNT_POINT.length)}`
+    try {
+      Module.FS.symlink(backupPath, path)
+    }
+    catch {}
+  }, undefined)(USR_MOUNT_POINT)
+}
+
 export function getInstalledPlugins() {
   try {
     const files = lsDir('/usr/plugin')

playwright.config.ts

@@ -4,6 +4,7 @@ const mobileTests = /.*\.mobile\.spec\.ts/
 
 export default defineConfig({
   testDir: 'tests',
+  retries: 3,
   projects: [{
     name: 'Chromium',
     use: {

src/CMakeLists.txt

@@ -19,11 +19,12 @@ set(EXE_LINK_OPTIONS
     "-sSTACK_SIZE=1MB" # fcitx5-chewing's log overflows the default 64KB stack
     "-sALLOW_MEMORY_GROWTH=1"
     "-sMAXIMUM_MEMORY=4GB"
-    "-sEXPORTED_RUNTIME_METHODS=['ccall','FS']"
+    "-sEXPORTED_RUNTIME_METHODS=['ccall','FS','IDBFS']"
     "-sENVIRONMENT=web,worker" # disable require("ws") in Fcitx5.js which is rejected by project user's bundler
     "--extern-pre-js ${PROJECT_BINARY_DIR}/pre.js"
     "--extern-post-js ${PROJECT_BINARY_DIR}/index.js"
     "-L${PROJECT_BINARY_DIR}/destdir/usr/lib"
+    "-lidbfs.js"
     Fcitx5::Core
 )
 

tests/test-idbfs.spec.ts

@@ -0,0 +1,25 @@
+import type { Page } from '@playwright/test'
+import { expect, test } from '@playwright/test'
+import { init } from './util'
+
+async function readTextFile(page: Page, path: string): Promise<string> {
+  return page.evaluate(({ path }) => window.fcitx.Module.FS.readFile(path, { encoding: 'utf8' }), { path })
+}
+
+test('IDBFS works', async ({ page }) => {
+  await init(page)
+
+  expect(await page.evaluate(() => window.fcitx.Module.FS.readdir('/backup')), '/backup/usr is automatically created').toEqual(['.', '..', 'usr'])
+
+  await page.evaluate(() => window.fcitx.Module.FS.mkdirTree('/backup/usr/local'))
+  await page.evaluate(() => window.fcitx.Module.FS.writeFile('/backup/usr/local/foo.txt', 'bar'))
+  await page.evaluate(() => window.fcitx.Module.FS.writeFile('/home/web_user/bar.txt', 'baz'))
+  await init(page)
+  expect(await readTextFile(page, '/backup/usr/local/foo.txt'), 'Data persists after reload').toBe('bar')
+  expect(await readTextFile(page, '/home/web_user/bar.txt')).toBe('baz')
+
+  await page.evaluate(() => window.fcitx.reset())
+  await init(page)
+  expect(await page.evaluate(() => window.fcitx.Module.FS.readdir('/backup/usr')), 'Data cleared after reset').toEqual(['.', '..'])
+  expect(await page.evaluate(() => window.fcitx.Module.FS.readdir('/home/web_user'))).toEqual(['.', '..', '.config'])
+})