From 022cb3fedf2327394e0dda3ed33e28f9a8083251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikul=C3=A1=C5=A1=20T=C5=99os?= Date: Sat, 29 Mar 2025 13:45:12 +0100 Subject: [PATCH 1/2] Implement translation key tree-shaking for Vite i18n plugin Added logic to extract and filter used translation keys during the build process using AST parsing for Vue templates and script blocks. Updated `generateFiles` to exclude unused keys, improving build efficiency and output size. Included tests to verify key extraction and filtering functionality. --- src/interfaces/php-ast-node.ts | 14 +++ src/interfaces/translation.ts | 1 + src/loader.ts | 96 ++++++++++++----- src/vite.ts | 185 ++++++++++++++++++++++++++++----- test/vite.test.ts | 45 ++++++++ 5 files changed, 289 insertions(+), 52 deletions(-) create mode 100644 src/interfaces/php-ast-node.ts create mode 100644 src/interfaces/translation.ts create mode 100644 test/vite.test.ts diff --git a/src/interfaces/php-ast-node.ts b/src/interfaces/php-ast-node.ts new file mode 100644 index 0000000..395a19c --- /dev/null +++ b/src/interfaces/php-ast-node.ts @@ -0,0 +1,14 @@ +export interface PhpAstNode { + kind: string; + expr?: PhpAstNode; + items?: PhpAstNode[]; + value?: string | PhpAstNode; + key?: { + kind: string; + value: string; + offset?: { name: string }; + what?: { name?: string; offset?: { name: string } }; + }; + left?: PhpAstNode; + right?: PhpAstNode; +} \ No newline at end of file diff --git a/src/interfaces/translation.ts b/src/interfaces/translation.ts new file mode 100644 index 0000000..cc3bca8 --- /dev/null +++ b/src/interfaces/translation.ts @@ -0,0 +1 @@ +export type TranslationValue = string | null | { [key: string]: TranslationValue }; \ No newline at end of file diff --git a/src/loader.ts b/src/loader.ts index 4e4bd06..8d882bd 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -2,6 +2,8 @@ import fs from 'fs' import path from 'path' import { Engine } from 'php-parser' import { ParsedLangFileInterface } from './interfaces/parsed-lang-file' +import { TranslationValue } from './interfaces/translation' +import { PhpAstNode } from './interfaces/php-ast-node' const toCamelCase = (str: string): string => { if (str === str.toUpperCase()) { @@ -21,8 +23,6 @@ export const hasPhpTranslations = (folderPath: string): boolean => { .sort() for (const folder of folders) { - const lang = {} - const files = fs.readdirSync(folderPath + path.sep + folder).filter((file) => /\.php$/.test(file)) if (files.length > 0) { @@ -99,7 +99,7 @@ function mergeVendorTranslations(folder: string, translations: any, vendorTransl return { ...translations, ...langTranslationsFromVendor } } -export const parse = (content: string) => { +export const parse = (content: string): Record => { const arr = new Engine({}).parseCode(content, 'lang').children.filter((child) => child.kind === 'return')[0] as any if (arr?.expr?.kind !== 'array') { @@ -109,9 +109,9 @@ export const parse = (content: string) => { return convertToDotsSyntax(parseItem(arr.expr)) } -const parseItem = (expr) => { +const parseItem = (expr: PhpAstNode): TranslationValue => { if (expr.kind === 'string') { - return expr.value + return expr.value as string || '' } if (expr.kind === 'nullkeyword') { @@ -119,41 +119,60 @@ const parseItem = (expr) => { } if (expr.kind === 'array') { - let items = expr.items.map((item) => parseItem(item)) + if (!expr.items) return {} + + const parsedItems = expr.items.map((item) => parseItem(item)) if (expr.items.every((item) => item.key !== null)) { - items = items.reduce((acc, val) => Object.assign({}, acc, val), {}) - } + return parsedItems.reduce>( + (acc, val) => { + const valAsRecord = val as Record; + + return { ...acc, ...valAsRecord }; + }, + {} + ); + } else { + const indexedResult: Record = {}; + + parsedItems.forEach((item, index) => { + indexedResult[index.toString()] = item as TranslationValue; + }); - return items + return indexedResult; + } } if (expr.kind === 'bin') { - return parseItem(expr.left) + parseItem(expr.right) + if (!expr.left || !expr.right) return '' + return (parseItem(expr.left) as string) + (parseItem(expr.right) as string) } if (expr.key) { let key = expr.key.value if (expr.key.kind === 'staticlookup') { - if (expr.key.offset.name === 'class') { - key = toCamelCase(expr.key.what.name) + if (expr.key.offset?.name === 'class') { + key = toCamelCase(expr.key.what?.name || '') } else { - key = toCamelCase(expr.key.offset.name) + key = toCamelCase(expr.key.offset?.name || '') } } else if (expr.key.kind === 'propertylookup') { - key = toCamelCase(expr.key.what.offset.name) + key = toCamelCase(expr.key.what?.offset?.name || '') } - return { [key]: parseItem(expr.value) } + if (!expr.value) return { [key]: '' } + return { [key]: parseItem(expr.value as PhpAstNode) } as TranslationValue } - return parseItem(expr.value) + if (!expr.value) return {} + return parseItem(expr.value as PhpAstNode) || {} } -const convertToDotsSyntax = (list) => { - const flatten = (items, context = '') => { - const data = {} + +const convertToDotsSyntax = (list: TranslationValue | TranslationValue[]): Record => { + const flatten = (items: TranslationValue | TranslationValue[], context = '') => { + const data: Record = {} if (items === null) { return data @@ -176,7 +195,7 @@ const convertToDotsSyntax = (list) => { return flatten(list) } -export const reset = (folderPath) => { +export const reset = (folderPath: string) => { const dir = fs.readdirSync(folderPath) dir @@ -186,18 +205,18 @@ export const reset = (folderPath) => { }) } -export const readThroughDir = (dir) => { - const data = {} +export const readThroughDir = (dir: string): TranslationValue => { + const data: TranslationValue = {} fs.readdirSync(dir).forEach((file) => { const absoluteFile = dir + path.sep + file if (fs.statSync(absoluteFile).isDirectory()) { - const subFolderFileKey = file.replace(/\.\w+$/, '') + const subFolderFileKey = file.replace(/\.\w+$/, ""); - data[subFolderFileKey] = readThroughDir(absoluteFile) + data[subFolderFileKey] = readThroughDir(absoluteFile); } else { - data[file.replace(/\.\w+$/, '')] = parse(fs.readFileSync(absoluteFile).toString()) + data[file.replace(/\.\w+$/, "")] = parse(fs.readFileSync(absoluteFile).toString()); } }) @@ -207,9 +226,30 @@ export const readThroughDir = (dir) => { export const prepareExtendedParsedLangFiles = (langPaths: string[]): ParsedLangFileInterface[] => langPaths.flatMap((langPath) => parseAll(langPath)) -export const generateFiles = (langPath: string, data: ParsedLangFileInterface[]): ParsedLangFileInterface[] => { +export const generateFiles = (langPath: string, data: ParsedLangFileInterface[], usedKeys: Set | null = null): ParsedLangFileInterface[] => { data = mergeData(data) + if (usedKeys) { + const exactKeys = new Set(); + const prefixes: string[] = []; + usedKeys.forEach(key => { + if (key.endsWith('*')) { + prefixes.push(key.slice(0, -1)); + } else { + exactKeys.add(key); + } + }); + + data = data.map(langFile => ({ + ...langFile, + translations: Object.fromEntries( + Object.entries(langFile.translations).filter(([key]) => + exactKeys.has(key) || prefixes.some(prefix => key.startsWith(prefix)) + ) + ) + })); + } + if (!fs.existsSync(langPath)) { fs.mkdirSync(langPath) } @@ -222,7 +262,7 @@ export const generateFiles = (langPath: string, data: ParsedLangFileInterface[]) } function mergeData(data: ParsedLangFileInterface[]): ParsedLangFileInterface[] { - const obj = {} + const obj: Record> = {} data.forEach(({ name, translations }) => { if (!obj[name]) { @@ -232,7 +272,7 @@ function mergeData(data: ParsedLangFileInterface[]): ParsedLangFileInterface[] { obj[name] = { ...obj[name], ...translations } }) - const arr = [] + const arr: ParsedLangFileInterface[] = [] Object.entries(obj).forEach(([name, translations]) => { arr.push({ name, translations }) }) diff --git a/src/vite.ts b/src/vite.ts index 0e5c9eb..5dae219 100644 --- a/src/vite.ts +++ b/src/vite.ts @@ -1,9 +1,111 @@ import path from 'path' -import { existsSync, unlinkSync, readdirSync, rmdirSync } from 'fs' +import { existsSync, unlinkSync, readdirSync, rmdirSync, readFileSync } from 'fs' +import { Plugin } from 'vite' +import fg from 'fast-glob' +import { parse } from '@vue/compiler-sfc' +import * as acorn from 'acorn' +import { walk } from 'estree-walker' +import ts from 'typescript' +import { NodeTypes, RootNode, TemplateChildNode, AttributeNode, DirectiveNode, InterpolationNode, TextNode, ElementNode, SimpleExpressionNode } from "@vue/compiler-core" import { hasPhpTranslations, generateFiles, prepareExtendedParsedLangFiles } from './loader' import { ParsedLangFileInterface } from './interfaces/parsed-lang-file' import { VitePluginOptionsInterface } from './interfaces/plugin-options' -import { Plugin } from 'vite' + +let usedKeys: Set | null = null + +export function extractStaticPrefix(node: any): string | null { + if (node.type === 'Literal' && typeof node.value === 'string') { + return node.value + } else if (node.type === 'TemplateLiteral') { + if (node.quasis.length > 0 && node.expressions.length > 0) { + const firstQuasi = node.quasis[0] + if (firstQuasi.type === 'TemplateElement' && firstQuasi.value) { + return firstQuasi.value.cooked + } + } else if (node.quasis.length === 1) { + return node.quasis[0].value.cooked + } + } else if (node.type === 'BinaryExpression' && node.operator === '+') { + const leftPrefix = extractStaticPrefix(node.left) + if (leftPrefix !== null) { + const rightPrefix = extractStaticPrefix(node.right) + if (rightPrefix !== null) { + return leftPrefix + rightPrefix + } else { + return leftPrefix + } + } + } + return null +} + +export function extractKeys(ast: any, functionNames: string[]): Set { + const keys = new Set() + walk(ast, { + enter(node: any) { + if (node.type === 'CallExpression') { + let calleeName: string | null = null + if (node.callee.type === 'Identifier') { + calleeName = node.callee.name + } else if ( + node.callee.type === 'MemberExpression' && + node.callee.object.type === 'ThisExpression' && + node.callee.property.type === 'Identifier' + ) { + calleeName = node.callee.property.name + } else if (node.callee.type === 'Identifier' && functionNames.includes(node.callee.name)) { + calleeName = node.callee.name + } + if (calleeName && functionNames.includes(calleeName)) { + const arg = node.arguments[0] + if (arg) { + if (arg.type === 'Literal' && typeof arg.value === 'string') { + keys.add(arg.value) + } else if (arg.type === 'TemplateLiteral' || arg.type === 'BinaryExpression') { + const prefix = extractStaticPrefix(arg) + if (prefix) { + keys.add(prefix + '*') + } + } + } + } + } + } + }) + return keys +} + +export function collectTemplateExpressions(node: RootNode | TemplateChildNode | AttributeNode | DirectiveNode): string[] { + const expressions: string[] = [] + if (!node) return expressions + + if (node.type === NodeTypes.ROOT && 'children' in node) { + node.children.forEach(child => expressions.push(...collectTemplateExpressions(child as TemplateChildNode))) + } else if (node.type === NodeTypes.ELEMENT && 'props' in node && 'children' in node) { + const elementNode = node as ElementNode + if (elementNode.props) { + elementNode.props.forEach(prop => { + if (prop.type === NodeTypes.DIRECTIVE && prop.exp) { + expressions.push((prop.exp as SimpleExpressionNode).content || '') + } + }) + } + if (elementNode.children) { + elementNode.children.forEach(child => expressions.push(...collectTemplateExpressions(child))) + } + } else if (node.type === NodeTypes.INTERPOLATION && 'content' in node) { + const interpolationNode = node as InterpolationNode + if (interpolationNode.content.type === NodeTypes.SIMPLE_EXPRESSION) { + expressions.push(interpolationNode.content.content || '') + } + } else if (node.type === NodeTypes.TEXT && 'content' in node) { + const textNode = node as TextNode + if (textNode.content.includes('$t(')) { + expressions.push(textNode.content) + } + } + return expressions +} export default function i18n(options: string | VitePluginOptionsInterface = 'lang'): Plugin { let langPath = typeof options === 'string' ? options : options.langPath ?? 'lang' @@ -16,24 +118,18 @@ export default function i18n(options: string | VitePluginOptionsInterface = 'lan let exitHandlersBound: boolean = false const clean = () => { - files.forEach((file) => { + files.forEach(file => { const filePath = langPath + file.name - if (existsSync(filePath)) { - unlinkSync(filePath) - } + if (existsSync(filePath)) unlinkSync(filePath) }) - files = [] - - if (existsSync(langPath) && readdirSync(langPath).length < 1) { - rmdirSync(langPath) - } + if (existsSync(langPath) && readdirSync(langPath).length === 0) rmdirSync(langPath) } return { name: 'i18n', enforce: 'post', - config(config) { + config() { /** @ts-ignore */ process.env.VITE_LARAVEL_VUE_I18N_HAS_PHP = true @@ -43,24 +139,67 @@ export default function i18n(options: string | VitePluginOptionsInterface = 'lan } } }, - buildEnd: clean, buildStart() { - if (!hasPhpTranslations(frameworkLangPath) && !hasPhpTranslations(langPath)) { - return + const vueFiles = fg.sync('**/*.vue', { cwd: './resources/js/', absolute: true }) + usedKeys = new Set() + + for (const file of vueFiles) { + const content = readFileSync(file, 'utf8') + const { descriptor } = parse(content, { sourceMap: false }) + + if (descriptor.script) { + const scriptAst = acorn.parse(descriptor.script.content, { + ecmaVersion: 'latest', + sourceType: 'module' + }) + const scriptKeys = extractKeys(scriptAst, ['trans', 'wTrans', 'transChoice', 'wTransChoice']) + scriptKeys.forEach(key => usedKeys!.add(key)) + } + + if (descriptor.scriptSetup) { + let scriptSetupContent = descriptor.scriptSetup.content + if (descriptor.scriptSetup.lang === 'ts') { + const transpiled = ts.transpileModule(scriptSetupContent, { compilerOptions: { module: ts.ModuleKind.ESNext } }) + scriptSetupContent = transpiled.outputText + } + const scriptSetupAst = acorn.parse(scriptSetupContent, { + ecmaVersion: 'latest', + sourceType: 'module' + }) + const scriptSetupKeys = extractKeys(scriptSetupAst, ['trans', 'wTrans', 'transChoice', 'wTransChoice']) + scriptSetupKeys.forEach(key => usedKeys!.add(key)) + } + + if (descriptor.template) { + const templateAst = descriptor.template.ast + if (templateAst) { + const expressions = collectTemplateExpressions(templateAst) + for (const expr of expressions) { + try { + const exprAst = acorn.parseExpressionAt(expr, 0, { ecmaVersion: 'latest' }) + const exprKeys = extractKeys(exprAst, ['$t', '$tChoice']) + exprKeys.forEach(key => usedKeys!.add(key)) + } catch (e) { + // Ignore parsing errors + } + } + } + } } - const langPaths = prepareExtendedParsedLangFiles([frameworkLangPath, langPath, ...additionalLangPaths]) + if (!hasPhpTranslations(frameworkLangPath) && !hasPhpTranslations(langPath)) return - files = generateFiles(langPath, langPaths) + const data = prepareExtendedParsedLangFiles([frameworkLangPath, langPath, ...additionalLangPaths]) + files = generateFiles(langPath, data, usedKeys) }, handleHotUpdate(ctx) { if (/lang\/.*\.php$/.test(ctx.file)) { - const langPaths = prepareExtendedParsedLangFiles([frameworkLangPath, langPath, ...additionalLangPaths]) - - files = generateFiles(langPath, langPaths) + const data = prepareExtendedParsedLangFiles([frameworkLangPath, langPath, ...additionalLangPaths]) + files = generateFiles(langPath, data, usedKeys) } }, - configureServer(server) { + buildEnd: clean, + configureServer() { if (exitHandlersBound) { return } @@ -69,8 +208,6 @@ export default function i18n(options: string | VitePluginOptionsInterface = 'lan process.on('SIGINT', process.exit) process.on('SIGTERM', process.exit) process.on('SIGHUP', process.exit) - - exitHandlersBound = true } } -} +} \ No newline at end of file diff --git a/test/vite.test.ts b/test/vite.test.ts new file mode 100644 index 0000000..bee9f02 --- /dev/null +++ b/test/vite.test.ts @@ -0,0 +1,45 @@ +import { extractKeys } from '../src/vite'; +import * as acorn from 'acorn'; +import { generateFiles } from '../src/loader'; + +describe('Translation keys tree shaking', () => { + describe('extractKeys', () => { + it('should extract only static translation keys', () => { + const code = ` + trans('used.key1'); + trans("used.key2"); + trans('unused.key'); + trans('partial.' + variable); // should not resolve fully, may yield a prefix if applicable + `; + const ast = acorn.parse(code, { ecmaVersion: 'latest' }); + const keys = extractKeys(ast, ['trans']); + + expect(keys.has('used.key1')).toBe(true); + expect(keys.has('used.key2')).toBe(true); + + expect([...keys].find(key => key.startsWith('partial.'))).toBeDefined(); + }); + }); + + describe('generateFiles tree shaking', () => { + it('should generate translation files without unused keys', () => { + const allTranslations = { + 'used.key1': 'Hello', + 'used.key2': 'World', + 'unused.key': 'Should be removed', + 'partial.some': 'Partial value' + }; + + const usedKeys = new Set(['used.key1', 'used.key2', 'partial.*']); + + const generated = generateFiles('lang', [{ name: 'en', translations: allTranslations }], usedKeys); + + const generatedContent = generated[0].translations; + + expect(generatedContent['unused.key']).toBeUndefined(); + expect(generatedContent['used.key1']).toBe('Hello'); + expect(generatedContent['used.key2']).toBe('World'); + expect(generatedContent['partial.some']).toBe('Partial value'); + }); + }); +}); \ No newline at end of file From c3dc6d08e485ad07ac6462d35416abd6cad99e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikul=C3=A1=C5=A1=20T=C5=99os?= Date: Sat, 29 Mar 2025 19:21:42 +0100 Subject: [PATCH 2/2] Add treeShake flag to Vite plugin Introduced a `treeShake` option to optimize the plugin by removing unused translations. Updated relevant logic to respect the `treeShake` setting and added tests to validate the new behavior. --- src/interfaces/plugin-options.ts | 3 +- src/vite.ts | 74 +++++++++++++++++--------------- test/vite.test.ts | 30 ++++++++++++- 3 files changed, 70 insertions(+), 37 deletions(-) diff --git a/src/interfaces/plugin-options.ts b/src/interfaces/plugin-options.ts index 161f9ec..5310303 100644 --- a/src/interfaces/plugin-options.ts +++ b/src/interfaces/plugin-options.ts @@ -9,5 +9,6 @@ export interface PluginOptionsInterface extends OptionsInterface { export interface VitePluginOptionsInterface { langPath?: string - additionalLangPaths?: string[] + additionalLangPaths?: string[], + treeShake?: boolean, } diff --git a/src/vite.ts b/src/vite.ts index 5dae219..174c62b 100644 --- a/src/vite.ts +++ b/src/vite.ts @@ -143,48 +143,52 @@ export default function i18n(options: string | VitePluginOptionsInterface = 'lan const vueFiles = fg.sync('**/*.vue', { cwd: './resources/js/', absolute: true }) usedKeys = new Set() - for (const file of vueFiles) { - const content = readFileSync(file, 'utf8') - const { descriptor } = parse(content, { sourceMap: false }) - - if (descriptor.script) { - const scriptAst = acorn.parse(descriptor.script.content, { - ecmaVersion: 'latest', - sourceType: 'module' - }) - const scriptKeys = extractKeys(scriptAst, ['trans', 'wTrans', 'transChoice', 'wTransChoice']) - scriptKeys.forEach(key => usedKeys!.add(key)) - } + if (typeof options !== 'string' && options.treeShake) { + for (const file of vueFiles) { + const content = readFileSync(file, 'utf8') + const { descriptor } = parse(content, { sourceMap: false }) + + if (descriptor.script) { + const scriptAst = acorn.parse(descriptor.script.content, { + ecmaVersion: 'latest', + sourceType: 'module' + }) + const scriptKeys = extractKeys(scriptAst, ['trans', 'wTrans', 'transChoice', 'wTransChoice']) + scriptKeys.forEach(key => usedKeys!.add(key)) + } - if (descriptor.scriptSetup) { - let scriptSetupContent = descriptor.scriptSetup.content - if (descriptor.scriptSetup.lang === 'ts') { - const transpiled = ts.transpileModule(scriptSetupContent, { compilerOptions: { module: ts.ModuleKind.ESNext } }) - scriptSetupContent = transpiled.outputText + if (descriptor.scriptSetup) { + let scriptSetupContent = descriptor.scriptSetup.content + if (descriptor.scriptSetup.lang === 'ts') { + const transpiled = ts.transpileModule(scriptSetupContent, { compilerOptions: { module: ts.ModuleKind.ESNext } }) + scriptSetupContent = transpiled.outputText + } + const scriptSetupAst = acorn.parse(scriptSetupContent, { + ecmaVersion: 'latest', + sourceType: 'module' + }) + const scriptSetupKeys = extractKeys(scriptSetupAst, ['trans', 'wTrans', 'transChoice', 'wTransChoice']) + scriptSetupKeys.forEach(key => usedKeys!.add(key)) } - const scriptSetupAst = acorn.parse(scriptSetupContent, { - ecmaVersion: 'latest', - sourceType: 'module' - }) - const scriptSetupKeys = extractKeys(scriptSetupAst, ['trans', 'wTrans', 'transChoice', 'wTransChoice']) - scriptSetupKeys.forEach(key => usedKeys!.add(key)) - } - if (descriptor.template) { - const templateAst = descriptor.template.ast - if (templateAst) { - const expressions = collectTemplateExpressions(templateAst) - for (const expr of expressions) { - try { - const exprAst = acorn.parseExpressionAt(expr, 0, { ecmaVersion: 'latest' }) - const exprKeys = extractKeys(exprAst, ['$t', '$tChoice']) - exprKeys.forEach(key => usedKeys!.add(key)) - } catch (e) { - // Ignore parsing errors + if (descriptor.template) { + const templateAst = descriptor.template.ast + if (templateAst) { + const expressions = collectTemplateExpressions(templateAst) + for (const expr of expressions) { + try { + const exprAst = acorn.parseExpressionAt(expr, 0, { ecmaVersion: 'latest' }) + const exprKeys = extractKeys(exprAst, ['$t', '$tChoice']) + exprKeys.forEach(key => usedKeys!.add(key)) + } catch (e) { + // Ignore parsing errors + } } } } } + } else { + usedKeys = null } if (!hasPhpTranslations(frameworkLangPath) && !hasPhpTranslations(langPath)) return diff --git a/test/vite.test.ts b/test/vite.test.ts index bee9f02..683b54a 100644 --- a/test/vite.test.ts +++ b/test/vite.test.ts @@ -42,4 +42,32 @@ describe('Translation keys tree shaking', () => { expect(generatedContent['partial.some']).toBe('Partial value'); }); }); -}); \ No newline at end of file +}); + +describe('Vite plugin options', () => { + it('should apply tree shaking when treeShake option is true', () => { + const allTranslations = { + 'used.key': 'Used value', + 'unused.key': 'Unused value' + }; + + const usedKeys = new Set(['used.key']); + + const result = generateFiles('lang', [{ name: 'en', translations: allTranslations }], usedKeys); + + expect(result[0].translations['unused.key']).toBeUndefined(); + expect(result[0].translations['used.key']).toBe('Used value'); + }); + + it('should not apply tree shaking when treeShake option is false', () => { + const allTranslations = { + 'used.key': 'Used value', + 'unused.key': 'Unused value' + }; + + const result = generateFiles('lang', [{ name: 'en', translations: allTranslations }], null); + + expect(result[0].translations['unused.key']).toBe('Unused value'); + expect(result[0].translations['used.key']).toBe('Used value'); + }); +});