diff --git a/clients/remote/package.json b/clients/remote/package.json index 26170ba1..81c48168 100644 --- a/clients/remote/package.json +++ b/clients/remote/package.json @@ -13,6 +13,7 @@ "test": "yarn node --experimental-vm-modules $(yarn bin jest)", "start": "yarn node dist/index.js -c ../test/config.json", "debug": "yarn node --inspect-brk dist/index.js -c ../test/config.json", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "db:bootstrap": "tsx ./src/index -c ../test/config.json db bootstrap", "db:migrate:up": "tsx ./src/index -c ../test/config.json db up", "db:migrate:down": "tsx ./src/index -c ../test/config.json db down", diff --git a/core/ledger-client/package.json b/core/ledger-client/package.json index d81a92c9..fc497ffa 100644 --- a/core/ledger-client/package.json +++ b/core/ledger-client/package.json @@ -13,6 +13,7 @@ "build": "tsc -b", "dev": "tsc -b --watch", "clean": "tsc -b --clean; rm -rf dist", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "test": "yarn node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, "devDependencies": { diff --git a/core/ledger-proto/package.json b/core/ledger-proto/package.json index 9dea88f4..26004565 100644 --- a/core/ledger-proto/package.json +++ b/core/ledger-proto/package.json @@ -12,6 +12,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist" }, "devDependencies": { diff --git a/core/rpc-errors/package.json b/core/rpc-errors/package.json index f4dda2b0..b210fd4d 100644 --- a/core/rpc-errors/package.json +++ b/core/rpc-errors/package.json @@ -17,6 +17,7 @@ }, "scripts": { "build": "tsc --build", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist" }, "dependencies": { diff --git a/core/rpc-generator/templates/client/typescript/_package.json b/core/rpc-generator/templates/client/typescript/_package.json index e111222d..cf8de873 100644 --- a/core/rpc-generator/templates/client/typescript/_package.json +++ b/core/rpc-generator/templates/client/typescript/_package.json @@ -10,6 +10,7 @@ "scripts": { "build": "tsc && typedoc src/index.ts --out docs --excludeExternals --excludePrivate --hideGenerator --readme none", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist" }, "files": ["dist/**"], diff --git a/core/signing-fireblocks/package.json b/core/signing-fireblocks/package.json index 971e5bfb..d6812d30 100644 --- a/core/signing-fireblocks/package.json +++ b/core/signing-fireblocks/package.json @@ -9,6 +9,7 @@ "scripts": { "build": "tsc -b", "clean": "tsc -b --clean && rm -rf ./dist", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "test": "jest" }, "main": "dist/index.js", diff --git a/core/signing-internal/package.json b/core/signing-internal/package.json index 2487c9ea..4060a38f 100644 --- a/core/signing-internal/package.json +++ b/core/signing-internal/package.json @@ -11,6 +11,7 @@ "scripts": { "build": "tsc -b", "clean": "tsc -b --clean && rm -rf ./dist", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "test": "jest" }, "dependencies": { diff --git a/core/signing-lib/package.json b/core/signing-lib/package.json index 9b5f27d5..4918b239 100644 --- a/core/signing-lib/package.json +++ b/core/signing-lib/package.json @@ -10,6 +10,7 @@ "types": "dist/index.d.ts", "scripts": { "build": "tsc -b", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean && rm -rf ./dist" }, "dependencies": { diff --git a/core/signing-participant/package.json b/core/signing-participant/package.json index eb64f1e6..a79adc7c 100644 --- a/core/signing-participant/package.json +++ b/core/signing-participant/package.json @@ -11,6 +11,7 @@ "scripts": { "build": "tsc -b", "clean": "tsc -b --clean && rm -rf ./dist", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "test": "yarn node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, "dependencies": { diff --git a/core/splice-client/package.json b/core/splice-client/package.json index 3401e997..8bf3ce50 100644 --- a/core/splice-client/package.json +++ b/core/splice-client/package.json @@ -11,6 +11,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist" }, "dependencies": { diff --git a/core/splice-provider/package.json b/core/splice-provider/package.json index 09bc34f1..a98233de 100644 --- a/core/splice-provider/package.json +++ b/core/splice-provider/package.json @@ -12,6 +12,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist" }, "devDependencies": { diff --git a/core/token-standard/package.json b/core/token-standard/package.json index 913f96c1..7eeff135 100644 --- a/core/token-standard/package.json +++ b/core/token-standard/package.json @@ -11,6 +11,7 @@ "scripts": { "build": "rollup -c", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist" }, "dependencies": { diff --git a/core/tx-visualizer/package.json b/core/tx-visualizer/package.json index 846799ad..e135449d 100644 --- a/core/tx-visualizer/package.json +++ b/core/tx-visualizer/package.json @@ -11,6 +11,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "test": "yarn node --experimental-vm-modules $(yarn bin jest)", "clean": "tsc -b --clean; rm -rf dist" }, diff --git a/core/types/package.json b/core/types/package.json index 4da605fe..5fbc1a24 100644 --- a/core/types/package.json +++ b/core/types/package.json @@ -12,6 +12,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist" }, "dependencies": { diff --git a/core/wallet-auth/package.json b/core/wallet-auth/package.json index ec69801a..cd7efd64 100644 --- a/core/wallet-auth/package.json +++ b/core/wallet-auth/package.json @@ -12,6 +12,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist", "test": "yarn node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, diff --git a/core/wallet-dapp-remote-rpc-client/package.json b/core/wallet-dapp-remote-rpc-client/package.json index 45787f32..1130039c 100644 --- a/core/wallet-dapp-remote-rpc-client/package.json +++ b/core/wallet-dapp-remote-rpc-client/package.json @@ -10,6 +10,7 @@ "scripts": { "build": "tsc && typedoc src/index.ts --out docs --excludeExternals --excludePrivate --hideGenerator --readme none", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist" }, "files": [ diff --git a/core/wallet-store-inmemory/package.json b/core/wallet-store-inmemory/package.json index 01a10a98..6ce31b92 100644 --- a/core/wallet-store-inmemory/package.json +++ b/core/wallet-store-inmemory/package.json @@ -12,6 +12,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist", "test": "yarn node --experimental-vm-modules $(yarn bin jest)" }, diff --git a/core/wallet-store-sql/package.json b/core/wallet-store-sql/package.json index abdf35c1..d9c19ccc 100644 --- a/core/wallet-store-sql/package.json +++ b/core/wallet-store-sql/package.json @@ -12,6 +12,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist", "test": "yarn node --experimental-vm-modules $(yarn bin jest)" }, diff --git a/core/wallet-store/package.json b/core/wallet-store/package.json index 986d768f..bb3490d2 100644 --- a/core/wallet-store/package.json +++ b/core/wallet-store/package.json @@ -12,6 +12,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist", "test": "yarn node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, diff --git a/core/wallet-ui-components/package.json b/core/wallet-ui-components/package.json index 3bbbfc34..374eb6e6 100644 --- a/core/wallet-ui-components/package.json +++ b/core/wallet-ui-components/package.json @@ -16,6 +16,7 @@ "build": "vite build && yarn workspace @canton-network/core-wallet-ui-components tsc --emitDeclarationOnly", "build:watch": "vite build --watch & tsc --emitDeclarationOnly --watch", "typecheck": "tsc --noEmit", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "rm -rf dist", "dev": "storybook dev -p 6006" }, diff --git a/nx.json b/nx.json index 755b891b..d5011c7d 100644 --- a/nx.json +++ b/nx.json @@ -26,6 +26,9 @@ } }, "targetDefaults": { + "flatpack": { + "dependsOn": ["build", "^flatpack"] + }, "build": { "dependsOn": ["^build"], "cache": true diff --git a/scripts/src/lib/flat-pack.ts b/scripts/src/lib/flat-pack.ts index 2d9613cd..c2b79fe3 100644 --- a/scripts/src/lib/flat-pack.ts +++ b/scripts/src/lib/flat-pack.ts @@ -11,26 +11,19 @@ import { getRepoRoot } from './utils.js' const repoRoot = getRepoRoot() -function run(cmd: string, opts: { cwd?: string } = {}) { +function run( + cmd: string, + opts: { cwd?: string; env?: Record } = {} +) { console.log(`$ ${cmd}`) execSync(cmd, { stdio: 'inherit', ...opts }) } -interface DependencyPackage { - name: string - dir: string -} - -interface DependencyPackageVendored { - name: string - dir: string - filename: string -} - /** Given a package directory within this repository: build, pack, and copy it as well as all its dependencies to an output directory */ export class FlatPack { private outDir: string private vendoredDir: string + private postInitHook: (() => void) | undefined = undefined constructor( private pkgDir: string, @@ -41,6 +34,10 @@ export class FlatPack { this.vendoredDir = path.join(this.outDir, '.vendored') } + public postInit(callback: () => void): void { + this.postInitHook = callback + } + /** * Build, pack, and copy the package and its dependencies to the output directory * @returns The path to the output directory @@ -48,51 +45,51 @@ export class FlatPack { public pack(): string { const mainPkgDir = this.pkgDir const mainPkgName = this.readPackageJson(mainPkgDir).name + const mainPkgFileName = `${mainPkgName.replaceAll('/', '-')}.tgz` + + console.log('Packing for: ' + mainPkgName) - console.log('packing for: ' + mainPkgName) this.init() - const dependencies = this.getRecursiveDependencies() + run(`yarn nx --tui=false run ${mainPkgName}:flatpack`, { + cwd: repoRoot, + env: { + ...process.env, + FLATPACK_OUTDIR: `${this.vendoredDir}/%s.tgz`, + }, + }) - for (const dep of dependencies) { - const vendored = this.buildDependency(dep) - this.overridePackageJson(vendored) + const overrides = + this.projectType === 'npm' ? 'overrides' : 'resolutions' + + const resolvedDependencies = {} as Record + + const dependencies = fs + .readdirSync(this.vendoredDir) + .filter((f) => f.endsWith('.tgz')) + .filter((f) => f !== mainPkgFileName) + + for (const file of dependencies) { + if (file.endsWith('.tgz')) { + const pkgName = file + .split('.tgz')[0] + .replaceAll('@canton-network-', '@canton-network/') + resolvedDependencies[pkgName] = `file:./.vendored/${file}` + } } - const mainPkg = this.buildDependency({ - name: this.readPackageJson(this.pkgDir).name, - dir: this.pkgDir, - }) - this.overridePackageJson(mainPkg) this.writePackageJson((pkgJson) => ({ ...pkgJson, dependencies: { - [mainPkgName]: `file:./.vendored/${mainPkg.filename}`, + ...pkgJson.dependencies, + [mainPkgName]: `file:./.vendored/${mainPkgFileName}`, }, + [overrides]: resolvedDependencies, })) return this.outDir } - public getRecursiveDependencies(): DependencyPackage[] { - const cmd = - 'yarn workspaces foreach --no-private --topological -R exec \'echo "$npm_package_name $(pwd)"\'' - - const workspaces = execSync(cmd, { - encoding: 'utf8', - cwd: path.join(repoRoot, this.pkgDir), - }) - - return workspaces - .trim() - .split('\n') - .filter((l) => !l.startsWith('Done')) - .map((line) => { - const [name, dir] = line.split(' ') - return { name, dir } - }) - } - private init() { fs.mkdirSync(path.join(this.vendoredDir), { recursive: true }) fs.writeFileSync( @@ -103,34 +100,19 @@ export class FlatPack { private: true, version: '0.0.0', description: 'Temporary package for flat packing', + ...(this.projectType === 'yarn' + ? { packageManager: 'yarn@4.9.4' } + : {}), dependencies: {}, }, null, 2 ) ) - } - - private pkgNameToFileName(name: string): string { - return ( - name.replaceAll('/', '_').replaceAll('-', '_').replaceAll('@', '') + - '.tgz' - ) - } - - private buildDependency(dep: DependencyPackage): DependencyPackageVendored { - console.log('building package in: ' + dep.dir) - - const filename = this.pkgNameToFileName(dep.name) - const outpath = path.join( - this.vendoredDir, - this.pkgNameToFileName(dep.name) - ) - run('yarn build', { cwd: dep.dir }) - run(`yarn pack --filename "${outpath}"`, { cwd: dep.dir }) - - return { ...dep, filename } + if (this.postInitHook) { + this.postInitHook() + } } private readPackageJson(parentDir: string): any { @@ -151,17 +133,4 @@ export class FlatPack { JSON.stringify(callback(pkgJson), null, 2) ) } - - private overridePackageJson(dep: DependencyPackageVendored) { - const overrides = - this.projectType === 'npm' ? 'overrides' : 'resolutions' - - this.writePackageJson((pkgJson) => ({ - ...pkgJson, - [overrides]: { - ...pkgJson[overrides], - [dep.name]: `file:./.vendored/${dep.filename}`, - }, - })) - } } diff --git a/scripts/src/package-and-verify-wallet-sdk.ts b/scripts/src/package-and-verify-wallet-sdk.ts index d3ad6dd8..e6885752 100644 --- a/scripts/src/package-and-verify-wallet-sdk.ts +++ b/scripts/src/package-and-verify-wallet-sdk.ts @@ -7,7 +7,8 @@ import { diffChars } from 'diff' import * as fs from 'fs' import os from 'os' import * as path from 'path' -import { error, getRepoRoot, success } from './lib/utils.js' +import { error, success } from './lib/utils.js' +import { FlatPack } from './lib/flat-pack.js' function run(cmd: string, opts: { cwd?: string } = {}) { console.log(`$ ${cmd}`) @@ -38,109 +39,42 @@ function runAssert(cmd: string, assertOutput: string, opts: { cwd?: string }) { return output } -const repoRoot = getRepoRoot() - -async function buildPackage( - name: string, - pkgDir: string, - tmpDir: string -): Promise { - const filename = path.join(tmpDir, `${name}.tgz`) - - run('yarn build', { cwd: pkgDir }) - run(`yarn pack --filename "${filename}"`, { cwd: pkgDir }) - run(`yarn add "${filename}"`, { cwd: tmpDir }) - - return filename -} - -function updateJsonResolutions( - tmpDir: string, - resolutions: Record -) { - const pkgJsonPath = path.join(tmpDir, 'package.json') - const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')) - pkgJson.resolutions = { - ...pkgJson.resolutions, - ...resolutions, - } - fs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2)) -} - -function getAllWalletSdkDependencies( - dir: string -): { name: string; dir: string }[] { - const cmd = - 'yarn workspaces foreach --no-private --topological -R exec \'echo "$npm_package_name $(pwd)"\'' - - const workspaces = execSync(cmd, { - encoding: 'utf8', - cwd: path.join(repoRoot, dir), - }) - - return workspaces - .trim() - .split('\n') - .filter((l) => !l.startsWith('Done')) - .map((line) => { - const [name, dir] = line.split(' ') - return { name, dir } - }) -} - async function main() { const sdkDir = 'sdk/wallet-sdk' // Create a temp dir for both the test and the tgz const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wallet-sdk-test-')) + const flatpacker = new FlatPack(sdkDir, 'yarn', tmpDir) // Prepare a yarn-based project in the tmp dir try { - // write Yarn config for isolated, node_modules-based test project - fs.writeFileSync( - path.join(tmpDir, '.yarnrc.yml'), - ['nodeLinker: node-modules', 'enableGlobalCache: false'].join( - '\n' - ) + '\n' - ) - - // Test import in temp dir - run('yarn init -y', { cwd: tmpDir }) + flatpacker.postInit(() => { + // write Yarn config for isolated, node_modules-based test project + fs.writeFileSync( + path.join(tmpDir, '.yarnrc.yml'), + ['nodeLinker: node-modules', 'enableGlobalCache: false'].join( + '\n' + ) + '\n' + ) + + // Write test import file + const testFile = path.join(tmpDir, 'test-import.ts') + fs.writeFileSync( + testFile, + `import { WalletSDKImpl } from '@canton-network/wallet-sdk';\n console.log('Import successful.' + WalletSDKImpl);` + ) + + run('yarn add typescript tsx', { cwd: tmpDir }) + }) + flatpacker.pack() run('yarn install --no-immutable', { cwd: tmpDir }) - - // Write test import file - const testFile = path.join(tmpDir, 'test-import.ts') - fs.writeFileSync( - testFile, - `import { WalletSDKImpl } from '@canton-network/wallet-sdk';\n console.log('Import successful.' + WalletSDKImpl);` - ) - run('yarn add typescript tsx', { cwd: tmpDir }) } catch (e) { fs.rmSync(tmpDir, { recursive: true, force: true }) throw e } - run('yarn install', { cwd: repoRoot }) - - const packages = getAllWalletSdkDependencies(sdkDir).filter( - (p) => p.name !== '@canton-network/wallet-sdk' - ) - - const paths = {} as Record - for (const pkg of packages) { - const pkgPath = await buildPackage(pkg.name, pkg.dir, tmpDir) - - paths[pkg.name] = `file:${pkgPath}` - updateJsonResolutions(tmpDir, paths) - } - - await buildPackage('wallet-sdk', sdkDir, tmpDir) - let ran = false try { - // Resolve token-standard to the packed tgz - updateJsonResolutions(tmpDir, paths) - runAssert( 'tsx test-import.ts', // Assert that the imported `WalletSDKImpl` object is not undefined or anything unexpected diff --git a/sdk/dapp-sdk/package.json b/sdk/dapp-sdk/package.json index 7d5a9722..0117fa17 100644 --- a/sdk/dapp-sdk/package.json +++ b/sdk/dapp-sdk/package.json @@ -11,6 +11,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist", "test": "echo \"Warning: no test specified\"" }, diff --git a/sdk/wallet-sdk/package.json b/sdk/wallet-sdk/package.json index 67966c6c..0c766837 100644 --- a/sdk/wallet-sdk/package.json +++ b/sdk/wallet-sdk/package.json @@ -11,6 +11,7 @@ "scripts": { "build": "tsc -b", "dev": "tsc -b --watch", + "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"", "clean": "tsc -b --clean; rm -rf dist", "test": "echo \"Warning: no test specified\"" },