From fd287b2b03b34b3690bc7c695331997d09c6c72e Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Thu, 26 Jun 2025 14:17:29 -0700 Subject: [PATCH 1/4] Export directory --- src/emulator/controller.ts | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/emulator/controller.ts b/src/emulator/controller.ts index 684bd60e8cb..bb0eef4220a 100755 --- a/src/emulator/controller.ts +++ b/src/emulator/controller.ts @@ -76,18 +76,26 @@ const START_LOGGING_EMULATOR = utils.envOverride( */ export async function exportOnExit(options: Options): Promise { // Note: options.exportOnExit is coerced to a string before this point in commandUtils.ts#setExportOnExitOptions - const exportOnExitDir = options.exportOnExit as string; - if (exportOnExitDir) { - try { - utils.logBullet( - `Automatically exporting data using ${FLAG_EXPORT_ON_EXIT_NAME} "${exportOnExitDir}" ` + - "please wait for the export to finish...", - ); - await exportEmulatorData(exportOnExitDir, options, /* initiatedBy= */ "exit"); - } catch (e: unknown) { - utils.logWarning(`${e}`); - utils.logWarning(`Automatic export to "${exportOnExitDir}" failed, going to exit now...`); - } + var exportOnExitDir: string + + if (options.config.src.emulators?.dataDir) { + exportOnExitDir = options.config.src.emulators.dataDir + } else if (options.exportOnExit) { + exportOnExitDir = options.exportOnExit as string; + } else { + // If no export directory, skip the export. + return; + } + + try { + utils.logBullet( + `Automatically exporting data using ${FLAG_EXPORT_ON_EXIT_NAME} "${exportOnExitDir}" ` + + "please wait for the export to finish...", + ); + await exportEmulatorData(exportOnExitDir, options, /* initiatedBy= */ "exit"); + } catch (e: unknown) { + utils.logWarning(`${e}`); + utils.logWarning(`Automatic export to "${exportOnExitDir}" failed, going to exit now...`); } } From c7ceb8da6fe41dafe68b2f7a452e25a70610daa2 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Thu, 26 Jun 2025 17:52:11 -0700 Subject: [PATCH 2/4] Tests --- .../firebase.json | 3 +- scripts/emulator-import-export-tests/tests.ts | 1063 +++++++++-------- src/emulator/controller.ts | 8 +- 3 files changed, 576 insertions(+), 498 deletions(-) diff --git a/scripts/emulator-import-export-tests/firebase.json b/scripts/emulator-import-export-tests/firebase.json index e37ee130128..323c50003e6 100644 --- a/scripts/emulator-import-export-tests/firebase.json +++ b/scripts/emulator-import-export-tests/firebase.json @@ -42,6 +42,7 @@ }, "storage": { "port": 9199 - } + }, + "dataDir": "hello" } } diff --git a/scripts/emulator-import-export-tests/tests.ts b/scripts/emulator-import-export-tests/tests.ts index 55c6f8b86bf..bd72021bdfd 100644 --- a/scripts/emulator-import-export-tests/tests.ts +++ b/scripts/emulator-import-export-tests/tests.ts @@ -53,527 +53,604 @@ function logIncludes(msg: string) { } describe("import/export end to end", () => { - it("should be able to import/export firestore data", async function (this) { + // it("should be able to import/export firestore data", async function (this) { + // this.timeout(2 * TEST_SETUP_TIMEOUT); + // await new Promise((resolve) => setTimeout(resolve, 2000)); + + // // Start up emulator suite + // const emulatorsCLI = new CLIProcess("1", __dirname); + // await emulatorsCLI.start( + // "emulators:start", + // FIREBASE_PROJECT, + // ["--only", "firestore", "--debug"], + // (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }, + // ); + + // // Ask for export + // const exportCLI = new CLIProcess("2", __dirname); + // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + // await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes("Export complete"); + // }); + // await exportCLI.stop(); + + // // Stop the suite + // await emulatorsCLI.stop(); + + // // Attempt to import + // const importCLI = new CLIProcess("3", __dirname); + // await importCLI.start( + // "emulators:start", + // FIREBASE_PROJECT, + // ["--only", "firestore", "--import", exportPath, "--debug"], + // (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }, + // ); + + // await importCLI.stop(); + + // expect(true).to.be.true; + // }); + + // it("should be able to import/export rtdb data", async function (this) { + // this.timeout(2 * TEST_SETUP_TIMEOUT); + // await new Promise((resolve) => setTimeout(resolve, 2000)); + + // // Start up emulator suite + // const emulatorsCLI = new CLIProcess("1", __dirname); + // await emulatorsCLI.start( + // "emulators:start", + // FIREBASE_PROJECT, + // ["--only", "database", "--debug"], + // (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }, + // ); + + // // Write some data to export + // const config = readConfig(); + // const port = config.emulators!.database.port; + // const host = await localhost(); + // const aApp = admin.initializeApp( + // { + // projectId: FIREBASE_PROJECT, + // databaseURL: `http://${host}:${port}?ns=namespace-a`, + // credential: ADMIN_CREDENTIAL, + // }, + // "rtdb-export-a", + // ); + // const bApp = admin.initializeApp( + // { + // projectId: FIREBASE_PROJECT, + // databaseURL: `http://${host}:${port}?ns=namespace-b`, + // credential: ADMIN_CREDENTIAL, + // }, + // "rtdb-export-b", + // ); + // const cApp = admin.initializeApp( + // { + // projectId: FIREBASE_PROJECT, + // databaseURL: `http://${host}:${port}?ns=namespace-c`, + // credential: ADMIN_CREDENTIAL, + // }, + // "rtdb-export-c", + // ); + + // // Write to two namespaces + // const aRef = aApp.database().ref("ns"); + // await aRef.set("namespace-a"); + // const bRef = bApp.database().ref("ns"); + // await bRef.set("namespace-b"); + + // // Read from a third + // const cRef = cApp.database().ref("ns"); + // await cRef.once("value"); + + // // Ask for export + // const exportCLI = new CLIProcess("2", __dirname); + // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + // await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes("Export complete"); + // }); + // await exportCLI.stop(); + + // // Check that the right export files are created + // const dbExportPath = path.join(exportPath, "database_export"); + // const dbExportFiles = fs.readdirSync(dbExportPath); + // expect(dbExportFiles).to.eql(["namespace-a.json", "namespace-b.json"]); + + // // Stop the suite + // await emulatorsCLI.stop(); + + // // Attempt to import + // const importCLI = new CLIProcess("3", __dirname); + // await importCLI.start( + // "emulators:start", + // FIREBASE_PROJECT, + // ["--only", "database", "--import", exportPath, "--export-on-exit", "--debug"], + // (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }, + // ); + + // // Read the data + // const aSnap = await aRef.once("value"); + // const bSnap = await bRef.once("value"); + // expect(aSnap.val()).to.eql("namespace-a"); + // expect(bSnap.val()).to.eql("namespace-b"); + + // // Delete all of the import files + // for (const f of fs.readdirSync(dbExportPath)) { + // const fullPath = path.join(dbExportPath, f); + // fs.unlinkSync(fullPath); + // } + + // // Delete all the data in one namespace + // await bApp.database().ref().set(null); + + // // Stop the CLI (which will export on exit) + // await importCLI.stop(); + + // // Confirm the data exported is as expected + // const aPath = path.join(dbExportPath, "namespace-a.json"); + // const aData = JSON.parse(fs.readFileSync(aPath).toString()); + // expect(aData).to.deep.equal({ ns: "namespace-a" }); + + // const bPath = path.join(dbExportPath, "namespace-b.json"); + // const bData = JSON.parse(fs.readFileSync(bPath).toString()); + // expect(bData).to.equal(null); + // }); + + // it("should be able to import/export auth data", async function (this) { + // this.timeout(2 * TEST_SETUP_TIMEOUT); + // await new Promise((resolve) => setTimeout(resolve, 2000)); + + // // Start up emulator suite + // const project = FIREBASE_PROJECT || "example"; + // const emulatorsCLI = new CLIProcess("1", __dirname); + + // await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }); + + // // Create some accounts to export: + // const config = readConfig(); + // const port = config.emulators!.auth.port; + // try { + // process.env.FIREBASE_AUTH_EMULATOR_HOST = `${await localhost()}:${port}`; + // const adminApp = admin.initializeApp( + // { + // projectId: project, + // credential: ADMIN_CREDENTIAL, + // }, + // "admin-app", + // ); + // await adminApp + // .auth() + // .createUser({ uid: "123", email: "foo@example.com", password: "testing" }); + // await adminApp + // .auth() + // .createUser({ uid: "456", email: "bar@example.com", emailVerified: true }); + + // // Ask for export + // const exportCLI = new CLIProcess("2", __dirname); + // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + // await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes("Export complete"); + // }); + // await exportCLI.stop(); + + // // Stop the suite + // await emulatorsCLI.stop(); + + // // Confirm the data is exported as expected + // const configPath = path.join(exportPath, "auth_export", "config.json"); + // const configData = JSON.parse(fs.readFileSync(configPath).toString()); + // expect(configData).to.deep.equal({ + // signIn: { + // allowDuplicateEmails: false, + // }, + // emailPrivacyConfig: { + // enableImprovedEmailPrivacy: false, + // }, + // }); + + // const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); + // const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); + // expect(accountsData.users).to.have.length(2); + // expect(accountsData.users[0]).to.deep.contain({ + // localId: "123", + // email: "foo@example.com", + // emailVerified: false, + // providerUserInfo: [ + // { + // email: "foo@example.com", + // federatedId: "foo@example.com", + // providerId: "password", + // rawId: "foo@example.com", + // }, + // ], + // }); + // expect(accountsData.users[0].passwordHash).to.match(/:password=testing$/); + // expect(accountsData.users[1]).to.deep.contain({ + // localId: "456", + // email: "bar@example.com", + // emailVerified: true, + // }); + + // // Attempt to import + // const importCLI = new CLIProcess("3", __dirname); + // await importCLI.start( + // "emulators:start", + // project, + // ["--only", "auth", "--import", exportPath, "--debug"], + // (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }, + // ); + + // // Check users are indeed imported correctly + // const user1 = await adminApp.auth().getUserByEmail("foo@example.com"); + // expect(user1.passwordHash).to.match(/:password=testing$/); + // const user2 = await adminApp.auth().getUser("456"); + // expect(user2.emailVerified).to.be.true; + + // await importCLI.stop(); + // } finally { + // delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + // } + // }); + + // it("should be able to import/export auth data with many users", async function (this) { + // this.timeout(2 * TEST_SETUP_TIMEOUT); + // await new Promise((resolve) => setTimeout(resolve, 2000)); + + // // Start up emulator suite + // const project = FIREBASE_PROJECT || "example"; + // const emulatorsCLI = new CLIProcess("1", __dirname); + + // await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }); + + // // Create some accounts to export: + // const accountCount = 777; // ~120KB data when exported + // const config = readConfig(); + // const port = config.emulators!.auth.port; + // try { + // process.env.FIREBASE_AUTH_EMULATOR_HOST = `${await localhost()}:${port}`; + // const adminApp = admin.initializeApp( + // { + // projectId: project, + // credential: ADMIN_CREDENTIAL, + // }, + // "admin-app2", + // ); + // for (let i = 0; i < accountCount; i++) { + // await adminApp + // .auth() + // .createUser({ uid: `u${i}`, email: `u${i}@example.com`, password: "testing" }); + // } + // // Ask for export + // const exportCLI = new CLIProcess("2", __dirname); + // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + // await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes("Export complete"); + // }); + // await exportCLI.stop(); + + // // Stop the suite + // await emulatorsCLI.stop(); + + // // Confirm the data is exported as expected + // const configPath = path.join(exportPath, "auth_export", "config.json"); + // const configData = JSON.parse(fs.readFileSync(configPath).toString()); + // expect(configData).to.deep.equal({ + // signIn: { + // allowDuplicateEmails: false, + // }, + // emailPrivacyConfig: { + // enableImprovedEmailPrivacy: false, + // }, + // }); + + // const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); + // const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); + // expect(accountsData.users).to.have.length(accountCount); + + // // Attempt to import + // const importCLI = new CLIProcess("3", __dirname); + // await importCLI.start( + // "emulators:start", + // project, + // ["--only", "auth", "--import", exportPath, "--debug"], + // (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }, + // ); + + // // Check users are indeed imported correctly + // const user = await adminApp.auth().getUserByEmail(`u${accountCount - 1}@example.com`); + // expect(user.passwordHash).to.match(/:password=testing$/); + + // await importCLI.stop(); + // } finally { + // delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + // } + // }); + // it("should be able to export / import auth data with no users", async function (this) { + // this.timeout(2 * TEST_SETUP_TIMEOUT); + // await new Promise((resolve) => setTimeout(resolve, 2000)); + + // // Start up emulator suite + // const project = FIREBASE_PROJECT || "example"; + // const emulatorsCLI = new CLIProcess("1", __dirname); + + // await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }); + + // // Ask for export (with no users) + // const exportCLI = new CLIProcess("2", __dirname); + // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + // await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes("Export complete"); + // }); + // await exportCLI.stop(); + + // // Stop the suite + // await emulatorsCLI.stop(); + + // // Confirm the data is exported as expected + // const configPath = path.join(exportPath, "auth_export", "config.json"); + // const configData = JSON.parse(fs.readFileSync(configPath).toString()); + // expect(configData).to.deep.equal({ + // signIn: { + // allowDuplicateEmails: false, + // }, + // emailPrivacyConfig: { + // enableImprovedEmailPrivacy: false, + // }, + // }); + + // const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); + // const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); + // expect(accountsData.users).to.have.length(0); + + // // Attempt to import + // const importCLI = new CLIProcess("3", __dirname); + // await importCLI.start( + // "emulators:start", + // project, + // ["--only", "auth", "--import", exportPath, "--debug"], + // (data: unknown) => { + // if (typeof data !== "string" && !Buffer.isBuffer(data)) { + // throw new Error(`data is not a string or buffer (${typeof data})`); + // } + // return data.includes(ALL_EMULATORS_STARTED_LOG); + // }, + // ); + + // await importCLI.stop(); + // }); + + // it("should be able to import/export storage data", async function (this) { + // this.timeout(2 * TEST_SETUP_TIMEOUT); + // await new Promise((resolve) => setTimeout(resolve, 2000)); + + // // Start up emulator suite + // const emulatorsCLI = new CLIProcess("1", __dirname); + // await emulatorsCLI.start( + // "emulators:start", + // FIREBASE_PROJECT, + // ["--only", "storage", "--debug"], + // logIncludes(ALL_EMULATORS_STARTED_LOG), + // ); + + // const credPath = path.join(__dirname, "service-account-key.json"); + // const credential = fs.existsSync(credPath) + // ? admin.credential.cert(credPath) + // : admin.credential.applicationDefault(); + + // const config = readConfig(); + // const port = config.emulators!.storage.port; + // process.env.STORAGE_EMULATOR_HOST = `http://${await localhost()}:${port}`; + + // // Write some data to export + // const aApp = admin.initializeApp( + // { + // projectId: FIREBASE_PROJECT, + // storageBucket: "bucket-a", + // credential, + // }, + // "storage-export-a", + // ); + // const bApp = admin.initializeApp( + // { + // projectId: FIREBASE_PROJECT, + // storageBucket: "bucket-b", + // credential, + // }, + // "storage-export-b", + // ); + + // // Write data to two buckets + // await aApp.storage().bucket().file("a/b.txt").save("a/b hello, world!"); + // await aApp.storage().bucket().file("c/d.txt").save("c/d hello, world!"); + // await bApp.storage().bucket().file("e/f.txt").save("e/f hello, world!"); + // await bApp.storage().bucket().file("g/h.txt").save("g/h hello, world!"); + + // // Ask for export + // const exportCLI = new CLIProcess("2", __dirname); + // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + // await exportCLI.start( + // "emulators:export", + // FIREBASE_PROJECT, + // [exportPath], + // logIncludes("Export complete"), + // ); + // await exportCLI.stop(); + + // // Check that the right export files are created + // const storageExportPath = path.join(exportPath, "storage_export"); + // const storageExportFiles = fs.readdirSync(storageExportPath).sort(); + // expect(storageExportFiles).to.eql(["blobs", "buckets.json", "metadata"]); + + // // Stop the suite + // await emulatorsCLI.stop(); + + // // Attempt to import + // const importCLI = new CLIProcess("3", __dirname); + // await importCLI.start( + // "emulators:start", + // FIREBASE_PROJECT, + // ["--only", "storage", "--import", exportPath, "--debug"], + // logIncludes(ALL_EMULATORS_STARTED_LOG), + // ); + + // // List the files + // const [aFiles] = await aApp.storage().bucket().getFiles({ + // prefix: "a/", + // }); + // const aFileNames = aFiles.map((f) => f.name).sort(); + // expect(aFileNames).to.eql(["a/b.txt"]); + + // const [bFiles] = await bApp.storage().bucket().getFiles({ + // prefix: "e/", + // }); + // const bFileNames = bFiles.map((f) => f.name).sort(); + // expect(bFileNames).to.eql(["e/f.txt"]); + + // // TODO: this operation fails due to a bug in the Storage emulator + // // https://github.com/firebase/firebase-tools/pull/3320 + // // + // // Read a file and check content + // // const [f] = await aApp.storage().bucket().file("a/b.txt").get(); + // // const [buf] = await f.download(); + // // expect(buf.toString()).to.eql("a/b hello, world!"); + + // await importCLI.stop(); + // }); + + + + + it("should automatically export to dataDir when specified in firebase.json", async function (this) { this.timeout(2 * TEST_SETUP_TIMEOUT); await new Promise((resolve) => setTimeout(resolve, 2000)); - // Start up emulator suite - const emulatorsCLI = new CLIProcess("1", __dirname); - await emulatorsCLI.start( - "emulators:start", - FIREBASE_PROJECT, - ["--only", "firestore", "--debug"], - (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }, - ); - - // Ask for export - const exportCLI = new CLIProcess("2", __dirname); - const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes("Export complete"); - }); - await exportCLI.stop(); - - // Stop the suite - await emulatorsCLI.stop(); - - // Attempt to import - const importCLI = new CLIProcess("3", __dirname); - await importCLI.start( - "emulators:start", - FIREBASE_PROJECT, - ["--only", "firestore", "--import", exportPath], - (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }, - ); - - await importCLI.stop(); - - expect(true).to.be.true; - }); + const dataDir = path.join(__dirname, "hello"); // This is taken from firebase.json dataDir - it("should be able to import/export rtdb data", async function (this) { - this.timeout(2 * TEST_SETUP_TIMEOUT); - await new Promise((resolve) => setTimeout(resolve, 2000)); + const firestoreData = { testCollection: { testDoc: { foo: "bar", baz: "buzz" } } }; - // Start up emulator suite - const emulatorsCLI = new CLIProcess("1", __dirname); + // Start emulator - it should export data to dataDir on exit + let emulatorsCLI = new CLIProcess("dataDir-export", __dirname); await emulatorsCLI.start( "emulators:start", - FIREBASE_PROJECT, - ["--only", "database"], - (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }, + "datadir-export-project", + ["--only", "firestore", "--debug"], + logIncludes(ALL_EMULATORS_STARTED_LOG), ); - // Write some data to export - const config = readConfig(); - const port = config.emulators!.database.port; - const host = await localhost(); - const aApp = admin.initializeApp( - { - projectId: FIREBASE_PROJECT, - databaseURL: `http://${host}:${port}?ns=namespace-a`, - credential: ADMIN_CREDENTIAL, - }, - "rtdb-export-a", - ); - const bApp = admin.initializeApp( + // Add data to Firestore + const configForAdmin = readConfig(); // Assuming readConfig() gets Firestore port + const port = configForAdmin.emulators!.firestore.port; + const host = await localhost(); // Make sure localhost is resolved + process.env.FIRESTORE_EMULATOR_HOST = `${host}:${port}`; + const adminApp = admin.initializeApp( { - projectId: FIREBASE_PROJECT, - databaseURL: `http://${host}:${port}?ns=namespace-b`, - credential: ADMIN_CREDENTIAL, + projectId: "datadir-export-project", }, - "rtdb-export-b", - ); - const cApp = admin.initializeApp( - { - projectId: FIREBASE_PROJECT, - databaseURL: `http://${host}:${port}?ns=namespace-c`, - credential: ADMIN_CREDENTIAL, - }, - "rtdb-export-c", + "firestore-dataDir-test-export", ); - // Write to two namespaces - const aRef = aApp.database().ref("ns"); - await aRef.set("namespace-a"); - const bRef = bApp.database().ref("ns"); - await bRef.set("namespace-b"); - - // Read from a third - const cRef = cApp.database().ref("ns"); - await cRef.once("value"); - - // Ask for export - const exportCLI = new CLIProcess("2", __dirname); - const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes("Export complete"); - }); - await exportCLI.stop(); - - // Check that the right export files are created - const dbExportPath = path.join(exportPath, "database_export"); - const dbExportFiles = fs.readdirSync(dbExportPath); - expect(dbExportFiles).to.eql(["namespace-a.json", "namespace-b.json"]); + const db = adminApp.firestore(); + const docRefId = (await db.collection("testCollection").add(firestoreData.testCollection.testDoc)).id; - // Stop the suite + await adminApp.delete(); + // Stop the emulator suite - this should trigger export to dataDir await emulatorsCLI.stop(); - // Attempt to import - const importCLI = new CLIProcess("3", __dirname); - await importCLI.start( - "emulators:start", - FIREBASE_PROJECT, - ["--only", "database", "--import", exportPath, "--export-on-exit"], - (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }, - ); - - // Read the data - const aSnap = await aRef.once("value"); - const bSnap = await bRef.once("value"); - expect(aSnap.val()).to.eql("namespace-a"); - expect(bSnap.val()).to.eql("namespace-b"); - - // Delete all of the import files - for (const f of fs.readdirSync(dbExportPath)) { - const fullPath = path.join(dbExportPath, f); - fs.unlinkSync(fullPath); - } - - // Delete all the data in one namespace - await bApp.database().ref().set(null); + // Verify data was exported to dataDir + // Firestore data is stored in a subdirectory named after the project ID, then 'fs_export_output' + // and then a metadata file and the actual data files. + const firestoreExportMetadataPath = path.join(dataDir, "firestore_export", "firestore_export.overall_export_metadata"); + expect(fs.existsSync(firestoreExportMetadataPath), `Firestore export metadata should exist at ${firestoreExportMetadataPath}`).to.be.true; - // Stop the CLI (which will export on exit) - await importCLI.stop(); - - // Confirm the data exported is as expected - const aPath = path.join(dbExportPath, "namespace-a.json"); - const aData = JSON.parse(fs.readFileSync(aPath).toString()); - expect(aData).to.deep.equal({ ns: "namespace-a" }); - - const bPath = path.join(dbExportPath, "namespace-b.json"); - const bData = JSON.parse(fs.readFileSync(bPath).toString()); - expect(bData).to.equal(null); - }); - - it("should be able to import/export auth data", async function (this) { - this.timeout(2 * TEST_SETUP_TIMEOUT); - await new Promise((resolve) => setTimeout(resolve, 2000)); - - // Start up emulator suite - const project = FIREBASE_PROJECT || "example"; - const emulatorsCLI = new CLIProcess("1", __dirname); - - await emulatorsCLI.start("emulators:start", project, ["--only", "auth"], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }); - - // Create some accounts to export: - const config = readConfig(); - const port = config.emulators!.auth.port; - try { - process.env.FIREBASE_AUTH_EMULATOR_HOST = `${await localhost()}:${port}`; - const adminApp = admin.initializeApp( - { - projectId: project, - credential: ADMIN_CREDENTIAL, - }, - "admin-app", - ); - await adminApp - .auth() - .createUser({ uid: "123", email: "foo@example.com", password: "testing" }); - await adminApp - .auth() - .createUser({ uid: "456", email: "bar@example.com", emailVerified: true }); - - // Ask for export - const exportCLI = new CLIProcess("2", __dirname); - const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes("Export complete"); - }); - await exportCLI.stop(); - - // Stop the suite - await emulatorsCLI.stop(); - - // Confirm the data is exported as expected - const configPath = path.join(exportPath, "auth_export", "config.json"); - const configData = JSON.parse(fs.readFileSync(configPath).toString()); - expect(configData).to.deep.equal({ - signIn: { - allowDuplicateEmails: false, - }, - emailPrivacyConfig: { - enableImprovedEmailPrivacy: false, - }, - }); - - const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); - const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); - expect(accountsData.users).to.have.length(2); - expect(accountsData.users[0]).to.deep.contain({ - localId: "123", - email: "foo@example.com", - emailVerified: false, - providerUserInfo: [ - { - email: "foo@example.com", - federatedId: "foo@example.com", - providerId: "password", - rawId: "foo@example.com", - }, - ], - }); - expect(accountsData.users[0].passwordHash).to.match(/:password=testing$/); - expect(accountsData.users[1]).to.deep.contain({ - localId: "456", - email: "bar@example.com", - emailVerified: true, - }); - - // Attempt to import - const importCLI = new CLIProcess("3", __dirname); - await importCLI.start( - "emulators:start", - project, - ["--only", "auth", "--import", exportPath], - (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }, - ); - - // Check users are indeed imported correctly - const user1 = await adminApp.auth().getUserByEmail("foo@example.com"); - expect(user1.passwordHash).to.match(/:password=testing$/); - const user2 = await adminApp.auth().getUser("456"); - expect(user2.emailVerified).to.be.true; - - await importCLI.stop(); - } finally { - delete process.env.FIREBASE_AUTH_EMULATOR_HOST; - } - }); - - it("should be able to import/export auth data with many users", async function (this) { - this.timeout(2 * TEST_SETUP_TIMEOUT); - await new Promise((resolve) => setTimeout(resolve, 2000)); - - // Start up emulator suite - const project = FIREBASE_PROJECT || "example"; - const emulatorsCLI = new CLIProcess("1", __dirname); - - await emulatorsCLI.start("emulators:start", project, ["--only", "auth"], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }); - - // Create some accounts to export: - const accountCount = 777; // ~120KB data when exported - const config = readConfig(); - const port = config.emulators!.auth.port; - try { - process.env.FIREBASE_AUTH_EMULATOR_HOST = `${await localhost()}:${port}`; - const adminApp = admin.initializeApp( - { - projectId: project, - credential: ADMIN_CREDENTIAL, - }, - "admin-app2", - ); - for (let i = 0; i < accountCount; i++) { - await adminApp - .auth() - .createUser({ uid: `u${i}`, email: `u${i}@example.com`, password: "testing" }); - } - // Ask for export - const exportCLI = new CLIProcess("2", __dirname); - const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes("Export complete"); - }); - await exportCLI.stop(); - - // Stop the suite - await emulatorsCLI.stop(); - - // Confirm the data is exported as expected - const configPath = path.join(exportPath, "auth_export", "config.json"); - const configData = JSON.parse(fs.readFileSync(configPath).toString()); - expect(configData).to.deep.equal({ - signIn: { - allowDuplicateEmails: false, - }, - emailPrivacyConfig: { - enableImprovedEmailPrivacy: false, - }, - }); - - const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); - const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); - expect(accountsData.users).to.have.length(accountCount); - - // Attempt to import - const importCLI = new CLIProcess("3", __dirname); - await importCLI.start( - "emulators:start", - project, - ["--only", "auth", "--import", exportPath], - (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }, - ); - - // Check users are indeed imported correctly - const user = await adminApp.auth().getUserByEmail(`u${accountCount - 1}@example.com`); - expect(user.passwordHash).to.match(/:password=testing$/); - - await importCLI.stop(); - } finally { - delete process.env.FIREBASE_AUTH_EMULATOR_HOST; - } - }); - it("should be able to export / import auth data with no users", async function (this) { - this.timeout(2 * TEST_SETUP_TIMEOUT); - await new Promise((resolve) => setTimeout(resolve, 2000)); - - // Start up emulator suite - const project = FIREBASE_PROJECT || "example"; - const emulatorsCLI = new CLIProcess("1", __dirname); - - await emulatorsCLI.start("emulators:start", project, ["--only", "auth"], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }); - - // Ask for export (with no users) - const exportCLI = new CLIProcess("2", __dirname); - const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes("Export complete"); - }); - await exportCLI.stop(); - - // Stop the suite - await emulatorsCLI.stop(); - - // Confirm the data is exported as expected - const configPath = path.join(exportPath, "auth_export", "config.json"); - const configData = JSON.parse(fs.readFileSync(configPath).toString()); - expect(configData).to.deep.equal({ - signIn: { - allowDuplicateEmails: false, - }, - emailPrivacyConfig: { - enableImprovedEmailPrivacy: false, - }, - }); - - const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); - const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); - expect(accountsData.users).to.have.length(0); - - // Attempt to import - const importCLI = new CLIProcess("3", __dirname); - await importCLI.start( - "emulators:start", - project, - ["--only", "auth", "--import", exportPath], - (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }, - ); - - await importCLI.stop(); - }); - - it("should be able to import/export storage data", async function (this) { - this.timeout(2 * TEST_SETUP_TIMEOUT); - await new Promise((resolve) => setTimeout(resolve, 2000)); - - // Start up emulator suite - const emulatorsCLI = new CLIProcess("1", __dirname); + // Start emulator again - it should automatically import from dataDir + emulatorsCLI = new CLIProcess("dataDir-import", __dirname); await emulatorsCLI.start( "emulators:start", - FIREBASE_PROJECT, - ["--only", "storage"], + "datadir-export-project", + // TODO(christhompsongoogle): Remove the import once imports from dataDir are implemented. + // Also note to self: ensure the import flag supercedes the dataDir property. + ["--only", "firestore", "--debug", "--import", dataDir], logIncludes(ALL_EMULATORS_STARTED_LOG), ); - const credPath = path.join(__dirname, "service-account-key.json"); - const credential = fs.existsSync(credPath) - ? admin.credential.cert(credPath) - : admin.credential.applicationDefault(); - - const config = readConfig(); - const port = config.emulators!.storage.port; - process.env.STORAGE_EMULATOR_HOST = `http://${await localhost()}:${port}`; - - // Write some data to export - const aApp = admin.initializeApp( + // Verify data was imported + const adminAppImport = admin.initializeApp( { - projectId: FIREBASE_PROJECT, - storageBucket: "bucket-a", - credential, + projectId: "datadir-export-project", }, - "storage-export-a", + "firestore-dataDir-test-import", ); - const bApp = admin.initializeApp( - { - projectId: FIREBASE_PROJECT, - storageBucket: "bucket-b", - credential, - }, - "storage-export-b", - ); - - // Write data to two buckets - await aApp.storage().bucket().file("a/b.txt").save("a/b hello, world!"); - await aApp.storage().bucket().file("c/d.txt").save("c/d hello, world!"); - await bApp.storage().bucket().file("e/f.txt").save("e/f hello, world!"); - await bApp.storage().bucket().file("g/h.txt").save("g/h hello, world!"); - - // Ask for export - const exportCLI = new CLIProcess("2", __dirname); - const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - await exportCLI.start( - "emulators:export", - FIREBASE_PROJECT, - [exportPath], - logIncludes("Export complete"), - ); - await exportCLI.stop(); - // Check that the right export files are created - const storageExportPath = path.join(exportPath, "storage_export"); - const storageExportFiles = fs.readdirSync(storageExportPath).sort(); - expect(storageExportFiles).to.eql(["blobs", "buckets.json", "metadata"]); + const dbImport = adminAppImport.firestore(); + const docSnap = await dbImport.collection("testCollection").doc(docRefId).get(); + expect(docSnap.exists, "Document should exist after import from dataDir").to.be.true; + expect(docSnap.data(), "Document data should match after import from dataDir").to.deep.equal(firestoreData.testCollection.testDoc); + await adminAppImport.delete(); - // Stop the suite + // Stop the emulator suite await emulatorsCLI.stop(); - // Attempt to import - const importCLI = new CLIProcess("3", __dirname); - await importCLI.start( - "emulators:start", - FIREBASE_PROJECT, - ["--only", "storage", "--import", exportPath], - logIncludes(ALL_EMULATORS_STARTED_LOG), - ); - - // List the files - const [aFiles] = await aApp.storage().bucket().getFiles({ - prefix: "a/", - }); - const aFileNames = aFiles.map((f) => f.name).sort(); - expect(aFileNames).to.eql(["a/b.txt"]); - - const [bFiles] = await bApp.storage().bucket().getFiles({ - prefix: "e/", - }); - const bFileNames = bFiles.map((f) => f.name).sort(); - expect(bFileNames).to.eql(["e/f.txt"]); - - // TODO: this operation fails due to a bug in the Storage emulator - // https://github.com/firebase/firebase-tools/pull/3320 - // - // Read a file and check content - // const [f] = await aApp.storage().bucket().file("a/b.txt").get(); - // const [buf] = await f.download(); - // expect(buf.toString()).to.eql("a/b hello, world!"); - - await importCLI.stop(); + // Clean up temporary directory + fs.rmSync(dataDir, { recursive: true, force: true }); }); }); diff --git a/src/emulator/controller.ts b/src/emulator/controller.ts index bb0eef4220a..39b9493c192 100755 --- a/src/emulator/controller.ts +++ b/src/emulator/controller.ts @@ -77,11 +77,11 @@ const START_LOGGING_EMULATOR = utils.envOverride( export async function exportOnExit(options: Options): Promise { // Note: options.exportOnExit is coerced to a string before this point in commandUtils.ts#setExportOnExitOptions var exportOnExitDir: string - - if (options.config.src.emulators?.dataDir) { - exportOnExitDir = options.config.src.emulators.dataDir - } else if (options.exportOnExit) { +console.log("wtf"); + if (options.exportOnExit) { // CLI flag takes priority over firebase.json value exportOnExitDir = options.exportOnExit as string; + } else if (options.config.src.emulators?.dataDir) { + exportOnExitDir = options.config.src.emulators.dataDir } else { // If no export directory, skip the export. return; From 77d64823f22a21eae8c0c7d809b291c5bc5fca71 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Thu, 26 Jun 2025 17:54:43 -0700 Subject: [PATCH 3/4] Tests --- scripts/emulator-import-export-tests/tests.ts | 1051 ++++++++--------- src/emulator/controller.ts | 1 - 2 files changed, 524 insertions(+), 528 deletions(-) diff --git a/scripts/emulator-import-export-tests/tests.ts b/scripts/emulator-import-export-tests/tests.ts index bd72021bdfd..cdedbb2b0c2 100644 --- a/scripts/emulator-import-export-tests/tests.ts +++ b/scripts/emulator-import-export-tests/tests.ts @@ -53,533 +53,530 @@ function logIncludes(msg: string) { } describe("import/export end to end", () => { - // it("should be able to import/export firestore data", async function (this) { - // this.timeout(2 * TEST_SETUP_TIMEOUT); - // await new Promise((resolve) => setTimeout(resolve, 2000)); - - // // Start up emulator suite - // const emulatorsCLI = new CLIProcess("1", __dirname); - // await emulatorsCLI.start( - // "emulators:start", - // FIREBASE_PROJECT, - // ["--only", "firestore", "--debug"], - // (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }, - // ); - - // // Ask for export - // const exportCLI = new CLIProcess("2", __dirname); - // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - // await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes("Export complete"); - // }); - // await exportCLI.stop(); - - // // Stop the suite - // await emulatorsCLI.stop(); - - // // Attempt to import - // const importCLI = new CLIProcess("3", __dirname); - // await importCLI.start( - // "emulators:start", - // FIREBASE_PROJECT, - // ["--only", "firestore", "--import", exportPath, "--debug"], - // (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }, - // ); - - // await importCLI.stop(); - - // expect(true).to.be.true; - // }); - - // it("should be able to import/export rtdb data", async function (this) { - // this.timeout(2 * TEST_SETUP_TIMEOUT); - // await new Promise((resolve) => setTimeout(resolve, 2000)); - - // // Start up emulator suite - // const emulatorsCLI = new CLIProcess("1", __dirname); - // await emulatorsCLI.start( - // "emulators:start", - // FIREBASE_PROJECT, - // ["--only", "database", "--debug"], - // (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }, - // ); - - // // Write some data to export - // const config = readConfig(); - // const port = config.emulators!.database.port; - // const host = await localhost(); - // const aApp = admin.initializeApp( - // { - // projectId: FIREBASE_PROJECT, - // databaseURL: `http://${host}:${port}?ns=namespace-a`, - // credential: ADMIN_CREDENTIAL, - // }, - // "rtdb-export-a", - // ); - // const bApp = admin.initializeApp( - // { - // projectId: FIREBASE_PROJECT, - // databaseURL: `http://${host}:${port}?ns=namespace-b`, - // credential: ADMIN_CREDENTIAL, - // }, - // "rtdb-export-b", - // ); - // const cApp = admin.initializeApp( - // { - // projectId: FIREBASE_PROJECT, - // databaseURL: `http://${host}:${port}?ns=namespace-c`, - // credential: ADMIN_CREDENTIAL, - // }, - // "rtdb-export-c", - // ); - - // // Write to two namespaces - // const aRef = aApp.database().ref("ns"); - // await aRef.set("namespace-a"); - // const bRef = bApp.database().ref("ns"); - // await bRef.set("namespace-b"); - - // // Read from a third - // const cRef = cApp.database().ref("ns"); - // await cRef.once("value"); - - // // Ask for export - // const exportCLI = new CLIProcess("2", __dirname); - // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - // await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes("Export complete"); - // }); - // await exportCLI.stop(); - - // // Check that the right export files are created - // const dbExportPath = path.join(exportPath, "database_export"); - // const dbExportFiles = fs.readdirSync(dbExportPath); - // expect(dbExportFiles).to.eql(["namespace-a.json", "namespace-b.json"]); - - // // Stop the suite - // await emulatorsCLI.stop(); - - // // Attempt to import - // const importCLI = new CLIProcess("3", __dirname); - // await importCLI.start( - // "emulators:start", - // FIREBASE_PROJECT, - // ["--only", "database", "--import", exportPath, "--export-on-exit", "--debug"], - // (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }, - // ); - - // // Read the data - // const aSnap = await aRef.once("value"); - // const bSnap = await bRef.once("value"); - // expect(aSnap.val()).to.eql("namespace-a"); - // expect(bSnap.val()).to.eql("namespace-b"); - - // // Delete all of the import files - // for (const f of fs.readdirSync(dbExportPath)) { - // const fullPath = path.join(dbExportPath, f); - // fs.unlinkSync(fullPath); - // } - - // // Delete all the data in one namespace - // await bApp.database().ref().set(null); - - // // Stop the CLI (which will export on exit) - // await importCLI.stop(); - - // // Confirm the data exported is as expected - // const aPath = path.join(dbExportPath, "namespace-a.json"); - // const aData = JSON.parse(fs.readFileSync(aPath).toString()); - // expect(aData).to.deep.equal({ ns: "namespace-a" }); - - // const bPath = path.join(dbExportPath, "namespace-b.json"); - // const bData = JSON.parse(fs.readFileSync(bPath).toString()); - // expect(bData).to.equal(null); - // }); - - // it("should be able to import/export auth data", async function (this) { - // this.timeout(2 * TEST_SETUP_TIMEOUT); - // await new Promise((resolve) => setTimeout(resolve, 2000)); - - // // Start up emulator suite - // const project = FIREBASE_PROJECT || "example"; - // const emulatorsCLI = new CLIProcess("1", __dirname); - - // await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }); - - // // Create some accounts to export: - // const config = readConfig(); - // const port = config.emulators!.auth.port; - // try { - // process.env.FIREBASE_AUTH_EMULATOR_HOST = `${await localhost()}:${port}`; - // const adminApp = admin.initializeApp( - // { - // projectId: project, - // credential: ADMIN_CREDENTIAL, - // }, - // "admin-app", - // ); - // await adminApp - // .auth() - // .createUser({ uid: "123", email: "foo@example.com", password: "testing" }); - // await adminApp - // .auth() - // .createUser({ uid: "456", email: "bar@example.com", emailVerified: true }); - - // // Ask for export - // const exportCLI = new CLIProcess("2", __dirname); - // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - // await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes("Export complete"); - // }); - // await exportCLI.stop(); - - // // Stop the suite - // await emulatorsCLI.stop(); - - // // Confirm the data is exported as expected - // const configPath = path.join(exportPath, "auth_export", "config.json"); - // const configData = JSON.parse(fs.readFileSync(configPath).toString()); - // expect(configData).to.deep.equal({ - // signIn: { - // allowDuplicateEmails: false, - // }, - // emailPrivacyConfig: { - // enableImprovedEmailPrivacy: false, - // }, - // }); - - // const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); - // const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); - // expect(accountsData.users).to.have.length(2); - // expect(accountsData.users[0]).to.deep.contain({ - // localId: "123", - // email: "foo@example.com", - // emailVerified: false, - // providerUserInfo: [ - // { - // email: "foo@example.com", - // federatedId: "foo@example.com", - // providerId: "password", - // rawId: "foo@example.com", - // }, - // ], - // }); - // expect(accountsData.users[0].passwordHash).to.match(/:password=testing$/); - // expect(accountsData.users[1]).to.deep.contain({ - // localId: "456", - // email: "bar@example.com", - // emailVerified: true, - // }); - - // // Attempt to import - // const importCLI = new CLIProcess("3", __dirname); - // await importCLI.start( - // "emulators:start", - // project, - // ["--only", "auth", "--import", exportPath, "--debug"], - // (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }, - // ); - - // // Check users are indeed imported correctly - // const user1 = await adminApp.auth().getUserByEmail("foo@example.com"); - // expect(user1.passwordHash).to.match(/:password=testing$/); - // const user2 = await adminApp.auth().getUser("456"); - // expect(user2.emailVerified).to.be.true; - - // await importCLI.stop(); - // } finally { - // delete process.env.FIREBASE_AUTH_EMULATOR_HOST; - // } - // }); - - // it("should be able to import/export auth data with many users", async function (this) { - // this.timeout(2 * TEST_SETUP_TIMEOUT); - // await new Promise((resolve) => setTimeout(resolve, 2000)); - - // // Start up emulator suite - // const project = FIREBASE_PROJECT || "example"; - // const emulatorsCLI = new CLIProcess("1", __dirname); - - // await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }); - - // // Create some accounts to export: - // const accountCount = 777; // ~120KB data when exported - // const config = readConfig(); - // const port = config.emulators!.auth.port; - // try { - // process.env.FIREBASE_AUTH_EMULATOR_HOST = `${await localhost()}:${port}`; - // const adminApp = admin.initializeApp( - // { - // projectId: project, - // credential: ADMIN_CREDENTIAL, - // }, - // "admin-app2", - // ); - // for (let i = 0; i < accountCount; i++) { - // await adminApp - // .auth() - // .createUser({ uid: `u${i}`, email: `u${i}@example.com`, password: "testing" }); - // } - // // Ask for export - // const exportCLI = new CLIProcess("2", __dirname); - // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - // await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes("Export complete"); - // }); - // await exportCLI.stop(); - - // // Stop the suite - // await emulatorsCLI.stop(); - - // // Confirm the data is exported as expected - // const configPath = path.join(exportPath, "auth_export", "config.json"); - // const configData = JSON.parse(fs.readFileSync(configPath).toString()); - // expect(configData).to.deep.equal({ - // signIn: { - // allowDuplicateEmails: false, - // }, - // emailPrivacyConfig: { - // enableImprovedEmailPrivacy: false, - // }, - // }); - - // const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); - // const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); - // expect(accountsData.users).to.have.length(accountCount); - - // // Attempt to import - // const importCLI = new CLIProcess("3", __dirname); - // await importCLI.start( - // "emulators:start", - // project, - // ["--only", "auth", "--import", exportPath, "--debug"], - // (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }, - // ); - - // // Check users are indeed imported correctly - // const user = await adminApp.auth().getUserByEmail(`u${accountCount - 1}@example.com`); - // expect(user.passwordHash).to.match(/:password=testing$/); - - // await importCLI.stop(); - // } finally { - // delete process.env.FIREBASE_AUTH_EMULATOR_HOST; - // } - // }); - // it("should be able to export / import auth data with no users", async function (this) { - // this.timeout(2 * TEST_SETUP_TIMEOUT); - // await new Promise((resolve) => setTimeout(resolve, 2000)); - - // // Start up emulator suite - // const project = FIREBASE_PROJECT || "example"; - // const emulatorsCLI = new CLIProcess("1", __dirname); - - // await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }); - - // // Ask for export (with no users) - // const exportCLI = new CLIProcess("2", __dirname); - // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - // await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes("Export complete"); - // }); - // await exportCLI.stop(); - - // // Stop the suite - // await emulatorsCLI.stop(); - - // // Confirm the data is exported as expected - // const configPath = path.join(exportPath, "auth_export", "config.json"); - // const configData = JSON.parse(fs.readFileSync(configPath).toString()); - // expect(configData).to.deep.equal({ - // signIn: { - // allowDuplicateEmails: false, - // }, - // emailPrivacyConfig: { - // enableImprovedEmailPrivacy: false, - // }, - // }); - - // const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); - // const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); - // expect(accountsData.users).to.have.length(0); - - // // Attempt to import - // const importCLI = new CLIProcess("3", __dirname); - // await importCLI.start( - // "emulators:start", - // project, - // ["--only", "auth", "--import", exportPath, "--debug"], - // (data: unknown) => { - // if (typeof data !== "string" && !Buffer.isBuffer(data)) { - // throw new Error(`data is not a string or buffer (${typeof data})`); - // } - // return data.includes(ALL_EMULATORS_STARTED_LOG); - // }, - // ); - - // await importCLI.stop(); - // }); - - // it("should be able to import/export storage data", async function (this) { - // this.timeout(2 * TEST_SETUP_TIMEOUT); - // await new Promise((resolve) => setTimeout(resolve, 2000)); - - // // Start up emulator suite - // const emulatorsCLI = new CLIProcess("1", __dirname); - // await emulatorsCLI.start( - // "emulators:start", - // FIREBASE_PROJECT, - // ["--only", "storage", "--debug"], - // logIncludes(ALL_EMULATORS_STARTED_LOG), - // ); - - // const credPath = path.join(__dirname, "service-account-key.json"); - // const credential = fs.existsSync(credPath) - // ? admin.credential.cert(credPath) - // : admin.credential.applicationDefault(); - - // const config = readConfig(); - // const port = config.emulators!.storage.port; - // process.env.STORAGE_EMULATOR_HOST = `http://${await localhost()}:${port}`; - - // // Write some data to export - // const aApp = admin.initializeApp( - // { - // projectId: FIREBASE_PROJECT, - // storageBucket: "bucket-a", - // credential, - // }, - // "storage-export-a", - // ); - // const bApp = admin.initializeApp( - // { - // projectId: FIREBASE_PROJECT, - // storageBucket: "bucket-b", - // credential, - // }, - // "storage-export-b", - // ); - - // // Write data to two buckets - // await aApp.storage().bucket().file("a/b.txt").save("a/b hello, world!"); - // await aApp.storage().bucket().file("c/d.txt").save("c/d hello, world!"); - // await bApp.storage().bucket().file("e/f.txt").save("e/f hello, world!"); - // await bApp.storage().bucket().file("g/h.txt").save("g/h hello, world!"); - - // // Ask for export - // const exportCLI = new CLIProcess("2", __dirname); - // const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); - // await exportCLI.start( - // "emulators:export", - // FIREBASE_PROJECT, - // [exportPath], - // logIncludes("Export complete"), - // ); - // await exportCLI.stop(); - - // // Check that the right export files are created - // const storageExportPath = path.join(exportPath, "storage_export"); - // const storageExportFiles = fs.readdirSync(storageExportPath).sort(); - // expect(storageExportFiles).to.eql(["blobs", "buckets.json", "metadata"]); - - // // Stop the suite - // await emulatorsCLI.stop(); - - // // Attempt to import - // const importCLI = new CLIProcess("3", __dirname); - // await importCLI.start( - // "emulators:start", - // FIREBASE_PROJECT, - // ["--only", "storage", "--import", exportPath, "--debug"], - // logIncludes(ALL_EMULATORS_STARTED_LOG), - // ); - - // // List the files - // const [aFiles] = await aApp.storage().bucket().getFiles({ - // prefix: "a/", - // }); - // const aFileNames = aFiles.map((f) => f.name).sort(); - // expect(aFileNames).to.eql(["a/b.txt"]); - - // const [bFiles] = await bApp.storage().bucket().getFiles({ - // prefix: "e/", - // }); - // const bFileNames = bFiles.map((f) => f.name).sort(); - // expect(bFileNames).to.eql(["e/f.txt"]); - - // // TODO: this operation fails due to a bug in the Storage emulator - // // https://github.com/firebase/firebase-tools/pull/3320 - // // - // // Read a file and check content - // // const [f] = await aApp.storage().bucket().file("a/b.txt").get(); - // // const [buf] = await f.download(); - // // expect(buf.toString()).to.eql("a/b hello, world!"); - - // await importCLI.stop(); - // }); - - - - + it("should be able to import/export firestore data", async function (this) { + this.timeout(2 * TEST_SETUP_TIMEOUT); + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Start up emulator suite + const emulatorsCLI = new CLIProcess("1", __dirname); + await emulatorsCLI.start( + "emulators:start", + FIREBASE_PROJECT, + ["--only", "firestore", "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); + + // Ask for export + const exportCLI = new CLIProcess("2", __dirname); + const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes("Export complete"); + }); + await exportCLI.stop(); + + // Stop the suite + await emulatorsCLI.stop(); + + // Attempt to import + const importCLI = new CLIProcess("3", __dirname); + await importCLI.start( + "emulators:start", + FIREBASE_PROJECT, + ["--only", "firestore", "--import", exportPath, "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); + + await importCLI.stop(); + + expect(true).to.be.true; + }); + + it("should be able to import/export rtdb data", async function (this) { + this.timeout(2 * TEST_SETUP_TIMEOUT); + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Start up emulator suite + const emulatorsCLI = new CLIProcess("1", __dirname); + await emulatorsCLI.start( + "emulators:start", + FIREBASE_PROJECT, + ["--only", "database", "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); + + // Write some data to export + const config = readConfig(); + const port = config.emulators!.database.port; + const host = await localhost(); + const aApp = admin.initializeApp( + { + projectId: FIREBASE_PROJECT, + databaseURL: `http://${host}:${port}?ns=namespace-a`, + credential: ADMIN_CREDENTIAL, + }, + "rtdb-export-a", + ); + const bApp = admin.initializeApp( + { + projectId: FIREBASE_PROJECT, + databaseURL: `http://${host}:${port}?ns=namespace-b`, + credential: ADMIN_CREDENTIAL, + }, + "rtdb-export-b", + ); + const cApp = admin.initializeApp( + { + projectId: FIREBASE_PROJECT, + databaseURL: `http://${host}:${port}?ns=namespace-c`, + credential: ADMIN_CREDENTIAL, + }, + "rtdb-export-c", + ); + + // Write to two namespaces + const aRef = aApp.database().ref("ns"); + await aRef.set("namespace-a"); + const bRef = bApp.database().ref("ns"); + await bRef.set("namespace-b"); + + // Read from a third + const cRef = cApp.database().ref("ns"); + await cRef.once("value"); + + // Ask for export + const exportCLI = new CLIProcess("2", __dirname); + const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes("Export complete"); + }); + await exportCLI.stop(); + + // Check that the right export files are created + const dbExportPath = path.join(exportPath, "database_export"); + const dbExportFiles = fs.readdirSync(dbExportPath); + expect(dbExportFiles).to.eql(["namespace-a.json", "namespace-b.json"]); + + // Stop the suite + await emulatorsCLI.stop(); + + // Attempt to import + const importCLI = new CLIProcess("3", __dirname); + await importCLI.start( + "emulators:start", + FIREBASE_PROJECT, + ["--only", "database", "--import", exportPath, "--export-on-exit", "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); + + // Read the data + const aSnap = await aRef.once("value"); + const bSnap = await bRef.once("value"); + expect(aSnap.val()).to.eql("namespace-a"); + expect(bSnap.val()).to.eql("namespace-b"); + + // Delete all of the import files + for (const f of fs.readdirSync(dbExportPath)) { + const fullPath = path.join(dbExportPath, f); + fs.unlinkSync(fullPath); + } + + // Delete all the data in one namespace + await bApp.database().ref().set(null); + + // Stop the CLI (which will export on exit) + await importCLI.stop(); + + // Confirm the data exported is as expected + const aPath = path.join(dbExportPath, "namespace-a.json"); + const aData = JSON.parse(fs.readFileSync(aPath).toString()); + expect(aData).to.deep.equal({ ns: "namespace-a" }); + + const bPath = path.join(dbExportPath, "namespace-b.json"); + const bData = JSON.parse(fs.readFileSync(bPath).toString()); + expect(bData).to.equal(null); + }); + + it("should be able to import/export auth data", async function (this) { + this.timeout(2 * TEST_SETUP_TIMEOUT); + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Start up emulator suite + const project = FIREBASE_PROJECT || "example"; + const emulatorsCLI = new CLIProcess("1", __dirname); + + await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }); + + // Create some accounts to export: + const config = readConfig(); + const port = config.emulators!.auth.port; + try { + process.env.FIREBASE_AUTH_EMULATOR_HOST = `${await localhost()}:${port}`; + const adminApp = admin.initializeApp( + { + projectId: project, + credential: ADMIN_CREDENTIAL, + }, + "admin-app", + ); + await adminApp + .auth() + .createUser({ uid: "123", email: "foo@example.com", password: "testing" }); + await adminApp + .auth() + .createUser({ uid: "456", email: "bar@example.com", emailVerified: true }); + + // Ask for export + const exportCLI = new CLIProcess("2", __dirname); + const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes("Export complete"); + }); + await exportCLI.stop(); + + // Stop the suite + await emulatorsCLI.stop(); + + // Confirm the data is exported as expected + const configPath = path.join(exportPath, "auth_export", "config.json"); + const configData = JSON.parse(fs.readFileSync(configPath).toString()); + expect(configData).to.deep.equal({ + signIn: { + allowDuplicateEmails: false, + }, + emailPrivacyConfig: { + enableImprovedEmailPrivacy: false, + }, + }); + + const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); + const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); + expect(accountsData.users).to.have.length(2); + expect(accountsData.users[0]).to.deep.contain({ + localId: "123", + email: "foo@example.com", + emailVerified: false, + providerUserInfo: [ + { + email: "foo@example.com", + federatedId: "foo@example.com", + providerId: "password", + rawId: "foo@example.com", + }, + ], + }); + expect(accountsData.users[0].passwordHash).to.match(/:password=testing$/); + expect(accountsData.users[1]).to.deep.contain({ + localId: "456", + email: "bar@example.com", + emailVerified: true, + }); + + // Attempt to import + const importCLI = new CLIProcess("3", __dirname); + await importCLI.start( + "emulators:start", + project, + ["--only", "auth", "--import", exportPath, "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); + + // Check users are indeed imported correctly + const user1 = await adminApp.auth().getUserByEmail("foo@example.com"); + expect(user1.passwordHash).to.match(/:password=testing$/); + const user2 = await adminApp.auth().getUser("456"); + expect(user2.emailVerified).to.be.true; + + await importCLI.stop(); + } finally { + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + } + }); + + it("should be able to import/export auth data with many users", async function (this) { + this.timeout(2 * TEST_SETUP_TIMEOUT); + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Start up emulator suite + const project = FIREBASE_PROJECT || "example"; + const emulatorsCLI = new CLIProcess("1", __dirname); + + await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }); + + // Create some accounts to export: + const accountCount = 777; // ~120KB data when exported + const config = readConfig(); + const port = config.emulators!.auth.port; + try { + process.env.FIREBASE_AUTH_EMULATOR_HOST = `${await localhost()}:${port}`; + const adminApp = admin.initializeApp( + { + projectId: project, + credential: ADMIN_CREDENTIAL, + }, + "admin-app2", + ); + for (let i = 0; i < accountCount; i++) { + await adminApp + .auth() + .createUser({ uid: `u${i}`, email: `u${i}@example.com`, password: "testing" }); + } + // Ask for export + const exportCLI = new CLIProcess("2", __dirname); + const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes("Export complete"); + }); + await exportCLI.stop(); + + // Stop the suite + await emulatorsCLI.stop(); + + // Confirm the data is exported as expected + const configPath = path.join(exportPath, "auth_export", "config.json"); + const configData = JSON.parse(fs.readFileSync(configPath).toString()); + expect(configData).to.deep.equal({ + signIn: { + allowDuplicateEmails: false, + }, + emailPrivacyConfig: { + enableImprovedEmailPrivacy: false, + }, + }); + + const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); + const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); + expect(accountsData.users).to.have.length(accountCount); + + // Attempt to import + const importCLI = new CLIProcess("3", __dirname); + await importCLI.start( + "emulators:start", + project, + ["--only", "auth", "--import", exportPath, "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); + + // Check users are indeed imported correctly + const user = await adminApp.auth().getUserByEmail(`u${accountCount - 1}@example.com`); + expect(user.passwordHash).to.match(/:password=testing$/); + + await importCLI.stop(); + } finally { + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + } + }); + it("should be able to export / import auth data with no users", async function (this) { + this.timeout(2 * TEST_SETUP_TIMEOUT); + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Start up emulator suite + const project = FIREBASE_PROJECT || "example"; + const emulatorsCLI = new CLIProcess("1", __dirname); + + await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }); + + // Ask for export (with no users) + const exportCLI = new CLIProcess("2", __dirname); + const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes("Export complete"); + }); + await exportCLI.stop(); + + // Stop the suite + await emulatorsCLI.stop(); + + // Confirm the data is exported as expected + const configPath = path.join(exportPath, "auth_export", "config.json"); + const configData = JSON.parse(fs.readFileSync(configPath).toString()); + expect(configData).to.deep.equal({ + signIn: { + allowDuplicateEmails: false, + }, + emailPrivacyConfig: { + enableImprovedEmailPrivacy: false, + }, + }); + + const accountsPath = path.join(exportPath, "auth_export", "accounts.json"); + const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString()); + expect(accountsData.users).to.have.length(0); + + // Attempt to import + const importCLI = new CLIProcess("3", __dirname); + await importCLI.start( + "emulators:start", + project, + ["--only", "auth", "--import", exportPath, "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); + + await importCLI.stop(); + }); + + it("should be able to import/export storage data", async function (this) { + this.timeout(2 * TEST_SETUP_TIMEOUT); + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Start up emulator suite + const emulatorsCLI = new CLIProcess("1", __dirname); + await emulatorsCLI.start( + "emulators:start", + FIREBASE_PROJECT, + ["--only", "storage", "--debug"], + logIncludes(ALL_EMULATORS_STARTED_LOG), + ); + + const credPath = path.join(__dirname, "service-account-key.json"); + const credential = fs.existsSync(credPath) + ? admin.credential.cert(credPath) + : admin.credential.applicationDefault(); + + const config = readConfig(); + const port = config.emulators!.storage.port; + process.env.STORAGE_EMULATOR_HOST = `http://${await localhost()}:${port}`; + + // Write some data to export + const aApp = admin.initializeApp( + { + projectId: FIREBASE_PROJECT, + storageBucket: "bucket-a", + credential, + }, + "storage-export-a", + ); + const bApp = admin.initializeApp( + { + projectId: FIREBASE_PROJECT, + storageBucket: "bucket-b", + credential, + }, + "storage-export-b", + ); + + // Write data to two buckets + await aApp.storage().bucket().file("a/b.txt").save("a/b hello, world!"); + await aApp.storage().bucket().file("c/d.txt").save("c/d hello, world!"); + await bApp.storage().bucket().file("e/f.txt").save("e/f hello, world!"); + await bApp.storage().bucket().file("g/h.txt").save("g/h hello, world!"); + + // Ask for export + const exportCLI = new CLIProcess("2", __dirname); + const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data")); + await exportCLI.start( + "emulators:export", + FIREBASE_PROJECT, + [exportPath], + logIncludes("Export complete"), + ); + await exportCLI.stop(); + + // Check that the right export files are created + const storageExportPath = path.join(exportPath, "storage_export"); + const storageExportFiles = fs.readdirSync(storageExportPath).sort(); + expect(storageExportFiles).to.eql(["blobs", "buckets.json", "metadata"]); + + // Stop the suite + await emulatorsCLI.stop(); + + // Attempt to import + const importCLI = new CLIProcess("3", __dirname); + await importCLI.start( + "emulators:start", + FIREBASE_PROJECT, + ["--only", "storage", "--import", exportPath, "--debug"], + logIncludes(ALL_EMULATORS_STARTED_LOG), + ); + + // List the files + const [aFiles] = await aApp.storage().bucket().getFiles({ + prefix: "a/", + }); + const aFileNames = aFiles.map((f) => f.name).sort(); + expect(aFileNames).to.eql(["a/b.txt"]); + + const [bFiles] = await bApp.storage().bucket().getFiles({ + prefix: "e/", + }); + const bFileNames = bFiles.map((f) => f.name).sort(); + expect(bFileNames).to.eql(["e/f.txt"]); + + // TODO: this operation fails due to a bug in the Storage emulator + // https://github.com/firebase/firebase-tools/pull/3320 + // + // Read a file and check content + // const [f] = await aApp.storage().bucket().file("a/b.txt").get(); + // const [buf] = await f.download(); + // expect(buf.toString()).to.eql("a/b hello, world!"); + + await importCLI.stop(); + }); + it("should automatically export to dataDir when specified in firebase.json", async function (this) { this.timeout(2 * TEST_SETUP_TIMEOUT); await new Promise((resolve) => setTimeout(resolve, 2000)); diff --git a/src/emulator/controller.ts b/src/emulator/controller.ts index 39b9493c192..74c3f215da3 100755 --- a/src/emulator/controller.ts +++ b/src/emulator/controller.ts @@ -77,7 +77,6 @@ const START_LOGGING_EMULATOR = utils.envOverride( export async function exportOnExit(options: Options): Promise { // Note: options.exportOnExit is coerced to a string before this point in commandUtils.ts#setExportOnExitOptions var exportOnExitDir: string -console.log("wtf"); if (options.exportOnExit) { // CLI flag takes priority over firebase.json value exportOnExitDir = options.exportOnExit as string; } else if (options.config.src.emulators?.dataDir) { From 4adc2037eee6002fa2ad021297d252d5babec254 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Thu, 26 Jun 2025 17:57:02 -0700 Subject: [PATCH 4/4] Lint --- scripts/emulator-import-export-tests/tests.ts | 70 +++++++++++++------ src/emulator/controller.ts | 9 +-- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/scripts/emulator-import-export-tests/tests.ts b/scripts/emulator-import-export-tests/tests.ts index cdedbb2b0c2..e1d88e8e39a 100644 --- a/scripts/emulator-import-export-tests/tests.ts +++ b/scripts/emulator-import-export-tests/tests.ts @@ -230,12 +230,17 @@ describe("import/export end to end", () => { const project = FIREBASE_PROJECT || "example"; const emulatorsCLI = new CLIProcess("1", __dirname); - await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }); + await emulatorsCLI.start( + "emulators:start", + project, + ["--only", "auth", "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); // Create some accounts to export: const config = readConfig(); @@ -339,12 +344,17 @@ describe("import/export end to end", () => { const project = FIREBASE_PROJECT || "example"; const emulatorsCLI = new CLIProcess("1", __dirname); - await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }); + await emulatorsCLI.start( + "emulators:start", + project, + ["--only", "auth", "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); // Create some accounts to export: const accountCount = 777; // ~120KB data when exported @@ -425,12 +435,17 @@ describe("import/export end to end", () => { const project = FIREBASE_PROJECT || "example"; const emulatorsCLI = new CLIProcess("1", __dirname); - await emulatorsCLI.start("emulators:start", project, ["--only", "auth", "--debug"], (data: unknown) => { - if (typeof data !== "string" && !Buffer.isBuffer(data)) { - throw new Error(`data is not a string or buffer (${typeof data})`); - } - return data.includes(ALL_EMULATORS_STARTED_LOG); - }); + await emulatorsCLI.start( + "emulators:start", + project, + ["--only", "auth", "--debug"], + (data: unknown) => { + if (typeof data !== "string" && !Buffer.isBuffer(data)) { + throw new Error(`data is not a string or buffer (${typeof data})`); + } + return data.includes(ALL_EMULATORS_STARTED_LOG); + }, + ); // Ask for export (with no users) const exportCLI = new CLIProcess("2", __dirname); @@ -607,7 +622,9 @@ describe("import/export end to end", () => { ); const db = adminApp.firestore(); - const docRefId = (await db.collection("testCollection").add(firestoreData.testCollection.testDoc)).id; + const docRefId = ( + await db.collection("testCollection").add(firestoreData.testCollection.testDoc) + ).id; await adminApp.delete(); // Stop the emulator suite - this should trigger export to dataDir @@ -616,8 +633,15 @@ describe("import/export end to end", () => { // Verify data was exported to dataDir // Firestore data is stored in a subdirectory named after the project ID, then 'fs_export_output' // and then a metadata file and the actual data files. - const firestoreExportMetadataPath = path.join(dataDir, "firestore_export", "firestore_export.overall_export_metadata"); - expect(fs.existsSync(firestoreExportMetadataPath), `Firestore export metadata should exist at ${firestoreExportMetadataPath}`).to.be.true; + const firestoreExportMetadataPath = path.join( + dataDir, + "firestore_export", + "firestore_export.overall_export_metadata", + ); + expect( + fs.existsSync(firestoreExportMetadataPath), + `Firestore export metadata should exist at ${firestoreExportMetadataPath}`, + ).to.be.true; // Start emulator again - it should automatically import from dataDir emulatorsCLI = new CLIProcess("dataDir-import", __dirname); @@ -641,7 +665,9 @@ describe("import/export end to end", () => { const dbImport = adminAppImport.firestore(); const docSnap = await dbImport.collection("testCollection").doc(docRefId).get(); expect(docSnap.exists, "Document should exist after import from dataDir").to.be.true; - expect(docSnap.data(), "Document data should match after import from dataDir").to.deep.equal(firestoreData.testCollection.testDoc); + expect(docSnap.data(), "Document data should match after import from dataDir").to.deep.equal( + firestoreData.testCollection.testDoc, + ); await adminAppImport.delete(); // Stop the emulator suite diff --git a/src/emulator/controller.ts b/src/emulator/controller.ts index 74c3f215da3..d40ffd34c13 100755 --- a/src/emulator/controller.ts +++ b/src/emulator/controller.ts @@ -76,16 +76,17 @@ const START_LOGGING_EMULATOR = utils.envOverride( */ export async function exportOnExit(options: Options): Promise { // Note: options.exportOnExit is coerced to a string before this point in commandUtils.ts#setExportOnExitOptions - var exportOnExitDir: string - if (options.exportOnExit) { // CLI flag takes priority over firebase.json value + let exportOnExitDir: string; + if (options.exportOnExit) { + // CLI flag takes priority over firebase.json value exportOnExitDir = options.exportOnExit as string; } else if (options.config.src.emulators?.dataDir) { - exportOnExitDir = options.config.src.emulators.dataDir + exportOnExitDir = options.config.src.emulators.dataDir; } else { // If no export directory, skip the export. return; } - + try { utils.logBullet( `Automatically exporting data using ${FLAG_EXPORT_ON_EXIT_NAME} "${exportOnExitDir}" ` +