From 0779305caaa4e69ef854745b3cc7a5d62d72e287 Mon Sep 17 00:00:00 2001 From: Alona Enraght-Moony Date: Tue, 17 Jun 2025 20:25:11 +0000 Subject: [PATCH] rustdoc-json: Postcard output --- Cargo.lock | 33 ++++++++++++ src/librustdoc/Cargo.toml | 1 + src/librustdoc/config.rs | 9 +++- src/librustdoc/json/mod.rs | 47 +++++++++++++++-- src/librustdoc/lib.rs | 7 +-- src/rustdoc-json-types/lib.rs | 13 +++++ src/rustdoc-json-types/tests.rs | 8 +++ src/tools/compiletest/src/runtest.rs | 30 +++++++++-- src/tools/compiletest/src/runtest/js_doc.rs | 4 +- src/tools/compiletest/src/runtest/rustdoc.rs | 3 +- .../compiletest/src/runtest/rustdoc_json.rs | 13 +++-- src/tools/jsondoclint/Cargo.toml | 2 + src/tools/jsondoclint/src/main.rs | 51 ++++++++++++++----- 13 files changed, 187 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df2842bddb386..87384c5075bb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,6 +630,12 @@ dependencies = [ "serde", ] +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + [[package]] name = "collect-license-metadata" version = "0.1.0" @@ -1096,6 +1102,18 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "ena" version = "0.14.3" @@ -1992,8 +2010,10 @@ name = "jsondoclint" version = "0.1.0" dependencies = [ "anyhow", + "camino", "clap", "fs-err", + "postcard", "rustc-hash 2.1.1", "rustdoc-json-types", "serde", @@ -2821,6 +2841,18 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.2" @@ -4661,6 +4693,7 @@ dependencies = [ "indexmap", "itertools", "minifier", + "postcard", "pulldown-cmark-escape", "regex", "rustdoc-json-types", diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index bba8e630bcc2b..815d35e8695a0 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -25,6 +25,7 @@ tracing = "0.1" tracing-tree = "0.3.0" threadpool = "1.8.1" unicode-segmentation = "1.9" +postcard = { version = "1.1.1", default-features = false, features = ["use-std"] } [dependencies.tracing-subscriber] version = "0.3.3" diff --git a/src/librustdoc/config.rs b/src/librustdoc/config.rs index f93aa8ffd0de9..4fd6623eef978 100644 --- a/src/librustdoc/config.rs +++ b/src/librustdoc/config.rs @@ -31,6 +31,7 @@ use crate::{html, opts, theme}; #[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] pub(crate) enum OutputFormat { Json, + Postcard, #[default] Html, Doctest, @@ -38,7 +39,7 @@ pub(crate) enum OutputFormat { impl OutputFormat { pub(crate) fn is_json(&self) -> bool { - matches!(self, OutputFormat::Json) + matches!(self, OutputFormat::Json | OutputFormat::Postcard) } } @@ -50,6 +51,7 @@ impl TryFrom<&str> for OutputFormat { "json" => Ok(OutputFormat::Json), "html" => Ok(OutputFormat::Html), "doctest" => Ok(OutputFormat::Doctest), + "postcard" => Ok(OutputFormat::Postcard), _ => Err(format!("unknown output format `{value}`")), } } @@ -305,6 +307,8 @@ pub(crate) struct RenderOptions { pub(crate) parts_out_dir: Option, /// disable minification of CSS/JS pub(crate) disable_minification: bool, + + pub(crate) output_format: OutputFormat, } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -488,7 +492,7 @@ impl Options { // If `-Zunstable-options` is used, nothing to check after this point. (_, false, true) => {} (None | Some(OutputFormat::Html), false, _) => {} - (Some(OutputFormat::Json), false, false) => { + (Some(OutputFormat::Json | OutputFormat::Postcard), false, false) => { dcx.fatal( "the -Z unstable-options flag must be passed to enable --output-format for documentation generation (see https://github.com/rust-lang/rust/issues/76578)", ); @@ -886,6 +890,7 @@ impl Options { include_parts_dir, parts_out_dir, disable_minification, + output_format, }; Some((input, options, render_options)) } diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index 2feadce26d09f..5ceb9f8cbc192 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -37,6 +37,12 @@ use crate::formats::cache::Cache; use crate::json::conversions::IntoJson; use crate::{clean, try_err}; +#[derive(Clone, Copy, Debug)] +pub(crate) enum OutputFormat { + Json, + Postcard, +} + pub(crate) struct JsonRenderer<'tcx> { tcx: TyCtxt<'tcx>, /// A mapping of IDs that contains all local items for this crate which gets output as a top @@ -49,6 +55,7 @@ pub(crate) struct JsonRenderer<'tcx> { cache: Rc, imported_items: DefIdSet, id_interner: RefCell, + output_format: OutputFormat, } impl<'tcx> JsonRenderer<'tcx> { @@ -114,10 +121,30 @@ impl<'tcx> JsonRenderer<'tcx> { path: &str, ) -> Result<(), Error> { self.sess().time("rustdoc_json_serialize_and_write", || { - try_err!( - serde_json::ser::to_writer(&mut writer, &output_crate).map_err(|e| e.to_string()), - path - ); + match self.output_format { + OutputFormat::Json => { + try_err!( + serde_json::ser::to_writer(&mut writer, &output_crate) + .map_err(|e| e.to_string()), + path + ); + } + OutputFormat::Postcard => { + let output = ( + rustdoc_json_types::postcard::Header { + magic: rustdoc_json_types::postcard::MAGIC, + format_version: rustdoc_json_types::FORMAT_VERSION, + }, + output_crate, + ); + + try_err!( + postcard::to_io(&output, &mut writer).map_err(|e| e.to_string()), + path + ); + } + } + try_err!(writer.flush(), path); Ok(()) }) @@ -201,6 +228,13 @@ impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> { out_dir: if options.output_to_stdout { None } else { Some(options.output) }, cache: Rc::new(cache), imported_items, + output_format: match options.output_format { + crate::config::OutputFormat::Json => OutputFormat::Json, + crate::config::OutputFormat::Postcard => OutputFormat::Postcard, + crate::config::OutputFormat::Html | crate::config::OutputFormat::Doctest => { + unreachable!() + } + }, id_interner: Default::default(), }, krate, @@ -366,7 +400,10 @@ impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> { let mut p = out_dir.clone(); p.push(output_crate.index.get(&output_crate.root).unwrap().name.clone().unwrap()); - p.set_extension("json"); + p.set_extension(match self.output_format { + OutputFormat::Json => "json", + OutputFormat::Postcard => "postcard", + }); self.serialize_and_write( output_crate, diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 025c135aff2a6..ab645cfdcfcfa 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -922,9 +922,10 @@ fn main_args(early_dcx: &mut EarlyDiagCtxt, at_args: &[String]) { config::OutputFormat::Html => sess.time("render_html", || { run_renderer::>(krate, render_opts, cache, tcx) }), - config::OutputFormat::Json => sess.time("render_json", || { - run_renderer::>(krate, render_opts, cache, tcx) - }), + config::OutputFormat::Json | config::OutputFormat::Postcard => sess + .time("render_json", || { + run_renderer::>(krate, render_opts, cache, tcx) + }), // Already handled above with doctest runners. config::OutputFormat::Doctest => unreachable!(), } diff --git a/src/rustdoc-json-types/lib.rs b/src/rustdoc-json-types/lib.rs index 8a3ab6f864072..ae248ae34bfbe 100644 --- a/src/rustdoc-json-types/lib.rs +++ b/src/rustdoc-json-types/lib.rs @@ -32,6 +32,19 @@ pub type FxHashMap = HashMap; // re-export for use in src/librustdoc /// Consuming code should assert that this value matches the format version(s) that it supports. pub const FORMAT_VERSION: u32 = 46; +pub mod postcard { + + pub type Magic = [u8; 22]; + pub const MAGIC: Magic = *b"\x00\xFFRustdocJsonPostcard\xFF"; + + #[derive(Clone, Debug, PartialEq, Eq, serde_derive::Serialize, serde_derive::Deserialize)] + pub struct Header { + // Order here matters + pub magic: Magic, + pub format_version: u32, + } +} + /// The root of the emitted JSON blob. /// /// It contains all type/documentation information diff --git a/src/rustdoc-json-types/tests.rs b/src/rustdoc-json-types/tests.rs index b9363fcf1b714..c200ea1b81ea9 100644 --- a/src/rustdoc-json-types/tests.rs +++ b/src/rustdoc-json-types/tests.rs @@ -38,3 +38,11 @@ fn test_union_info_roundtrip() { let decoded: ItemEnum = bincode::deserialize(&encoded).unwrap(); assert_eq!(u, decoded); } + +#[test] +fn magic_never_dies() { + // Extra check to make sure that the postcard magic header never changes. + // Don't change this value + assert_eq!(crate::postcard::MAGIC, *b"\x00\xFFRustdocJsonPostcard\xFF"); + // Don't change that value +} diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 75f24adb70fa5..82a1141ab2197 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -245,6 +245,13 @@ enum Emit { LinkArgsAsm, } +#[derive(Clone, Copy, Debug, PartialEq)] +enum DocKind { + Html, + Json, + Postcard, +} + impl<'test> TestCx<'test> { /// Code executed for each revision in turn (or, if there are no /// revisions, exactly once, with revision == None). @@ -861,8 +868,15 @@ impl<'test> TestCx<'test> { /// `root_out_dir` and `root_testpaths` refer to the parameters of the actual test being run. /// Auxiliaries, no matter how deep, have the same root_out_dir and root_testpaths. - fn document(&self, root_out_dir: &Utf8Path, root_testpaths: &TestPaths) -> ProcRes { + fn document( + &self, + root_out_dir: &Utf8Path, + root_testpaths: &TestPaths, + kind: DocKind, + ) -> ProcRes { if self.props.build_aux_docs { + assert_eq!(kind, DocKind::Html, "build-aux-docs doesn't make sense for rustdoc json"); + for rel_ab in &self.props.aux.builds { let aux_testpaths = self.compute_aux_test_paths(root_testpaths, rel_ab); let props_for_aux = @@ -877,7 +891,7 @@ impl<'test> TestCx<'test> { create_dir_all(aux_cx.output_base_dir()).unwrap(); // use root_testpaths here, because aux-builds should have the // same --out-dir and auxiliary directory. - let auxres = aux_cx.document(&root_out_dir, root_testpaths); + let auxres = aux_cx.document(&root_out_dir, root_testpaths, kind); if !auxres.status.success() { return auxres; } @@ -922,8 +936,14 @@ impl<'test> TestCx<'test> { .args(&self.props.compile_flags) .args(&self.props.doc_flags); - if self.config.mode == RustdocJson { - rustdoc.arg("--output-format").arg("json").arg("-Zunstable-options"); + match kind { + DocKind::Html => {} + DocKind::Json => { + rustdoc.arg("--output-format").arg("json").arg("-Zunstable-options"); + } + DocKind::Postcard => { + rustdoc.arg("--output-format").arg("postcard").arg("-Zunstable-options"); + } } if let Some(ref linker) = self.config.target_linker { @@ -1992,7 +2012,7 @@ impl<'test> TestCx<'test> { let aux_dir = new_rustdoc.aux_output_dir(); new_rustdoc.build_all_auxiliary(&new_rustdoc.testpaths, &aux_dir, &mut rustc); - let proc_res = new_rustdoc.document(&compare_dir, &new_rustdoc.testpaths); + let proc_res = new_rustdoc.document(&compare_dir, &new_rustdoc.testpaths, DocKind::Html); if !proc_res.status.success() { eprintln!("failed to run nightly rustdoc"); return; diff --git a/src/tools/compiletest/src/runtest/js_doc.rs b/src/tools/compiletest/src/runtest/js_doc.rs index fd53f01ca1746..93b05617e6f8c 100644 --- a/src/tools/compiletest/src/runtest/js_doc.rs +++ b/src/tools/compiletest/src/runtest/js_doc.rs @@ -1,13 +1,13 @@ use std::process::Command; -use super::TestCx; +use super::{DocKind, TestCx}; impl TestCx<'_> { pub(super) fn run_rustdoc_js_test(&self) { if let Some(nodejs) = &self.config.nodejs { let out_dir = self.output_base_dir(); - self.document(&out_dir, &self.testpaths); + self.document(&out_dir, &self.testpaths, DocKind::Html); let file_stem = self.testpaths.file.file_stem().expect("no file stem"); let res = self.run_command_to_procres( diff --git a/src/tools/compiletest/src/runtest/rustdoc.rs b/src/tools/compiletest/src/runtest/rustdoc.rs index 637ea833357a2..02ad2bf029563 100644 --- a/src/tools/compiletest/src/runtest/rustdoc.rs +++ b/src/tools/compiletest/src/runtest/rustdoc.rs @@ -1,6 +1,7 @@ use std::process::Command; use super::{TestCx, remove_and_create_dir_all}; +use crate::runtest::DocKind; impl TestCx<'_> { pub(super) fn run_rustdoc_test(&self) { @@ -11,7 +12,7 @@ impl TestCx<'_> { panic!("failed to remove and recreate output directory `{out_dir}`: {e}") }); - let proc_res = self.document(&out_dir, &self.testpaths); + let proc_res = self.document(&out_dir, &self.testpaths, DocKind::Html); if !proc_res.status.success() { self.fatal_proc_rec("rustdoc failed!", &proc_res); } diff --git a/src/tools/compiletest/src/runtest/rustdoc_json.rs b/src/tools/compiletest/src/runtest/rustdoc_json.rs index 9f88faca89268..75c07df0c17f2 100644 --- a/src/tools/compiletest/src/runtest/rustdoc_json.rs +++ b/src/tools/compiletest/src/runtest/rustdoc_json.rs @@ -1,6 +1,7 @@ use std::process::Command; use super::{TestCx, remove_and_create_dir_all}; +use crate::runtest::DocKind; impl TestCx<'_> { pub(super) fn run_rustdoc_json_test(&self) { @@ -13,7 +14,11 @@ impl TestCx<'_> { panic!("failed to remove and recreate output directory `{out_dir}`: {e}") }); - let proc_res = self.document(&out_dir, &self.testpaths); + let proc_res = self.document(&out_dir, &self.testpaths, DocKind::Json); + if !proc_res.status.success() { + self.fatal_proc_rec("rustdoc failed!", &proc_res); + } + let proc_res = self.document(&out_dir, &self.testpaths, DocKind::Postcard); if !proc_res.status.success() { self.fatal_proc_rec("rustdoc failed!", &proc_res); } @@ -35,11 +40,11 @@ impl TestCx<'_> { }) } - let mut json_out = out_dir.join(self.testpaths.file.file_stem().unwrap()); - json_out.set_extension("json"); + let postcard_out = json_out.with_extension("postcard"); let res = self.run_command_to_procres( - Command::new(self.config.jsondoclint_path.as_ref().unwrap()).arg(&json_out), + Command::new(self.config.jsondoclint_path.as_ref().unwrap()) + .args([&json_out, &postcard_out]), ); if !res.status.success() { diff --git a/src/tools/jsondoclint/Cargo.toml b/src/tools/jsondoclint/Cargo.toml index cc8ecefd530b4..e77d9cd69066d 100644 --- a/src/tools/jsondoclint/Cargo.toml +++ b/src/tools/jsondoclint/Cargo.toml @@ -7,8 +7,10 @@ edition = "2021" [dependencies] anyhow = "1.0.62" +camino = "1.1.10" clap = { version = "4.0.15", features = ["derive"] } fs-err = "2.8.1" +postcard = { version = "1.1.1", default-features = false, features = ["use-std"] } rustc-hash = "2.0.0" rustdoc-json-types = { version = "0.1.0", path = "../../rustdoc-json-types" } serde = { version = "1.0", features = ["derive"] } diff --git a/src/tools/jsondoclint/src/main.rs b/src/tools/jsondoclint/src/main.rs index 5cbf346086062..4e7673349754f 100644 --- a/src/tools/jsondoclint/src/main.rs +++ b/src/tools/jsondoclint/src/main.rs @@ -1,7 +1,7 @@ use std::io::{BufWriter, Write}; -use std::path::{Path, PathBuf}; use anyhow::{Result, bail}; +use camino::{Utf8Path, Utf8PathBuf}; use clap::Parser; use fs_err as fs; use rustdoc_json_types::{Crate, FORMAT_VERSION, Id}; @@ -26,14 +26,16 @@ enum ErrorKind { #[derive(Debug, Serialize)] struct JsonOutput { - path: PathBuf, + path: String, errors: Vec, } #[derive(Parser)] struct Cli { /// The path to the json file to be linted - path: String, + json_path: String, + + postcard_path: String, /// Show verbose output #[arg(long)] @@ -44,25 +46,40 @@ struct Cli { } fn main() -> Result<()> { - let Cli { path, verbose, json_output } = Cli::parse(); + let Cli { json_path, postcard_path, verbose, json_output } = Cli::parse(); - // We convert `-` into `_` for the file name to be sure the JSON path will always be correct. - let path = Path::new(&path); - let filename = path.file_name().unwrap().to_str().unwrap().replace('-', "_"); - let parent = path.parent().unwrap(); - let path = parent.join(&filename); + let json_path = normalize_path(&json_path); + let postcard_path = normalize_path(&postcard_path); - let contents = fs::read_to_string(&path)?; + let contents = fs::read_to_string(&json_path)?; let krate: Crate = serde_json::from_str(&contents)?; assert_eq!(krate.format_version, FORMAT_VERSION); + { + let postcard_contents = fs::read(&postcard_path)?; + + use rustdoc_json_types::postcard::{MAGIC, Magic}; + + assert_eq!(postcard_contents[..MAGIC.len()], MAGIC, "missing magic"); + + let (magic, format_version, postcard_crate): (Magic, u32, Crate) = + postcard::from_bytes(&postcard_contents)?; + + assert_eq!(magic, MAGIC); + assert_eq!(format_version, FORMAT_VERSION); + + if postcard_crate != krate { + bail!("{postcard_path} doesn't equal {json_path}"); + } + } + let krate_json: Value = serde_json::from_str(&contents)?; let mut validator = validator::Validator::new(&krate, krate_json); validator.check_crate(); if let Some(json_output) = json_output { - let output = JsonOutput { path: path.clone(), errors: validator.errs.clone() }; + let output = JsonOutput { path: json_path.to_string(), errors: validator.errs.clone() }; let mut f = BufWriter::new(fs::File::create(json_output)?); serde_json::to_writer(&mut f, &output)?; f.flush()?; @@ -108,8 +125,18 @@ fn main() -> Result<()> { ErrorKind::Custom(msg) => eprintln!("{}: {}", err.id.0, msg), } } - bail!("Errors validating json {}", path.display()); + bail!("Errors validating json {json_path}"); } Ok(()) } + +fn normalize_path(path: &str) -> Utf8PathBuf { + // We convert `-` into `_` for the file name to be sure the JSON path will always be correct. + + let path = Utf8Path::new(&path); + let filename = path.file_name().unwrap().to_string().replace('-', "_"); + let parent = path.parent().unwrap(); + let path = parent.join(&filename); + path +}