From f27ed880531867fa548c9780144558cecb6fbe71 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 15 May 2024 21:51:32 -0700 Subject: [PATCH 01/32] codegen: test frame pointer attr prefers CLI opt This test only makes sense if you send it back in time and run it with a now-old Rust commit, e.g. 50e0cc59ffcacda5b48f4edb95e5a5c353624fb0 However, if you do go back that far in time, you will see it pass. --- tests/codegen/frame-pointer-cli-control.rs | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/codegen/frame-pointer-cli-control.rs diff --git a/tests/codegen/frame-pointer-cli-control.rs b/tests/codegen/frame-pointer-cli-control.rs new file mode 100644 index 0000000000000..12282eea8f48b --- /dev/null +++ b/tests/codegen/frame-pointer-cli-control.rs @@ -0,0 +1,35 @@ +// compile-flags: --crate-type=rlib -Copt-level=0 +// revisions: force-on aarch64-apple aarch64-apple-off +// [force-on] compile-flags: -Cforce-frame-pointers=on +// [aarch64-apple] needs-llvm-components: aarch64 +// [aarch64-apple] compile-flags: --target=aarch64-apple-darwin +// [aarch64-apple-off] needs-llvm-components: aarch64 +// [aarch64-apple-off] compile-flags: --target=aarch64-apple-darwin -Cforce-frame-pointers=off +/* +Tests that the frame pointers can be controlled by the CLI. We find aarch64-apple-darwin useful +because of its icy-clear policy regarding frame pointers (software SHALL be compiled with them), +e.g. https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms says: + +* The frame pointer register (x29) must always address a valid frame record. Some functions — + such as leaf functions or tail calls — may opt not to create an entry in this list. + As a result, stack traces are always meaningful, even without debug information. +*/ +#![feature(no_core, lang_items)] +#![no_core] +#[lang = "sized"] +trait Sized {} +#[lang = "copy"] +trait Copy {} +impl Copy for u32 {} + +// CHECK: define i32 @peach{{.*}}[[PEACH_ATTRS:\#[0-9]+]] { +#[no_mangle] +pub fn peach(x: u32) -> u32 { + x +} + +// CHECK: attributes [[PEACH_ATTRS]] = { +// force-on-SAME: {{.*}}"frame-pointer"="all" +// aarch64-apple-SAME: {{.*}}"frame-pointer"="all" +// aarch64-apple-off-NOT: {{.*}}"frame-pointer"{{.*}} +// CHECK-SAME: } From a6b62d893f620e81cc04a39acd67e6f638d29358 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 15 May 2024 21:57:01 -0700 Subject: [PATCH 02/32] codegen: modernize frame-pointer-cli-control.rs Update this time-traveler on the changes in compiletest and target specs that they missed over the pass ~3 years by being caught in a time rift. The aarch64-apple rev splits into itself and aarch64-apple-on, because rustc obtained support for non-leaf frame-pointers ever since 9b67cba implemented them and used them in aarch64-apple-darwin's spec. Note that the aarch64-apple-off revision fails, despite modernization. This is because 9b67cba also changed the behavior of rustc to defer to the spec over the command-line interface. --- tests/codegen/frame-pointer-cli-control.rs | 27 +++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/codegen/frame-pointer-cli-control.rs b/tests/codegen/frame-pointer-cli-control.rs index 12282eea8f48b..2ce9fa22ddb09 100644 --- a/tests/codegen/frame-pointer-cli-control.rs +++ b/tests/codegen/frame-pointer-cli-control.rs @@ -1,10 +1,13 @@ -// compile-flags: --crate-type=rlib -Copt-level=0 -// revisions: force-on aarch64-apple aarch64-apple-off -// [force-on] compile-flags: -Cforce-frame-pointers=on -// [aarch64-apple] needs-llvm-components: aarch64 -// [aarch64-apple] compile-flags: --target=aarch64-apple-darwin -// [aarch64-apple-off] needs-llvm-components: aarch64 -// [aarch64-apple-off] compile-flags: --target=aarch64-apple-darwin -Cforce-frame-pointers=off +//@ add-core-stubs +//@ compile-flags: --crate-type=rlib -Copt-level=0 +//@ revisions: force-on aarch64-apple aarch64-apple-on aarch64-apple-off +//@ [force-on] compile-flags: -Cforce-frame-pointers=on +//@ [aarch64-apple] needs-llvm-components: aarch64 +//@ [aarch64-apple] compile-flags: --target=aarch64-apple-darwin +//@ [aarch64-apple-on] needs-llvm-components: aarch64 +//@ [aarch64-apple-on] compile-flags: --target=aarch64-apple-darwin -Cforce-frame-pointers=on +//@ [aarch64-apple-off] needs-llvm-components: aarch64 +//@ [aarch64-apple-off] compile-flags: --target=aarch64-apple-darwin -Cforce-frame-pointers=off /* Tests that the frame pointers can be controlled by the CLI. We find aarch64-apple-darwin useful because of its icy-clear policy regarding frame pointers (software SHALL be compiled with them), @@ -16,11 +19,8 @@ e.g. https://developer.apple.com/documentation/xcode/writing-arm64-code-for-appl */ #![feature(no_core, lang_items)] #![no_core] -#[lang = "sized"] -trait Sized {} -#[lang = "copy"] -trait Copy {} -impl Copy for u32 {} + +extern crate minicore; // CHECK: define i32 @peach{{.*}}[[PEACH_ATTRS:\#[0-9]+]] { #[no_mangle] @@ -30,6 +30,7 @@ pub fn peach(x: u32) -> u32 { // CHECK: attributes [[PEACH_ATTRS]] = { // force-on-SAME: {{.*}}"frame-pointer"="all" -// aarch64-apple-SAME: {{.*}}"frame-pointer"="all" +// aarch64-apple-SAME: {{.*}}"frame-pointer"="non-leaf" +// aarch64-apple-on-SAME: {{.*}}"frame-pointer"="all" // aarch64-apple-off-NOT: {{.*}}"frame-pointer"{{.*}} // CHECK-SAME: } From e57b4b19e8ba0ba680461da46c8e2d42886c634a Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 15 May 2024 23:47:48 -0700 Subject: [PATCH 03/32] encode compiler team acceptance of `-Cforce-frame-pointers` change --- tests/codegen/frame-pointer-cli-control.rs | 35 ++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/tests/codegen/frame-pointer-cli-control.rs b/tests/codegen/frame-pointer-cli-control.rs index 2ce9fa22ddb09..d1f2e2b685f72 100644 --- a/tests/codegen/frame-pointer-cli-control.rs +++ b/tests/codegen/frame-pointer-cli-control.rs @@ -8,14 +8,37 @@ //@ [aarch64-apple-on] compile-flags: --target=aarch64-apple-darwin -Cforce-frame-pointers=on //@ [aarch64-apple-off] needs-llvm-components: aarch64 //@ [aarch64-apple-off] compile-flags: --target=aarch64-apple-darwin -Cforce-frame-pointers=off -/* -Tests that the frame pointers can be controlled by the CLI. We find aarch64-apple-darwin useful -because of its icy-clear policy regarding frame pointers (software SHALL be compiled with them), -e.g. https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms says: +/*! + +Tests the extent to which frame pointers can be controlled by the CLI. +The behavior of our frame pointer options, at present, is an irreversible ratchet, where +a "weaker" option that allows omitting frame pointers may be overridden by the target demanding +that all code (or all non-leaf code, more often) must be compiled with frame pointers. +This was discussed on 2025-05-22 in the T-compiler meeting and accepted as an intentional change, +ratifying the prior decisions by compiler contributors and reviewers as correct, +though it was also acknowledged that the flag allows somewhat confusing inputs. + +We find aarch64-apple-darwin useful because of its icy-clear policy regarding frame pointers, +e.g. says: * The frame pointer register (x29) must always address a valid frame record. Some functions — such as leaf functions or tail calls — may opt not to create an entry in this list. As a result, stack traces are always meaningful, even without debug information. + +Many Rust fn, if externally visible, may be expected to follow target ABI by tools or asm code! +This can make it a problem to generate ABI-incorrect code, which may mean "with frame pointers". +For this and other reasons, `-Cforce-frame-pointers=off` cannot override the target definition. +This can cause some confusion because it is "reverse polarity" relative to C compilers, which have +commands like `-fomit-frame-pointer`, `-fomit-leaf-frame-pointer`, or `-fno-omit-frame-pointer`! + +Specific cases where platforms or tools rely on frame pointers for sound or correct unwinding: +- illumos: +- aarch64-windows: +- aarch64-linux: +- dtrace (freebsd and openbsd): +- openbsd: +- i686-msvc +- i686-mingw: */ #![feature(no_core, lang_items)] #![no_core] @@ -32,5 +55,7 @@ pub fn peach(x: u32) -> u32 { // force-on-SAME: {{.*}}"frame-pointer"="all" // aarch64-apple-SAME: {{.*}}"frame-pointer"="non-leaf" // aarch64-apple-on-SAME: {{.*}}"frame-pointer"="all" -// aarch64-apple-off-NOT: {{.*}}"frame-pointer"{{.*}} +// +// yes, we are testing this doesn't do anything: +// aarch64-apple-off-SAME: {{.*}}"frame-pointer"="non-leaf" // CHECK-SAME: } From 1b23b64be4bf9e827cbc9d7c4012da77e8f05a85 Mon Sep 17 00:00:00 2001 From: Josh Triplett Date: Fri, 17 Jan 2025 23:23:56 +0200 Subject: [PATCH 04/32] Add `-Z hint-mostly-unused` to tell rustc that most of a crate will go unused This hint allows the compiler to optimize its operation based on this assumption, in order to compile faster. This is a hint, and does not guarantee any particular behavior. This option can substantially speed up compilation if applied to a large dependency where the majority of the dependency does not get used. This flag may slow down compilation in other cases. Currently, this option makes the compiler defer as much code generation as possible from functions in the crate, until later crates invoke those functions. Functions that never get invoked will never have code generated for them. For instance, if a crate provides thousands of functions, but only a few of them will get called, this flag will result in the compiler only doing code generation for the called functions. (This uses the same mechanisms as cross-crate inlining of functions.) This does not affect `extern` functions, or functions marked as `#[inline(never)]`. Some performance numbers, based on a crate with many dependencies having just *one* large dependency set to `-Z hint-mostly-unused` (using Cargo's `profile-rustflags` option): A release build went from 4m07s to 2m04s. A non-release build went from 2m26s to 1m28s. --- compiler/rustc_interface/src/tests.rs | 1 + .../src/cross_crate_inline.rs | 7 ++++ compiler/rustc_session/src/options.rs | 2 ++ .../src/compiler-flags/hint-mostly-unused.md | 33 +++++++++++++++++++ 4 files changed, 43 insertions(+) create mode 100644 src/doc/unstable-book/src/compiler-flags/hint-mostly-unused.md diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 558f13a832c43..5d4dd16389731 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -801,6 +801,7 @@ fn test_unstable_options_tracking_hash() { tracked!(force_unstable_if_unmarked, true); tracked!(function_return, FunctionReturn::ThunkExtern); tracked!(function_sections, Some(false)); + tracked!(hint_mostly_unused, true); tracked!(human_readable_cgu_names, true); tracked!(incremental_ignore_spans, true); tracked!(inline_mir, Some(true)); diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 727d4a126d21e..6d7b7e10ef697 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -50,6 +50,13 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { _ => {} } + // If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until + // the function is referenced, in order to skip codegen for unused functions. This is + // intentionally after the check for `inline(never)`, so that `inline(never)` wins. + if tcx.sess.opts.unstable_opts.hint_mostly_unused { + return true; + } + let sig = tcx.fn_sig(def_id).instantiate_identity(); for ty in sig.inputs().skip_binder().iter().chain(std::iter::once(&sig.output().skip_binder())) { diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 12fa05118caf5..7ded572a60e30 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2233,6 +2233,8 @@ options! { environment variable `RUSTC_GRAPHVIZ_FONT` (default: `Courier, monospace`)"), has_thread_local: Option = (None, parse_opt_bool, [TRACKED], "explicitly enable the `cfg(target_thread_local)` directive"), + hint_mostly_unused: bool = (false, parse_bool, [TRACKED], + "hint that most of this crate will go unused, to minimize work for uncalled functions"), human_readable_cgu_names: bool = (false, parse_bool, [TRACKED], "generate human-readable, predictable names for codegen units (default: no)"), identify_regions: bool = (false, parse_bool, [UNTRACKED], diff --git a/src/doc/unstable-book/src/compiler-flags/hint-mostly-unused.md b/src/doc/unstable-book/src/compiler-flags/hint-mostly-unused.md new file mode 100644 index 0000000000000..80f5b1c44500d --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/hint-mostly-unused.md @@ -0,0 +1,33 @@ +# `hint-mostly-unused` + +This flag hints to the compiler that most of the crate will probably go unused. +The compiler can optimize its operation based on this assumption, in order to +compile faster. This is a hint, and does not guarantee any particular behavior. + +This option can substantially speed up compilation if applied to a large +dependency where the majority of the dependency does not get used. This flag +may slow down compilation in other cases. + +Currently, this option makes the compiler defer as much code generation as +possible from functions in the crate, until later crates invoke those +functions. Functions that never get invoked will never have code generated for +them. For instance, if a crate provides thousands of functions, but only a few +of them will get called, this flag will result in the compiler only doing code +generation for the called functions. (This uses the same mechanisms as +cross-crate inlining of functions.) This does not affect `extern` functions, or +functions marked as `#[inline(never)]`. + +To try applying this flag to one dependency out of a dependency tree, use the +[`profile-rustflags`](https://doc.rust-lang.org/cargo/reference/unstable.html#profile-rustflags-option) +feature of nightly cargo: + +```toml +cargo-features = ["profile-rustflags"] + +# ... +[dependencies] +mostly-unused-dependency = "1.2.3" + +[profile.release.package.mostly-unused-dependency] +rustflags = ["-Zhint-mostly-unused"] +``` From fe04ae7fafab80cebd01ce9c199248556ec3654f Mon Sep 17 00:00:00 2001 From: Boxy Date: Wed, 11 Jun 2025 15:30:15 +0100 Subject: [PATCH 05/32] stabilize gai --- compiler/rustc_ast_lowering/messages.ftl | 3 - compiler/rustc_ast_lowering/src/lib.rs | 13 +---- compiler/rustc_feature/src/accepted.rs | 2 + compiler/rustc_feature/src/unstable.rs | 2 - .../rustc_hir_analysis/src/collect/type_of.rs | 17 +----- .../src/hir_ty_lowering/generics.rs | 15 +---- library/core/src/lib.rs | 1 - .../tests/ui/single_range_in_vec_init.rs | 1 - .../tests/ui/single_range_in_vec_init.stderr | 20 +++---- tests/crashes/111419.rs | 2 +- .../suggest-array-length.fixed | 15 ++--- .../array-slice-vec/suggest-array-length.rs | 5 -- .../suggest-array-length.stderr | 55 +------------------ tests/ui/async-await/issues/issue-95307.rs | 1 - .../ui/async-await/issues/issue-95307.stderr | 15 +---- .../binder/forbid_ambig_const_infers.rs | 2 +- .../binder/forbid_ambig_type_infers.rs | 2 +- .../ui/closures/binder/forbid_const_infer.rs | 2 +- .../equality_bound_with_infer.rs | 2 +- .../array-repeat-expr-lib.rs | 1 - .../generic_arg_infer/array-repeat-expr.rs | 1 - .../generic_arg_infer/dont-use-defaults.rs | 3 - .../generic_arg_infer/in-signature.rs | 1 - .../generic_arg_infer/in-signature.stderr | 30 +++++----- .../generic_arg_infer/infer-arg-test.rs | 2 - .../generic_arg_infer/infer-arg-test.stderr | 10 ++-- .../infer_arg_and_const_arg.rs | 1 - .../generic_arg_infer/issue-91614.rs | 1 - .../generic_arg_infer/issue-91614.stderr | 4 +- .../parend_infer.nogate.stderr | 53 ------------------ .../generic_arg_infer/parend_infer.rs | 9 +-- .../const_kind_expr/wf_obligation.rs | 2 +- .../poly-const-uneval-ice-106423.rs | 10 +--- .../bad_inference.rs | 7 +-- .../bad_inference.stderr | 4 +- .../evaluate_const_parameter_in_mir.rs | 7 +-- .../inferred_from_arg.rs | 2 +- .../unrelated_inferred_arg.rs | 7 +-- .../issues/issue-62878.min.stderr | 15 +---- tests/ui/const-generics/issues/issue-62878.rs | 3 +- .../min_const_generics/inferred_const.rs | 1 - ...ature-gate-generic_arg_infer.normal.stderr | 33 ----------- .../feature-gate-generic_arg_infer.rs | 21 ------- .../lang-item-generic-requirements.rs | 4 +- .../object-lifetime-default-inferred.rs | 1 - tests/ui/parser/issues/issue-14303-fncall.rs | 5 +- .../parser/issues/issue-14303-fncall.stderr | 9 +++ tests/ui/pattern/slice-array-infer.rs | 1 - ...py-check-const-element-uninferred-count.rs | 2 - ...heck-const-element-uninferred-count.stderr | 2 +- .../copy-check-deferred-after-fallback.rs | 2 - .../copy-check-deferred-after-fallback.stderr | 2 +- .../copy-check-deferred-before-fallback.rs | 1 - .../copy-check-inference-side-effects.rs | 2 - .../copy-check-inference-side-effects.stderr | 4 +- .../copy-check-when-count-inferred-later.rs | 2 - ...opy-check-when-count-inferred-later.stderr | 2 +- .../repeat-expr/dont-require-copy-on-infer.rs | 1 - .../no-conservative-copy-impl-requirement.rs | 2 - ...-conservative-copy-impl-requirement.stderr | 2 +- tests/ui/simd/const-err-trumps-simd-err.rs | 1 - .../ui/simd/const-err-trumps-simd-err.stderr | 6 +- .../span/issue-42234-unknown-receiver-type.rs | 3 - .../issue-42234-unknown-receiver-type.stderr | 23 ++++++++ .../tilde-const-and-const-params.rs | 1 - .../tilde-const-and-const-params.stderr | 12 ++-- 66 files changed, 116 insertions(+), 377 deletions(-) delete mode 100644 tests/ui/const-generics/generic_arg_infer/parend_infer.nogate.stderr delete mode 100644 tests/ui/feature-gates/feature-gate-generic_arg_infer.normal.stderr delete mode 100644 tests/ui/feature-gates/feature-gate-generic_arg_infer.rs create mode 100644 tests/ui/parser/issues/issue-14303-fncall.stderr create mode 100644 tests/ui/span/issue-42234-unknown-receiver-type.stderr diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl index 5ef76fb64aaf2..a8498d771da67 100644 --- a/compiler/rustc_ast_lowering/messages.ftl +++ b/compiler/rustc_ast_lowering/messages.ftl @@ -172,9 +172,6 @@ ast_lowering_template_modifier = template modifier ast_lowering_this_not_async = this is not `async` -ast_lowering_underscore_array_length_unstable = - using `_` for array lengths is unstable - ast_lowering_underscore_expr_lhs_assign = in expressions, `_` can only be used on the left-hand side of an assignment .label = `_` not allowed here diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index b99df8bd7e552..f070759ca8174 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -48,7 +48,7 @@ use rustc_data_structures::sorted_map::SortedMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::spawn; use rustc_data_structures::tagged_ptr::TaggedRef; -use rustc_errors::{DiagArgFromDisplay, DiagCtxtHandle, StashKey}; +use rustc_errors::{DiagArgFromDisplay, DiagCtxtHandle}; use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res}; use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId}; use rustc_hir::{ @@ -59,7 +59,7 @@ use rustc_index::{Idx, IndexSlice, IndexVec}; use rustc_macros::extension; use rustc_middle::span_bug; use rustc_middle::ty::{ResolverAstLowering, TyCtxt}; -use rustc_session::parse::{add_feature_diagnostics, feature_err}; +use rustc_session::parse::add_feature_diagnostics; use rustc_span::symbol::{Ident, Symbol, kw, sym}; use rustc_span::{DUMMY_SP, DesugaringKind, Span}; use smallvec::SmallVec; @@ -2068,15 +2068,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // `ExprKind::Paren(ExprKind::Underscore)` and should also be lowered to `GenericArg::Infer` match c.value.peel_parens().kind { ExprKind::Underscore => { - if !self.tcx.features().generic_arg_infer() { - feature_err( - &self.tcx.sess, - sym::generic_arg_infer, - c.value.span, - fluent_generated::ast_lowering_underscore_array_length_unstable, - ) - .stash(c.value.span, StashKey::UnderscoreForArrayLengths); - } let ct_kind = hir::ConstArgKind::Infer(self.lower_span(c.value.span), ()); self.arena.alloc(hir::ConstArg { hir_id: self.lower_node_id(c.id), kind: ct_kind }) } diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs index b1c185220f45b..cfe0f4e5d6cb6 100644 --- a/compiler/rustc_feature/src/accepted.rs +++ b/compiler/rustc_feature/src/accepted.rs @@ -220,6 +220,8 @@ declare_features! ( (accepted, fn_must_use, "1.27.0", Some(43302)), /// Allows capturing variables in scope using format_args! (accepted, format_args_capture, "1.58.0", Some(67984)), + /// Infer generic args for both consts and types. + (accepted, generic_arg_infer, "CURRENT_RUSTC_VERSION", Some(85077)), /// Allows associated types to be generic, e.g., `type Foo;` (RFC 1598). (accepted, generic_associated_types, "1.65.0", Some(44265)), /// Allows attributes on lifetime/type formal parameters in generics (RFC 1327). diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs index 594021d78d2e4..8c6411342f28d 100644 --- a/compiler/rustc_feature/src/unstable.rs +++ b/compiler/rustc_feature/src/unstable.rs @@ -516,8 +516,6 @@ declare_features! ( (unstable, frontmatter, "1.88.0", Some(136889)), /// Allows defining gen blocks and `gen fn`. (unstable, gen_blocks, "1.75.0", Some(117078)), - /// Infer generic args for both consts and types. - (unstable, generic_arg_infer, "1.55.0", Some(85077)), /// Allows non-trivial generic constants which have to have wfness manually propagated to callers (incomplete, generic_const_exprs, "1.56.0", Some(76560)), /// Allows generic parameters and where-clauses on free & associated const items. diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs index 141d96b57e579..902a2e15dffde 100644 --- a/compiler/rustc_hir_analysis/src/collect/type_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs @@ -452,13 +452,6 @@ fn infer_placeholder_type<'tcx>( if let Some(ty) = node.ty() { visitor.visit_ty_unambig(ty); } - // If we have just one span, let's try to steal a const `_` feature error. - let try_steal_span = if !tcx.features().generic_arg_infer() && visitor.spans.len() == 1 - { - visitor.spans.first().copied() - } else { - None - }; // If we didn't find any infer tys, then just fallback to `span`. if visitor.spans.is_empty() { visitor.spans.push(span); @@ -489,15 +482,7 @@ fn infer_placeholder_type<'tcx>( } } - if let Some(try_steal_span) = try_steal_span { - cx.dcx().try_steal_replace_and_emit_err( - try_steal_span, - StashKey::UnderscoreForArrayLengths, - diag, - ) - } else { - diag.emit() - } + diag.emit() }); Ty::new_error(tcx, guar) } diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/generics.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/generics.rs index 3a26b8331f8b3..8c7c3750865cf 100644 --- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/generics.rs +++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/generics.rs @@ -8,7 +8,7 @@ use rustc_middle::ty::{ self, GenericArgsRef, GenericParamDef, GenericParamDefKind, IsSuggestable, Ty, }; use rustc_session::lint::builtin::LATE_BOUND_LIFETIME_ARGUMENTS; -use rustc_span::{kw, sym}; +use rustc_span::kw; use smallvec::SmallVec; use tracing::{debug, instrument}; @@ -258,19 +258,6 @@ pub fn lower_generic_args<'tcx: 'a, 'a>( GenericParamDefKind::Const { .. }, _, ) => { - if let GenericParamDefKind::Const { .. } = param.kind - && let GenericArg::Infer(inf) = arg - && !tcx.features().generic_arg_infer() - { - rustc_session::parse::feature_err( - tcx.sess, - sym::generic_arg_infer, - inf.span, - "const arguments cannot yet be inferred with `_`", - ) - .emit(); - } - // We lower to an infer even when the feature gate is not enabled // as it is useful for diagnostics to be able to see a `ConstKind::Infer` args.push(ctx.provided_kind(&args, param, arg)); diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index 88855831788db..6231c3425c880 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -153,7 +153,6 @@ #![feature(f16)] #![feature(freeze_impls)] #![feature(fundamental)] -#![feature(generic_arg_infer)] #![feature(if_let_guard)] #![feature(intra_doc_pointers)] #![feature(intrinsics)] diff --git a/src/tools/clippy/tests/ui/single_range_in_vec_init.rs b/src/tools/clippy/tests/ui/single_range_in_vec_init.rs index 25884450b0842..0888019e101ce 100644 --- a/src/tools/clippy/tests/ui/single_range_in_vec_init.rs +++ b/src/tools/clippy/tests/ui/single_range_in_vec_init.rs @@ -2,7 +2,6 @@ //@no-rustfix: overlapping suggestions #![allow(clippy::no_effect, clippy::unnecessary_operation, clippy::useless_vec, unused)] #![warn(clippy::single_range_in_vec_init)] -#![feature(generic_arg_infer)] #[macro_use] extern crate proc_macros; diff --git a/src/tools/clippy/tests/ui/single_range_in_vec_init.stderr b/src/tools/clippy/tests/ui/single_range_in_vec_init.stderr index a99127a7606fb..b21338e38a3cb 100644 --- a/src/tools/clippy/tests/ui/single_range_in_vec_init.stderr +++ b/src/tools/clippy/tests/ui/single_range_in_vec_init.stderr @@ -1,5 +1,5 @@ error: an array of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:26:5 + --> tests/ui/single_range_in_vec_init.rs:25:5 | LL | [0..200]; | ^^^^^^^^ @@ -18,7 +18,7 @@ LL + [0; 200]; | error: a `Vec` of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:28:5 + --> tests/ui/single_range_in_vec_init.rs:27:5 | LL | vec![0..200]; | ^^^^^^^^^^^^ @@ -35,7 +35,7 @@ LL + vec![0; 200]; | error: an array of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:30:5 + --> tests/ui/single_range_in_vec_init.rs:29:5 | LL | [0u8..200]; | ^^^^^^^^^^ @@ -52,7 +52,7 @@ LL + [0u8; 200]; | error: an array of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:32:5 + --> tests/ui/single_range_in_vec_init.rs:31:5 | LL | [0usize..200]; | ^^^^^^^^^^^^^ @@ -69,7 +69,7 @@ LL + [0usize; 200]; | error: an array of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:34:5 + --> tests/ui/single_range_in_vec_init.rs:33:5 | LL | [0..200usize]; | ^^^^^^^^^^^^^ @@ -86,7 +86,7 @@ LL + [0; 200usize]; | error: a `Vec` of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:36:5 + --> tests/ui/single_range_in_vec_init.rs:35:5 | LL | vec![0u8..200]; | ^^^^^^^^^^^^^^ @@ -103,7 +103,7 @@ LL + vec![0u8; 200]; | error: a `Vec` of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:38:5 + --> tests/ui/single_range_in_vec_init.rs:37:5 | LL | vec![0usize..200]; | ^^^^^^^^^^^^^^^^^ @@ -120,7 +120,7 @@ LL + vec![0usize; 200]; | error: a `Vec` of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:40:5 + --> tests/ui/single_range_in_vec_init.rs:39:5 | LL | vec![0..200usize]; | ^^^^^^^^^^^^^^^^^ @@ -137,7 +137,7 @@ LL + vec![0; 200usize]; | error: an array of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:43:5 + --> tests/ui/single_range_in_vec_init.rs:42:5 | LL | [0..200isize]; | ^^^^^^^^^^^^^ @@ -149,7 +149,7 @@ LL + (0..200isize).collect::>(); | error: a `Vec` of `Range` that is only one element - --> tests/ui/single_range_in_vec_init.rs:45:5 + --> tests/ui/single_range_in_vec_init.rs:44:5 | LL | vec![0..200isize]; | ^^^^^^^^^^^^^^^^^ diff --git a/tests/crashes/111419.rs b/tests/crashes/111419.rs index 3a1a13df1985a..36f15e1d0a26e 100644 --- a/tests/crashes/111419.rs +++ b/tests/crashes/111419.rs @@ -1,6 +1,6 @@ //@ known-bug: #111419 #![allow(incomplete_features)] -#![feature(generic_const_exprs, generic_arg_infer)] +#![feature(generic_const_exprs)] pub trait Example where diff --git a/tests/ui/array-slice-vec/suggest-array-length.fixed b/tests/ui/array-slice-vec/suggest-array-length.fixed index 2eacc2517d310..ae1c6583c238b 100644 --- a/tests/ui/array-slice-vec/suggest-array-length.fixed +++ b/tests/ui/array-slice-vec/suggest-array-length.fixed @@ -10,14 +10,9 @@ fn main() { //~^ ERROR the placeholder `_` is not allowed within types on item signatures for static variables static REF_STATIK: &[u8; 1] = &[1]; //~^ ERROR the placeholder `_` is not allowed within types on item signatures for static variables - let foo: [i32; 3] = [1, 2, 3]; - //~^ ERROR using `_` for array lengths is unstable - let bar: [i32; 3] = [0; 3]; - //~^ ERROR using `_` for array lengths is unstable - let ref_foo: &[i32; 3] = &[1, 2, 3]; - //~^ ERROR using `_` for array lengths is unstable - let ref_bar: &[i32; 3] = &[0; 3]; - //~^ ERROR using `_` for array lengths is unstable - let multiple_ref_foo: &&[i32; 3] = &&[1, 2, 3]; - //~^ ERROR using `_` for array lengths is unstable + let foo: [i32; _] = [1, 2, 3]; + let bar: [i32; _] = [0; 3]; + let ref_foo: &[i32; _] = &[1, 2, 3]; + let ref_bar: &[i32; _] = &[0; 3]; + let multiple_ref_foo: &&[i32; _] = &&[1, 2, 3]; } diff --git a/tests/ui/array-slice-vec/suggest-array-length.rs b/tests/ui/array-slice-vec/suggest-array-length.rs index fb4424cfed99d..e53118014b2a4 100644 --- a/tests/ui/array-slice-vec/suggest-array-length.rs +++ b/tests/ui/array-slice-vec/suggest-array-length.rs @@ -11,13 +11,8 @@ fn main() { static REF_STATIK: &[u8; _] = &[1]; //~^ ERROR the placeholder `_` is not allowed within types on item signatures for static variables let foo: [i32; _] = [1, 2, 3]; - //~^ ERROR using `_` for array lengths is unstable let bar: [i32; _] = [0; 3]; - //~^ ERROR using `_` for array lengths is unstable let ref_foo: &[i32; _] = &[1, 2, 3]; - //~^ ERROR using `_` for array lengths is unstable let ref_bar: &[i32; _] = &[0; 3]; - //~^ ERROR using `_` for array lengths is unstable let multiple_ref_foo: &&[i32; _] = &&[1, 2, 3]; - //~^ ERROR using `_` for array lengths is unstable } diff --git a/tests/ui/array-slice-vec/suggest-array-length.stderr b/tests/ui/array-slice-vec/suggest-array-length.stderr index 14d10832e3608..e498f2ca4f580 100644 --- a/tests/ui/array-slice-vec/suggest-array-length.stderr +++ b/tests/ui/array-slice-vec/suggest-array-length.stderr @@ -46,57 +46,6 @@ LL - static REF_STATIK: &[u8; _] = &[1]; LL + static REF_STATIK: &[u8; 1] = &[1]; | -error[E0658]: using `_` for array lengths is unstable - --> $DIR/suggest-array-length.rs:13:20 - | -LL | let foo: [i32; _] = [1, 2, 3]; - | ^ help: consider specifying the array length: `3` - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: using `_` for array lengths is unstable - --> $DIR/suggest-array-length.rs:15:20 - | -LL | let bar: [i32; _] = [0; 3]; - | ^ help: consider specifying the array length: `3` - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: using `_` for array lengths is unstable - --> $DIR/suggest-array-length.rs:17:25 - | -LL | let ref_foo: &[i32; _] = &[1, 2, 3]; - | ^ help: consider specifying the array length: `3` - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: using `_` for array lengths is unstable - --> $DIR/suggest-array-length.rs:19:25 - | -LL | let ref_bar: &[i32; _] = &[0; 3]; - | ^ help: consider specifying the array length: `3` - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: using `_` for array lengths is unstable - --> $DIR/suggest-array-length.rs:21:35 - | -LL | let multiple_ref_foo: &&[i32; _] = &&[1, 2, 3]; - | ^ help: consider specifying the array length: `3` - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error: aborting due to 9 previous errors +error: aborting due to 4 previous errors -Some errors have detailed explanations: E0121, E0658. -For more information about an error, try `rustc --explain E0121`. +For more information about this error, try `rustc --explain E0121`. diff --git a/tests/ui/async-await/issues/issue-95307.rs b/tests/ui/async-await/issues/issue-95307.rs index 27903a667fb5c..83df65612b486 100644 --- a/tests/ui/async-await/issues/issue-95307.rs +++ b/tests/ui/async-await/issues/issue-95307.rs @@ -6,7 +6,6 @@ pub trait C { async fn new() -> [u8; _]; //~^ ERROR: the placeholder `_` is not allowed within types on item signatures for functions - //~| ERROR using `_` for array lengths is unstable } fn main() {} diff --git a/tests/ui/async-await/issues/issue-95307.stderr b/tests/ui/async-await/issues/issue-95307.stderr index 90100f391637a..c670686f7c9d5 100644 --- a/tests/ui/async-await/issues/issue-95307.stderr +++ b/tests/ui/async-await/issues/issue-95307.stderr @@ -4,17 +4,6 @@ error[E0121]: the placeholder `_` is not allowed within types on item signatures LL | async fn new() -> [u8; _]; | ^ not allowed in type signatures -error[E0658]: using `_` for array lengths is unstable - --> $DIR/issue-95307.rs:7:28 - | -LL | async fn new() -> [u8; _]; - | ^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error: aborting due to 2 previous errors +error: aborting due to 1 previous error -Some errors have detailed explanations: E0121, E0658. -For more information about an error, try `rustc --explain E0121`. +For more information about this error, try `rustc --explain E0121`. diff --git a/tests/ui/closures/binder/forbid_ambig_const_infers.rs b/tests/ui/closures/binder/forbid_ambig_const_infers.rs index e9d783711ee3e..eb258e0ed9f75 100644 --- a/tests/ui/closures/binder/forbid_ambig_const_infers.rs +++ b/tests/ui/closures/binder/forbid_ambig_const_infers.rs @@ -1,4 +1,4 @@ -#![feature(generic_arg_infer, closure_lifetime_binder)] +#![feature(closure_lifetime_binder)] struct Foo([u32; N]); diff --git a/tests/ui/closures/binder/forbid_ambig_type_infers.rs b/tests/ui/closures/binder/forbid_ambig_type_infers.rs index 4e717ef3a179d..ca44a5db96d0f 100644 --- a/tests/ui/closures/binder/forbid_ambig_type_infers.rs +++ b/tests/ui/closures/binder/forbid_ambig_type_infers.rs @@ -1,4 +1,4 @@ -#![feature(generic_arg_infer, closure_lifetime_binder)] +#![feature(closure_lifetime_binder)] struct Foo(T); diff --git a/tests/ui/closures/binder/forbid_const_infer.rs b/tests/ui/closures/binder/forbid_const_infer.rs index f5b8bf188dfee..8c8f0456f503e 100644 --- a/tests/ui/closures/binder/forbid_const_infer.rs +++ b/tests/ui/closures/binder/forbid_const_infer.rs @@ -1,4 +1,4 @@ -#![feature(generic_arg_infer, closure_lifetime_binder)] +#![feature(closure_lifetime_binder)] fn main() { let c = for<'a> |b: &'a [u32; _]| -> u32 { b[0] }; diff --git a/tests/ui/const-generics/associated_const_equality/equality_bound_with_infer.rs b/tests/ui/const-generics/associated_const_equality/equality_bound_with_infer.rs index f45b7c3268b17..dc42e00c2e834 100644 --- a/tests/ui/const-generics/associated_const_equality/equality_bound_with_infer.rs +++ b/tests/ui/const-generics/associated_const_equality/equality_bound_with_infer.rs @@ -1,4 +1,4 @@ -#![feature(generic_arg_infer, associated_const_equality, generic_const_items)] +#![feature(associated_const_equality, generic_const_items)] #![expect(incomplete_features)] // Regression test for #133066 where we would try to evaluate `<() as Foo>::ASSOC<_>` even diff --git a/tests/ui/const-generics/generic_arg_infer/array-repeat-expr-lib.rs b/tests/ui/const-generics/generic_arg_infer/array-repeat-expr-lib.rs index c1f725db126a0..c3a6767011483 100644 --- a/tests/ui/const-generics/generic_arg_infer/array-repeat-expr-lib.rs +++ b/tests/ui/const-generics/generic_arg_infer/array-repeat-expr-lib.rs @@ -1,6 +1,5 @@ //@ check-pass -#![feature(generic_arg_infer)] #![crate_type = "lib"] // Test that encoding the hallucinated `DefId` for the `_` const argument doesn't diff --git a/tests/ui/const-generics/generic_arg_infer/array-repeat-expr.rs b/tests/ui/const-generics/generic_arg_infer/array-repeat-expr.rs index 34091badfa7f1..fff9f2cc94d84 100644 --- a/tests/ui/const-generics/generic_arg_infer/array-repeat-expr.rs +++ b/tests/ui/const-generics/generic_arg_infer/array-repeat-expr.rs @@ -1,7 +1,6 @@ //@ run-pass // To avoid having to `or` gate `_` as an expr. -#![feature(generic_arg_infer)] fn foo() -> [u8; 3] { let x: [u8; _] = [0; _]; diff --git a/tests/ui/const-generics/generic_arg_infer/dont-use-defaults.rs b/tests/ui/const-generics/generic_arg_infer/dont-use-defaults.rs index 613ea9da99da3..d4a1468c04965 100644 --- a/tests/ui/const-generics/generic_arg_infer/dont-use-defaults.rs +++ b/tests/ui/const-generics/generic_arg_infer/dont-use-defaults.rs @@ -1,7 +1,4 @@ //@ run-pass -#![feature(generic_arg_infer)] - -// test that we dont use defaults to aide in type inference struct Foo; impl Foo { diff --git a/tests/ui/const-generics/generic_arg_infer/in-signature.rs b/tests/ui/const-generics/generic_arg_infer/in-signature.rs index cd852a269435e..cd0235bf45aa8 100644 --- a/tests/ui/const-generics/generic_arg_infer/in-signature.rs +++ b/tests/ui/const-generics/generic_arg_infer/in-signature.rs @@ -1,5 +1,4 @@ #![crate_type = "rlib"] -#![feature(generic_arg_infer)] struct Foo; struct Bar(T); diff --git a/tests/ui/const-generics/generic_arg_infer/in-signature.stderr b/tests/ui/const-generics/generic_arg_infer/in-signature.stderr index 12d84268f955c..f964fc8d2f2c5 100644 --- a/tests/ui/const-generics/generic_arg_infer/in-signature.stderr +++ b/tests/ui/const-generics/generic_arg_infer/in-signature.stderr @@ -1,5 +1,5 @@ error[E0121]: the placeholder `_` is not allowed within types on item signatures for return types - --> $DIR/in-signature.rs:7:21 + --> $DIR/in-signature.rs:6:21 | LL | fn arr_fn() -> [u8; _] { | -----^- @@ -8,7 +8,7 @@ LL | fn arr_fn() -> [u8; _] { | help: replace with the correct return type: `[u8; 3]` error[E0121]: the placeholder `_` is not allowed within types on item signatures for return types - --> $DIR/in-signature.rs:12:24 + --> $DIR/in-signature.rs:11:24 | LL | fn ty_fn() -> Bar { | ---------^- @@ -17,7 +17,7 @@ LL | fn ty_fn() -> Bar { | help: replace with the correct return type: `Bar` error[E0121]: the placeholder `_` is not allowed within types on item signatures for return types - --> $DIR/in-signature.rs:17:25 + --> $DIR/in-signature.rs:16:25 | LL | fn ty_fn_mixed() -> Bar<_, _> { | ----^--^- @@ -27,7 +27,7 @@ LL | fn ty_fn_mixed() -> Bar<_, _> { | help: replace with the correct return type: `Bar` error[E0121]: the placeholder `_` is not allowed within types on item signatures for constants - --> $DIR/in-signature.rs:22:20 + --> $DIR/in-signature.rs:21:20 | LL | const ARR_CT: [u8; _] = [0; 3]; | ^ not allowed in type signatures @@ -39,7 +39,7 @@ LL + const ARR_CT: [u8; 3] = [0; 3]; | error[E0121]: the placeholder `_` is not allowed within types on item signatures for static variables - --> $DIR/in-signature.rs:24:25 + --> $DIR/in-signature.rs:23:25 | LL | static ARR_STATIC: [u8; _] = [0; 3]; | ^ not allowed in type signatures @@ -51,7 +51,7 @@ LL + static ARR_STATIC: [u8; 3] = [0; 3]; | error[E0121]: the placeholder `_` is not allowed within types on item signatures for constants - --> $DIR/in-signature.rs:26:23 + --> $DIR/in-signature.rs:25:23 | LL | const TY_CT: Bar = Bar::(0); | ^ not allowed in type signatures @@ -63,7 +63,7 @@ LL + const TY_CT: Bar = Bar::(0); | error[E0121]: the placeholder `_` is not allowed within types on item signatures for static variables - --> $DIR/in-signature.rs:28:28 + --> $DIR/in-signature.rs:27:28 | LL | static TY_STATIC: Bar = Bar::(0); | ^ not allowed in type signatures @@ -75,7 +75,7 @@ LL + static TY_STATIC: Bar = Bar::(0); | error[E0121]: the placeholder `_` is not allowed within types on item signatures for constants - --> $DIR/in-signature.rs:30:24 + --> $DIR/in-signature.rs:29:24 | LL | const TY_CT_MIXED: Bar<_, _> = Bar::(0); | ^ ^ not allowed in type signatures @@ -89,7 +89,7 @@ LL + const TY_CT_MIXED: Bar = Bar::(0); | error[E0121]: the placeholder `_` is not allowed within types on item signatures for static variables - --> $DIR/in-signature.rs:32:29 + --> $DIR/in-signature.rs:31:29 | LL | static TY_STATIC_MIXED: Bar<_, _> = Bar::(0); | ^ ^ not allowed in type signatures @@ -103,19 +103,19 @@ LL + static TY_STATIC_MIXED: Bar = Bar::(0); | error[E0121]: the placeholder `_` is not allowed within types on item signatures for associated types - --> $DIR/in-signature.rs:51:23 + --> $DIR/in-signature.rs:50:23 | LL | type Assoc = [u8; _]; | ^ not allowed in type signatures error[E0121]: the placeholder `_` is not allowed within types on item signatures for associated types - --> $DIR/in-signature.rs:55:27 + --> $DIR/in-signature.rs:54:27 | LL | type Assoc = Bar; | ^ not allowed in type signatures error[E0121]: the placeholder `_` is not allowed within types on item signatures for associated types - --> $DIR/in-signature.rs:59:22 + --> $DIR/in-signature.rs:58:22 | LL | type Assoc = Bar<_, _>; | ^ ^ not allowed in type signatures @@ -123,19 +123,19 @@ LL | type Assoc = Bar<_, _>; | not allowed in type signatures error[E0121]: the placeholder `_` is not allowed within types on item signatures for associated constants - --> $DIR/in-signature.rs:35:21 + --> $DIR/in-signature.rs:34:21 | LL | const ARR: [u8; _]; | ^ not allowed in type signatures error[E0121]: the placeholder `_` is not allowed within types on item signatures for associated constants - --> $DIR/in-signature.rs:39:25 + --> $DIR/in-signature.rs:38:25 | LL | const ARR: Bar; | ^ not allowed in type signatures error[E0121]: the placeholder `_` is not allowed within types on item signatures for associated constants - --> $DIR/in-signature.rs:43:20 + --> $DIR/in-signature.rs:42:20 | LL | const ARR: Bar<_, _>; | ^ ^ not allowed in type signatures diff --git a/tests/ui/const-generics/generic_arg_infer/infer-arg-test.rs b/tests/ui/const-generics/generic_arg_infer/infer-arg-test.rs index c254b4ee09d2e..dcdcd250ea926 100644 --- a/tests/ui/const-generics/generic_arg_infer/infer-arg-test.rs +++ b/tests/ui/const-generics/generic_arg_infer/infer-arg-test.rs @@ -1,5 +1,3 @@ -#![feature(generic_arg_infer)] - struct All<'a, T, const N: usize> { v: &'a T, } diff --git a/tests/ui/const-generics/generic_arg_infer/infer-arg-test.stderr b/tests/ui/const-generics/generic_arg_infer/infer-arg-test.stderr index a9c57dbf26a0e..88645f839fc57 100644 --- a/tests/ui/const-generics/generic_arg_infer/infer-arg-test.stderr +++ b/tests/ui/const-generics/generic_arg_infer/infer-arg-test.stderr @@ -1,17 +1,17 @@ error: expected identifier, found reserved identifier `_` - --> $DIR/infer-arg-test.rs:7:17 + --> $DIR/infer-arg-test.rs:5:17 | LL | struct BadInfer<_>; | ^ expected identifier, found reserved identifier error: expected identifier, found reserved identifier `_` - --> $DIR/infer-arg-test.rs:13:17 + --> $DIR/infer-arg-test.rs:11:17 | LL | fn bad_infer_fn<_>() {} | ^ expected identifier, found reserved identifier error[E0392]: type parameter `_` is never used - --> $DIR/infer-arg-test.rs:7:17 + --> $DIR/infer-arg-test.rs:5:17 | LL | struct BadInfer<_>; | ^ unused type parameter @@ -20,7 +20,7 @@ LL | struct BadInfer<_>; = help: if you intended `_` to be a const parameter, use `const _: /* Type */` instead error[E0107]: struct takes 2 generic arguments but 3 generic arguments were supplied - --> $DIR/infer-arg-test.rs:18:10 + --> $DIR/infer-arg-test.rs:16:10 | LL | let a: All<_, _, _>; | ^^^ --- help: remove the unnecessary generic argument @@ -28,7 +28,7 @@ LL | let a: All<_, _, _>; | expected 2 generic arguments | note: struct defined here, with 2 generic parameters: `T`, `N` - --> $DIR/infer-arg-test.rs:3:8 + --> $DIR/infer-arg-test.rs:1:8 | LL | struct All<'a, T, const N: usize> { | ^^^ - -------------- diff --git a/tests/ui/const-generics/generic_arg_infer/infer_arg_and_const_arg.rs b/tests/ui/const-generics/generic_arg_infer/infer_arg_and_const_arg.rs index 35b3fe4f4359b..e82250444d93f 100644 --- a/tests/ui/const-generics/generic_arg_infer/infer_arg_and_const_arg.rs +++ b/tests/ui/const-generics/generic_arg_infer/infer_arg_and_const_arg.rs @@ -1,5 +1,4 @@ //@ check-pass -#![feature(generic_arg_infer)] struct Foo; struct Bar; diff --git a/tests/ui/const-generics/generic_arg_infer/issue-91614.rs b/tests/ui/const-generics/generic_arg_infer/issue-91614.rs index a386b1e5c2bf5..4a3d85499da81 100644 --- a/tests/ui/const-generics/generic_arg_infer/issue-91614.rs +++ b/tests/ui/const-generics/generic_arg_infer/issue-91614.rs @@ -1,5 +1,4 @@ #![feature(portable_simd)] -#![feature(generic_arg_infer)] use std::simd::Mask; fn main() { diff --git a/tests/ui/const-generics/generic_arg_infer/issue-91614.stderr b/tests/ui/const-generics/generic_arg_infer/issue-91614.stderr index b07e1f29d0d90..164bcc7111ca6 100644 --- a/tests/ui/const-generics/generic_arg_infer/issue-91614.stderr +++ b/tests/ui/const-generics/generic_arg_infer/issue-91614.stderr @@ -1,5 +1,5 @@ error[E0284]: type annotations needed for `Mask<_, _>` - --> $DIR/issue-91614.rs:6:9 + --> $DIR/issue-91614.rs:5:9 | LL | let y = Mask::<_, _>::splat(false); | ^ ------------ type must be known at this point @@ -12,7 +12,7 @@ LL | let y: Mask<_, N> = Mask::<_, _>::splat(false); | ++++++++++++ error[E0284]: type annotations needed for `Mask<_, _>` - --> $DIR/issue-91614.rs:6:9 + --> $DIR/issue-91614.rs:5:9 | LL | let y = Mask::<_, _>::splat(false); | ^ -------------------------- type must be known at this point diff --git a/tests/ui/const-generics/generic_arg_infer/parend_infer.nogate.stderr b/tests/ui/const-generics/generic_arg_infer/parend_infer.nogate.stderr deleted file mode 100644 index d0a5da9676df4..0000000000000 --- a/tests/ui/const-generics/generic_arg_infer/parend_infer.nogate.stderr +++ /dev/null @@ -1,53 +0,0 @@ -error[E0658]: const arguments cannot yet be inferred with `_` - --> $DIR/parend_infer.rs:24:16 - | -LL | let c: Foo<_> = Foo::<1>; - | ^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: const arguments cannot yet be inferred with `_` - --> $DIR/parend_infer.rs:26:16 - | -LL | let c: Foo<(_)> = Foo::<1>; - | ^^^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: const arguments cannot yet be inferred with `_` - --> $DIR/parend_infer.rs:28:16 - | -LL | let c: Foo<(((_)))> = Foo::<1>; - | ^^^^^^^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: using `_` for array lengths is unstable - --> $DIR/parend_infer.rs:17:17 - | -LL | let b: [u8; (_)] = [1; (((((_)))))]; - | ^^^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: using `_` for array lengths is unstable - --> $DIR/parend_infer.rs:17:28 - | -LL | let b: [u8; (_)] = [1; (((((_)))))]; - | ^^^^^^^^^^^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error: aborting due to 5 previous errors - -For more information about this error, try `rustc --explain E0658`. diff --git a/tests/ui/const-generics/generic_arg_infer/parend_infer.rs b/tests/ui/const-generics/generic_arg_infer/parend_infer.rs index 3dc27a702de40..9d7df8016cb1e 100644 --- a/tests/ui/const-generics/generic_arg_infer/parend_infer.rs +++ b/tests/ui/const-generics/generic_arg_infer/parend_infer.rs @@ -1,6 +1,4 @@ -//@[gate] check-pass -//@ revisions: gate nogate -#![cfg_attr(gate, feature(generic_arg_infer))] +//@ check-pass struct Foo; @@ -15,16 +13,11 @@ fn main() { // AST Exprs similarly preserve parens for pretty printing reasons. #[rustfmt::skip] let b: [u8; (_)] = [1; (((((_)))))]; - //[nogate]~^ error: using `_` for array lengths is unstable - //[nogate]~| error: using `_` for array lengths is unstable let b: [u8; 2] = b; // This is the same case as AST types as the parser doesn't distinguish between const // and type args when they share syntax let c: Foo<_> = Foo::<1>; - //[nogate]~^ error: const arguments cannot yet be inferred with `_` let c: Foo<(_)> = Foo::<1>; - //[nogate]~^ error: const arguments cannot yet be inferred with `_` let c: Foo<(((_)))> = Foo::<1>; - //[nogate]~^ error: const arguments cannot yet be inferred with `_` } diff --git a/tests/ui/const-generics/generic_const_exprs/const_kind_expr/wf_obligation.rs b/tests/ui/const-generics/generic_const_exprs/const_kind_expr/wf_obligation.rs index 6093fc70b1696..a82ea45b1232c 100644 --- a/tests/ui/const-generics/generic_const_exprs/const_kind_expr/wf_obligation.rs +++ b/tests/ui/const-generics/generic_const_exprs/const_kind_expr/wf_obligation.rs @@ -1,4 +1,4 @@ -#![feature(generic_const_exprs, generic_arg_infer)] +#![feature(generic_const_exprs)] #![allow(incomplete_features)] // minimized repro for #105205 diff --git a/tests/ui/const-generics/generic_const_exprs/poly-const-uneval-ice-106423.rs b/tests/ui/const-generics/generic_const_exprs/poly-const-uneval-ice-106423.rs index ed5ba32b62105..eca0404fcd01b 100644 --- a/tests/ui/const-generics/generic_const_exprs/poly-const-uneval-ice-106423.rs +++ b/tests/ui/const-generics/generic_const_exprs/poly-const-uneval-ice-106423.rs @@ -3,7 +3,7 @@ //@ edition:2021 //@ check-pass -#![feature(generic_const_exprs, generic_arg_infer)] +#![feature(generic_const_exprs)] #![allow(incomplete_features)] #![allow(unused)] @@ -41,17 +41,13 @@ where DigitalFilter::Ba(zpk) } -pub fn zpk2tf_st( - _z: &Arr, - _p: &Arr, -) -> BaFormatFilter<{ N + 1 }> +pub fn zpk2tf_st(_z: &Arr, _p: &Arr) -> BaFormatFilter<{ N + 1 }> where [(); N + 1]: Sized, { BaFormatFilter {} } - fn main() { - iirfilter_st_copy::<4, 2>([10., 50.,]); + iirfilter_st_copy::<4, 2>([10., 50.]); } diff --git a/tests/ui/const-generics/generic_const_parameter_types/bad_inference.rs b/tests/ui/const-generics/generic_const_parameter_types/bad_inference.rs index de2e687e870c4..9748c14d655df 100644 --- a/tests/ui/const-generics/generic_const_parameter_types/bad_inference.rs +++ b/tests/ui/const-generics/generic_const_parameter_types/bad_inference.rs @@ -1,9 +1,4 @@ -#![feature( - adt_const_params, - unsized_const_params, - generic_const_parameter_types, - generic_arg_infer -)] +#![feature(adt_const_params, unsized_const_params, generic_const_parameter_types)] #![allow(incomplete_features)] use std::marker::ConstParamTy_; diff --git a/tests/ui/const-generics/generic_const_parameter_types/bad_inference.stderr b/tests/ui/const-generics/generic_const_parameter_types/bad_inference.stderr index 1ac67fe622b55..4652187b9ce18 100644 --- a/tests/ui/const-generics/generic_const_parameter_types/bad_inference.stderr +++ b/tests/ui/const-generics/generic_const_parameter_types/bad_inference.stderr @@ -1,11 +1,11 @@ error: anonymous constants with inferred types are not yet supported - --> $DIR/bad_inference.rs:17:25 + --> $DIR/bad_inference.rs:12:25 | LL | let a = foo::<_, _, { [12_u8; 2] }>(); | ^^^^^^^^^^^^^^ error: anonymous constants with inferred types are not yet supported - --> $DIR/bad_inference.rs:21:34 + --> $DIR/bad_inference.rs:16:34 | LL | let b: [u8; 2] = foo::<_, _, { [12; _] }>(); | ^^^^^^^^^^^ diff --git a/tests/ui/const-generics/generic_const_parameter_types/evaluate_const_parameter_in_mir.rs b/tests/ui/const-generics/generic_const_parameter_types/evaluate_const_parameter_in_mir.rs index 910deb6632d73..9f3ab1be2502e 100644 --- a/tests/ui/const-generics/generic_const_parameter_types/evaluate_const_parameter_in_mir.rs +++ b/tests/ui/const-generics/generic_const_parameter_types/evaluate_const_parameter_in_mir.rs @@ -1,11 +1,6 @@ //@ check-pass -#![feature( - adt_const_params, - unsized_const_params, - generic_const_parameter_types, - generic_arg_infer -)] +#![feature(adt_const_params, unsized_const_params, generic_const_parameter_types)] #![allow(incomplete_features)] use std::marker::ConstParamTy_; diff --git a/tests/ui/const-generics/generic_const_parameter_types/inferred_from_arg.rs b/tests/ui/const-generics/generic_const_parameter_types/inferred_from_arg.rs index d655fc174ee5c..a4e9aa54c01bb 100644 --- a/tests/ui/const-generics/generic_const_parameter_types/inferred_from_arg.rs +++ b/tests/ui/const-generics/generic_const_parameter_types/inferred_from_arg.rs @@ -1,6 +1,6 @@ //@ check-pass -#![feature(adt_const_params, generic_arg_infer, generic_const_parameter_types)] +#![feature(adt_const_params, generic_const_parameter_types)] #![expect(incomplete_features)] struct Bar; diff --git a/tests/ui/const-generics/generic_const_parameter_types/unrelated_inferred_arg.rs b/tests/ui/const-generics/generic_const_parameter_types/unrelated_inferred_arg.rs index b389e12884ea5..80117a27a23d2 100644 --- a/tests/ui/const-generics/generic_const_parameter_types/unrelated_inferred_arg.rs +++ b/tests/ui/const-generics/generic_const_parameter_types/unrelated_inferred_arg.rs @@ -1,11 +1,6 @@ //@ check-pass -#![feature( - adt_const_params, - unsized_const_params, - generic_const_parameter_types, - generic_arg_infer -)] +#![feature(adt_const_params, unsized_const_params, generic_const_parameter_types)] #![allow(incomplete_features)] use std::marker::ConstParamTy_; diff --git a/tests/ui/const-generics/issues/issue-62878.min.stderr b/tests/ui/const-generics/issues/issue-62878.min.stderr index d3d4fa4387109..d7ca0e1e2db59 100644 --- a/tests/ui/const-generics/issues/issue-62878.min.stderr +++ b/tests/ui/const-generics/issues/issue-62878.min.stderr @@ -16,17 +16,6 @@ help: add `#![feature(adt_const_params)]` to the crate attributes to enable more LL + #![feature(adt_const_params)] | -error[E0658]: const arguments cannot yet be inferred with `_` - --> $DIR/issue-62878.rs:10:11 - | -LL | foo::<_, { [1] }>(); - | ^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error: aborting due to 3 previous errors +error: aborting due to 2 previous errors -Some errors have detailed explanations: E0658, E0770. -For more information about an error, try `rustc --explain E0658`. +For more information about this error, try `rustc --explain E0770`. diff --git a/tests/ui/const-generics/issues/issue-62878.rs b/tests/ui/const-generics/issues/issue-62878.rs index c80b46ddbc440..0c143b34ce91e 100644 --- a/tests/ui/const-generics/issues/issue-62878.rs +++ b/tests/ui/const-generics/issues/issue-62878.rs @@ -1,5 +1,5 @@ //@ revisions: full min -#![cfg_attr(full, feature(adt_const_params, generic_arg_infer))] +#![cfg_attr(full, feature(adt_const_params))] #![cfg_attr(full, allow(incomplete_features))] fn foo() {} @@ -8,5 +8,4 @@ fn foo() {} fn main() { foo::<_, { [1] }>(); - //[min]~^ ERROR: const arguments cannot yet be inferred with `_` } diff --git a/tests/ui/const-generics/min_const_generics/inferred_const.rs b/tests/ui/const-generics/min_const_generics/inferred_const.rs index 0256ef732a346..4a4fb417ab1dd 100644 --- a/tests/ui/const-generics/min_const_generics/inferred_const.rs +++ b/tests/ui/const-generics/min_const_generics/inferred_const.rs @@ -1,4 +1,3 @@ -#![feature(generic_arg_infer)] //@ run-pass fn foo(_data: [u32; N]) -> [u32; K] { diff --git a/tests/ui/feature-gates/feature-gate-generic_arg_infer.normal.stderr b/tests/ui/feature-gates/feature-gate-generic_arg_infer.normal.stderr deleted file mode 100644 index 73e6988b09cc8..0000000000000 --- a/tests/ui/feature-gates/feature-gate-generic_arg_infer.normal.stderr +++ /dev/null @@ -1,33 +0,0 @@ -error[E0658]: using `_` for array lengths is unstable - --> $DIR/feature-gate-generic_arg_infer.rs:13:18 - | -LL | let _y: [u8; _] = [0; 3]; - | ^ help: consider specifying the array length: `3` - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: const arguments cannot yet be inferred with `_` - --> $DIR/feature-gate-generic_arg_infer.rs:18:20 - | -LL | let _x = foo::<_>([1, 2]); - | ^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error[E0658]: using `_` for array lengths is unstable - --> $DIR/feature-gate-generic_arg_infer.rs:11:27 - | -LL | let _x: [u8; 3] = [0; _]; - | ^ - | - = note: see issue #85077 for more information - = help: add `#![feature(generic_arg_infer)]` to the crate attributes to enable - = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date - -error: aborting due to 3 previous errors - -For more information about this error, try `rustc --explain E0658`. diff --git a/tests/ui/feature-gates/feature-gate-generic_arg_infer.rs b/tests/ui/feature-gates/feature-gate-generic_arg_infer.rs deleted file mode 100644 index 147978b0557a6..0000000000000 --- a/tests/ui/feature-gates/feature-gate-generic_arg_infer.rs +++ /dev/null @@ -1,21 +0,0 @@ -//@ [feature] run-pass -//@ revisions: normal feature - -#![cfg_attr(feature, feature(generic_arg_infer))] - -fn foo(_: [u8; N]) -> [u8; N] { - [0; N] -} - -fn bar() { - let _x: [u8; 3] = [0; _]; - //[normal]~^ ERROR: using `_` for array lengths is unstable - let _y: [u8; _] = [0; 3]; - //[normal]~^ ERROR: using `_` for array lengths is unstable -} - -fn main() { - let _x = foo::<_>([1, 2]); - //[normal]~^ ERROR: const arguments cannot yet be inferred with `_` - bar(); -} diff --git a/tests/ui/lang-items/lang-item-generic-requirements.rs b/tests/ui/lang-items/lang-item-generic-requirements.rs index 25a4ff283ba6a..602c7f534288e 100644 --- a/tests/ui/lang-items/lang-item-generic-requirements.rs +++ b/tests/ui/lang-items/lang-item-generic-requirements.rs @@ -50,14 +50,14 @@ fn ice() { let arr = [0; 5]; //~^ ERROR requires `copy` lang_item let _ = arr[2]; - //~^ ERROR cannot index into a value of type `[{integer}; 5]` + //~^ ERROR: cannot index into a value of type `[{integer}; 5]` // Use phantomdata let _ = MyPhantomData::<(), i32>; // Use Foo let _: () = Foo; - //~^ ERROR mismatched types + //~^ ERROR: mismatched types } // use `start` diff --git a/tests/ui/object-lifetime/object-lifetime-default-inferred.rs b/tests/ui/object-lifetime/object-lifetime-default-inferred.rs index 5abe09e272920..1ab821764de08 100644 --- a/tests/ui/object-lifetime/object-lifetime-default-inferred.rs +++ b/tests/ui/object-lifetime/object-lifetime-default-inferred.rs @@ -4,7 +4,6 @@ #![allow(dead_code)] -#![feature(generic_arg_infer)] trait Test { fn foo(&self) { } diff --git a/tests/ui/parser/issues/issue-14303-fncall.rs b/tests/ui/parser/issues/issue-14303-fncall.rs index 8f7fbec947067..1e8ef6af3fa93 100644 --- a/tests/ui/parser/issues/issue-14303-fncall.rs +++ b/tests/ui/parser/issues/issue-14303-fncall.rs @@ -1,7 +1,5 @@ -//@ revisions: full generic_arg // can't run rustfix because it doesn't handle multipart suggestions correctly // we need the above to avoid ast borrowck failure in recovered code -#![cfg_attr(generic_arg, feature(generic_arg_infer))] struct S<'a, T> { a: &'a T, @@ -10,8 +8,7 @@ struct S<'a, T> { fn foo<'a, 'b>(start: &'a usize, end: &'a usize) { let _x = (*start..*end).map(|x| S { a: start, b: end }).collect::>>(); - //[generic_arg]~^ ERROR placeholder provided when a lifetime was expected - //[full]~^^ ERROR placeholder provided when a lifetime was expected + //~^ ERROR placeholder provided when a lifetime was expected } fn main() {} diff --git a/tests/ui/parser/issues/issue-14303-fncall.stderr b/tests/ui/parser/issues/issue-14303-fncall.stderr new file mode 100644 index 0000000000000..c42a23fa9d3e9 --- /dev/null +++ b/tests/ui/parser/issues/issue-14303-fncall.stderr @@ -0,0 +1,9 @@ +error[E0747]: placeholder provided when a lifetime was expected + --> $DIR/issue-14303-fncall.rs:10:77 + | +LL | let _x = (*start..*end).map(|x| S { a: start, b: end }).collect::>>(); + | ^ + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0747`. diff --git a/tests/ui/pattern/slice-array-infer.rs b/tests/ui/pattern/slice-array-infer.rs index fdead488ea11d..8d471b31beaf6 100644 --- a/tests/ui/pattern/slice-array-infer.rs +++ b/tests/ui/pattern/slice-array-infer.rs @@ -1,7 +1,6 @@ //@ check-pass #![allow(unused_variables)] -#![feature(generic_arg_infer)] struct Zeroes; impl Into<&'static [usize; 3]> for Zeroes { diff --git a/tests/ui/repeat-expr/copy-check-const-element-uninferred-count.rs b/tests/ui/repeat-expr/copy-check-const-element-uninferred-count.rs index 6115146539c19..722fdae6c99a2 100644 --- a/tests/ui/repeat-expr/copy-check-const-element-uninferred-count.rs +++ b/tests/ui/repeat-expr/copy-check-const-element-uninferred-count.rs @@ -1,5 +1,3 @@ -#![feature(generic_arg_infer)] - // Test when deferring repeat expr copy checks to end of typechecking whether elements // that are const items allow for repeat counts to go uninferred without an error being // emitted if they would later wind up inferred by integer fallback. diff --git a/tests/ui/repeat-expr/copy-check-const-element-uninferred-count.stderr b/tests/ui/repeat-expr/copy-check-const-element-uninferred-count.stderr index 2f52537fa9407..9c9cfefd66383 100644 --- a/tests/ui/repeat-expr/copy-check-const-element-uninferred-count.stderr +++ b/tests/ui/repeat-expr/copy-check-const-element-uninferred-count.stderr @@ -1,5 +1,5 @@ error[E0284]: type annotations needed for `[String; _]` - --> $DIR/copy-check-const-element-uninferred-count.rs:64:9 + --> $DIR/copy-check-const-element-uninferred-count.rs:62:9 | LL | let a = [const { String::new() }; _]; | ^ ---------------------------- type must be known at this point diff --git a/tests/ui/repeat-expr/copy-check-deferred-after-fallback.rs b/tests/ui/repeat-expr/copy-check-deferred-after-fallback.rs index 3f310f07de0fe..a6bd5b299c96a 100644 --- a/tests/ui/repeat-expr/copy-check-deferred-after-fallback.rs +++ b/tests/ui/repeat-expr/copy-check-deferred-after-fallback.rs @@ -1,5 +1,3 @@ -#![feature(generic_arg_infer)] - // Test when deferring repeat expr copy checks to end of typechecking whether they're // checked before integer fallback occurs or not. We accomplish this by having a repeat // count that can only be inferred after integer fallback has occured. This test will diff --git a/tests/ui/repeat-expr/copy-check-deferred-after-fallback.stderr b/tests/ui/repeat-expr/copy-check-deferred-after-fallback.stderr index 103b074dda7c8..0cd7ebe7494ff 100644 --- a/tests/ui/repeat-expr/copy-check-deferred-after-fallback.stderr +++ b/tests/ui/repeat-expr/copy-check-deferred-after-fallback.stderr @@ -1,5 +1,5 @@ error[E0282]: type annotations needed for `[Foo<{integer}>; _]` - --> $DIR/copy-check-deferred-after-fallback.rs:39:9 + --> $DIR/copy-check-deferred-after-fallback.rs:37:9 | LL | let b = [Foo(PhantomData); _]; | ^ ---------------- type must be known at this point diff --git a/tests/ui/repeat-expr/copy-check-deferred-before-fallback.rs b/tests/ui/repeat-expr/copy-check-deferred-before-fallback.rs index 4fbb8f0a00caf..23b13348f5a67 100644 --- a/tests/ui/repeat-expr/copy-check-deferred-before-fallback.rs +++ b/tests/ui/repeat-expr/copy-check-deferred-before-fallback.rs @@ -1,5 +1,4 @@ //@ check-pass -#![feature(generic_arg_infer)] // Test when deferring repeat expr checks to end of typechecking whether they're // checked before integer fallback occurs. We accomplish this by having the repeat diff --git a/tests/ui/repeat-expr/copy-check-inference-side-effects.rs b/tests/ui/repeat-expr/copy-check-inference-side-effects.rs index 4e3bfdead26b4..8587f1f9ce91c 100644 --- a/tests/ui/repeat-expr/copy-check-inference-side-effects.rs +++ b/tests/ui/repeat-expr/copy-check-inference-side-effects.rs @@ -1,5 +1,3 @@ -#![feature(generic_arg_infer)] - struct Foo; impl Clone for Foo<1> { diff --git a/tests/ui/repeat-expr/copy-check-inference-side-effects.stderr b/tests/ui/repeat-expr/copy-check-inference-side-effects.stderr index 505beff0f6b2e..bf4ae9b60bb31 100644 --- a/tests/ui/repeat-expr/copy-check-inference-side-effects.stderr +++ b/tests/ui/repeat-expr/copy-check-inference-side-effects.stderr @@ -1,5 +1,5 @@ error[E0282]: type annotations needed for `[Foo<_>; 2]` - --> $DIR/copy-check-inference-side-effects.rs:17:9 + --> $DIR/copy-check-inference-side-effects.rs:15:9 | LL | let a /* : [Foo; 2] */ = [Foo::<_>; 2]; | ^ @@ -13,7 +13,7 @@ LL | let a: [Foo; 2] /* : [Foo; 2] */ = [Foo::<_>; 2]; | +++++++++++++ error[E0282]: type annotations needed for `[String; _]` - --> $DIR/copy-check-inference-side-effects.rs:27:9 + --> $DIR/copy-check-inference-side-effects.rs:25:9 | LL | let b /* : [String; ?x] */ = ["string".to_string(); _]; | ^ -------------------- type must be known at this point diff --git a/tests/ui/repeat-expr/copy-check-when-count-inferred-later.rs b/tests/ui/repeat-expr/copy-check-when-count-inferred-later.rs index b9d123cbefae6..72467e6f32e24 100644 --- a/tests/ui/repeat-expr/copy-check-when-count-inferred-later.rs +++ b/tests/ui/repeat-expr/copy-check-when-count-inferred-later.rs @@ -1,5 +1,3 @@ -#![feature(generic_arg_infer)] - // Test that we enforce repeat expr element types are `Copy` even // when the repeat count is only inferred at a later point in type // checking. diff --git a/tests/ui/repeat-expr/copy-check-when-count-inferred-later.stderr b/tests/ui/repeat-expr/copy-check-when-count-inferred-later.stderr index 1c862f2b606a8..6b8049e77cc51 100644 --- a/tests/ui/repeat-expr/copy-check-when-count-inferred-later.stderr +++ b/tests/ui/repeat-expr/copy-check-when-count-inferred-later.stderr @@ -1,5 +1,5 @@ error[E0277]: the trait bound `String: Copy` is not satisfied - --> $DIR/copy-check-when-count-inferred-later.rs:8:14 + --> $DIR/copy-check-when-count-inferred-later.rs:6:14 | LL | let a = [String::new(); _]; | ^^^^^^^^^^^^^ the trait `Copy` is not implemented for `String` diff --git a/tests/ui/repeat-expr/dont-require-copy-on-infer.rs b/tests/ui/repeat-expr/dont-require-copy-on-infer.rs index e81bf1595be1f..ad0e4bd2be0ce 100644 --- a/tests/ui/repeat-expr/dont-require-copy-on-infer.rs +++ b/tests/ui/repeat-expr/dont-require-copy-on-infer.rs @@ -1,5 +1,4 @@ //@ check-pass -#![feature(generic_arg_infer)] fn main() { let a: [_; 1] = [String::new(); _]; diff --git a/tests/ui/repeat-expr/no-conservative-copy-impl-requirement.rs b/tests/ui/repeat-expr/no-conservative-copy-impl-requirement.rs index eb70df62996fb..df79ad51b4215 100644 --- a/tests/ui/repeat-expr/no-conservative-copy-impl-requirement.rs +++ b/tests/ui/repeat-expr/no-conservative-copy-impl-requirement.rs @@ -1,5 +1,3 @@ -#![feature(generic_arg_infer)] - struct Foo; impl Clone for Foo<1> { diff --git a/tests/ui/repeat-expr/no-conservative-copy-impl-requirement.stderr b/tests/ui/repeat-expr/no-conservative-copy-impl-requirement.stderr index 04f8ff33fdab3..bf1e46e4ef829 100644 --- a/tests/ui/repeat-expr/no-conservative-copy-impl-requirement.stderr +++ b/tests/ui/repeat-expr/no-conservative-copy-impl-requirement.stderr @@ -1,5 +1,5 @@ error[E0282]: type annotations needed for `&[Foo<_>; _]` - --> $DIR/no-conservative-copy-impl-requirement.rs:17:9 + --> $DIR/no-conservative-copy-impl-requirement.rs:15:9 | LL | let x = &[Foo::<_>; _]; | ^ -------- type must be known at this point diff --git a/tests/ui/simd/const-err-trumps-simd-err.rs b/tests/ui/simd/const-err-trumps-simd-err.rs index 8d9870855f807..33f0abb06f3ea 100644 --- a/tests/ui/simd/const-err-trumps-simd-err.rs +++ b/tests/ui/simd/const-err-trumps-simd-err.rs @@ -4,7 +4,6 @@ //! Make sure that monomorphization-time const errors from `static_assert` take priority over the //! error from simd_extract. Basically this checks that if a const fails to evaluate in some //! function, we don't bother codegen'ing the function. -#![feature(generic_arg_infer)] #![feature(core_intrinsics)] #![feature(repr_simd)] diff --git a/tests/ui/simd/const-err-trumps-simd-err.stderr b/tests/ui/simd/const-err-trumps-simd-err.stderr index d4ba54a28da7e..93d1fce637f2f 100644 --- a/tests/ui/simd/const-err-trumps-simd-err.stderr +++ b/tests/ui/simd/const-err-trumps-simd-err.stderr @@ -1,17 +1,17 @@ error[E0080]: evaluation panicked: assertion failed: LANE < 4 - --> $DIR/const-err-trumps-simd-err.rs:18:13 + --> $DIR/const-err-trumps-simd-err.rs:17:13 | LL | const { assert!(LANE < 4); } // the error should be here... | ^^^^^^^^^^^^^^^^^ evaluation of `get_elem::<4>::{constant#0}` failed here note: erroneous constant encountered - --> $DIR/const-err-trumps-simd-err.rs:18:5 + --> $DIR/const-err-trumps-simd-err.rs:17:5 | LL | const { assert!(LANE < 4); } // the error should be here... | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ note: the above error was encountered while instantiating `fn get_elem::<4>` - --> $DIR/const-err-trumps-simd-err.rs:24:5 + --> $DIR/const-err-trumps-simd-err.rs:23:5 | LL | get_elem::<4>(int8x4_t([0, 0, 0, 0])); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/ui/span/issue-42234-unknown-receiver-type.rs b/tests/ui/span/issue-42234-unknown-receiver-type.rs index 53d1e3eed820e..8f7bbf0fe5e09 100644 --- a/tests/ui/span/issue-42234-unknown-receiver-type.rs +++ b/tests/ui/span/issue-42234-unknown-receiver-type.rs @@ -1,6 +1,3 @@ -//@ revisions: full generic_arg -#![cfg_attr(generic_arg, feature(generic_arg_infer))] - // When the type of a method call's receiver is unknown, the span should point // to the receiver (and not the entire call, as was previously the case before // the fix of which this tests). diff --git a/tests/ui/span/issue-42234-unknown-receiver-type.stderr b/tests/ui/span/issue-42234-unknown-receiver-type.stderr new file mode 100644 index 0000000000000..10308ec07da5a --- /dev/null +++ b/tests/ui/span/issue-42234-unknown-receiver-type.stderr @@ -0,0 +1,23 @@ +error[E0282]: type annotations needed + --> $DIR/issue-42234-unknown-receiver-type.rs:6:24 + | +LL | let x: Option<_> = None; + | ^^^^ cannot infer type of the type parameter `T` declared on the enum `Option` +LL | x.unwrap().method_that_could_exist_on_some_type(); + | ---------- type must be known at this point + | +help: consider specifying the generic argument + | +LL | let x: Option<_> = None::; + | +++++ + +error[E0282]: type annotations needed + --> $DIR/issue-42234-unknown-receiver-type.rs:12:10 + | +LL | .sum::<_>() + | ^^^ cannot infer type of the type parameter `S` declared on the method `sum` + | + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0282`. diff --git a/tests/ui/traits/const-traits/tilde-const-and-const-params.rs b/tests/ui/traits/const-traits/tilde-const-and-const-params.rs index 706c77b6200a8..428223d92c01e 100644 --- a/tests/ui/traits/const-traits/tilde-const-and-const-params.rs +++ b/tests/ui/traits/const-traits/tilde-const-and-const-params.rs @@ -1,5 +1,4 @@ #![feature(const_trait_impl)] -#![feature(generic_arg_infer)] #![feature(generic_const_exprs)] #![allow(incomplete_features)] diff --git a/tests/ui/traits/const-traits/tilde-const-and-const-params.stderr b/tests/ui/traits/const-traits/tilde-const-and-const-params.stderr index f77d63db054a0..95e684bd0c47d 100644 --- a/tests/ui/traits/const-traits/tilde-const-and-const-params.stderr +++ b/tests/ui/traits/const-traits/tilde-const-and-const-params.stderr @@ -1,35 +1,35 @@ error: `~const` is not allowed here - --> $DIR/tilde-const-and-const-params.rs:9:15 + --> $DIR/tilde-const-and-const-params.rs:8:15 | LL | fn add(self) -> Foo<{ A::add(N) }> { | ^^^^^^ | note: this function is not `const`, so it cannot have `~const` trait bounds - --> $DIR/tilde-const-and-const-params.rs:9:8 + --> $DIR/tilde-const-and-const-params.rs:8:8 | LL | fn add(self) -> Foo<{ A::add(N) }> { | ^^^ error: `~const` is not allowed here - --> $DIR/tilde-const-and-const-params.rs:27:11 + --> $DIR/tilde-const-and-const-params.rs:26:11 | LL | fn bar(_: Foo) -> Foo<{ A::add(N) }> { | ^^^^^^ | note: this function is not `const`, so it cannot have `~const` trait bounds - --> $DIR/tilde-const-and-const-params.rs:27:4 + --> $DIR/tilde-const-and-const-params.rs:26:4 | LL | fn bar(_: Foo) -> Foo<{ A::add(N) }> { | ^^^ error[E0277]: the trait bound `A: const Add42` is not satisfied - --> $DIR/tilde-const-and-const-params.rs:27:61 + --> $DIR/tilde-const-and-const-params.rs:26:61 | LL | fn bar(_: Foo) -> Foo<{ A::add(N) }> { | ^ error[E0277]: the trait bound `A: const Add42` is not satisfied - --> $DIR/tilde-const-and-const-params.rs:9:44 + --> $DIR/tilde-const-and-const-params.rs:8:44 | LL | fn add(self) -> Foo<{ A::add(N) }> { | ^ From 35c5144394c1b93784867d330f694fa7c8f480e3 Mon Sep 17 00:00:00 2001 From: "Celina G. Val" Date: Wed, 11 Jun 2025 10:43:59 -0700 Subject: [PATCH 06/32] Move rayon-core to rustc_thread_pool files as is This commit literally copied the directory rayon-core from revision `5fadf44`. Link: https://github.com/rust-lang/rustc-rayon/tree/5fadf44/rayon-core --- compiler/rustc_thread_pool/Cargo.toml | 56 + compiler/rustc_thread_pool/LICENSE-APACHE | 201 ++++ compiler/rustc_thread_pool/LICENSE-MIT | 25 + compiler/rustc_thread_pool/README.md | 13 + .../rustc_thread_pool/src/broadcast/mod.rs | 151 +++ .../rustc_thread_pool/src/broadcast/test.rs | 263 +++++ .../rustc_thread_pool/src/compile_fail/mod.rs | 7 + .../src/compile_fail/quicksort_race1.rs | 28 + .../src/compile_fail/quicksort_race2.rs | 28 + .../src/compile_fail/quicksort_race3.rs | 28 + .../src/compile_fail/rc_return.rs | 17 + .../src/compile_fail/rc_upvar.rs | 9 + .../src/compile_fail/scope_join_bad.rs | 24 + compiler/rustc_thread_pool/src/job.rs | 277 +++++ compiler/rustc_thread_pool/src/join/mod.rs | 202 ++++ compiler/rustc_thread_pool/src/join/test.rs | 150 +++ compiler/rustc_thread_pool/src/latch.rs | 459 ++++++++ compiler/rustc_thread_pool/src/lib.rs | 922 +++++++++++++++ compiler/rustc_thread_pool/src/private.rs | 26 + compiler/rustc_thread_pool/src/registry.rs | 1048 +++++++++++++++++ compiler/rustc_thread_pool/src/scope/mod.rs | 783 ++++++++++++ compiler/rustc_thread_pool/src/scope/test.rs | 622 ++++++++++ .../rustc_thread_pool/src/sleep/README.md | 252 ++++ .../rustc_thread_pool/src/sleep/counters.rs | 277 +++++ compiler/rustc_thread_pool/src/sleep/mod.rs | 392 ++++++ compiler/rustc_thread_pool/src/spawn/mod.rs | 164 +++ compiler/rustc_thread_pool/src/spawn/test.rs | 255 ++++ compiler/rustc_thread_pool/src/test.rs | 200 ++++ .../rustc_thread_pool/src/thread_pool/mod.rs | 514 ++++++++ .../rustc_thread_pool/src/thread_pool/test.rs | 418 +++++++ compiler/rustc_thread_pool/src/tlv.rs | 31 + compiler/rustc_thread_pool/src/unwind.rs | 31 + .../rustc_thread_pool/src/worker_local.rs | 78 ++ .../tests/double_init_fail.rs | 15 + .../tests/init_zero_threads.rs | 10 + .../rustc_thread_pool/tests/scope_join.rs | 45 + .../tests/scoped_threadpool.rs | 99 ++ .../rustc_thread_pool/tests/simple_panic.rs | 7 + .../tests/stack_overflow_crash.rs | 98 ++ 39 files changed, 8225 insertions(+) create mode 100644 compiler/rustc_thread_pool/Cargo.toml create mode 100644 compiler/rustc_thread_pool/LICENSE-APACHE create mode 100644 compiler/rustc_thread_pool/LICENSE-MIT create mode 100644 compiler/rustc_thread_pool/README.md create mode 100644 compiler/rustc_thread_pool/src/broadcast/mod.rs create mode 100644 compiler/rustc_thread_pool/src/broadcast/test.rs create mode 100644 compiler/rustc_thread_pool/src/compile_fail/mod.rs create mode 100644 compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs create mode 100644 compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs create mode 100644 compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs create mode 100644 compiler/rustc_thread_pool/src/compile_fail/rc_return.rs create mode 100644 compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs create mode 100644 compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs create mode 100644 compiler/rustc_thread_pool/src/job.rs create mode 100644 compiler/rustc_thread_pool/src/join/mod.rs create mode 100644 compiler/rustc_thread_pool/src/join/test.rs create mode 100644 compiler/rustc_thread_pool/src/latch.rs create mode 100644 compiler/rustc_thread_pool/src/lib.rs create mode 100644 compiler/rustc_thread_pool/src/private.rs create mode 100644 compiler/rustc_thread_pool/src/registry.rs create mode 100644 compiler/rustc_thread_pool/src/scope/mod.rs create mode 100644 compiler/rustc_thread_pool/src/scope/test.rs create mode 100644 compiler/rustc_thread_pool/src/sleep/README.md create mode 100644 compiler/rustc_thread_pool/src/sleep/counters.rs create mode 100644 compiler/rustc_thread_pool/src/sleep/mod.rs create mode 100644 compiler/rustc_thread_pool/src/spawn/mod.rs create mode 100644 compiler/rustc_thread_pool/src/spawn/test.rs create mode 100644 compiler/rustc_thread_pool/src/test.rs create mode 100644 compiler/rustc_thread_pool/src/thread_pool/mod.rs create mode 100644 compiler/rustc_thread_pool/src/thread_pool/test.rs create mode 100644 compiler/rustc_thread_pool/src/tlv.rs create mode 100644 compiler/rustc_thread_pool/src/unwind.rs create mode 100644 compiler/rustc_thread_pool/src/worker_local.rs create mode 100644 compiler/rustc_thread_pool/tests/double_init_fail.rs create mode 100644 compiler/rustc_thread_pool/tests/init_zero_threads.rs create mode 100644 compiler/rustc_thread_pool/tests/scope_join.rs create mode 100644 compiler/rustc_thread_pool/tests/scoped_threadpool.rs create mode 100644 compiler/rustc_thread_pool/tests/simple_panic.rs create mode 100644 compiler/rustc_thread_pool/tests/stack_overflow_crash.rs diff --git a/compiler/rustc_thread_pool/Cargo.toml b/compiler/rustc_thread_pool/Cargo.toml new file mode 100644 index 0000000000000..f7a3d1306b43f --- /dev/null +++ b/compiler/rustc_thread_pool/Cargo.toml @@ -0,0 +1,56 @@ +[package] +name = "rustc-rayon-core" +version = "0.5.1" +authors = ["Niko Matsakis ", + "Josh Stone "] +description = "Core APIs for Rayon - fork for rustc" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/rustc-rayon" +documentation = "https://docs.rs/rustc-rayon-core/" +rust-version = "1.63" +edition = "2021" +readme = "README.md" +keywords = ["parallel", "thread", "concurrency", "join", "performance"] +categories = ["concurrency"] + +[lib] +name = "rayon_core" + +# Some dependencies may not be their latest version, in order to support older rustc. +[dependencies] +crossbeam-deque = "0.8.1" +crossbeam-utils = "0.8.0" + +[dev-dependencies] +rand = "0.9" +rand_xorshift = "0.4" +scoped-tls = "1.0" + +[target.'cfg(unix)'.dev-dependencies] +libc = "0.2" + +[[test]] +name = "stack_overflow_crash" +path = "tests/stack_overflow_crash.rs" + +# NB: having one [[test]] manually defined means we need to declare them all + +[[test]] +name = "double_init_fail" +path = "tests/double_init_fail.rs" + +[[test]] +name = "init_zero_threads" +path = "tests/init_zero_threads.rs" + +[[test]] +name = "scope_join" +path = "tests/scope_join.rs" + +[[test]] +name = "simple_panic" +path = "tests/simple_panic.rs" + +[[test]] +name = "scoped_threadpool" +path = "tests/scoped_threadpool.rs" diff --git a/compiler/rustc_thread_pool/LICENSE-APACHE b/compiler/rustc_thread_pool/LICENSE-APACHE new file mode 100644 index 0000000000000..16fe87b06e802 --- /dev/null +++ b/compiler/rustc_thread_pool/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/compiler/rustc_thread_pool/LICENSE-MIT b/compiler/rustc_thread_pool/LICENSE-MIT new file mode 100644 index 0000000000000..25597d5838fa4 --- /dev/null +++ b/compiler/rustc_thread_pool/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2010 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/compiler/rustc_thread_pool/README.md b/compiler/rustc_thread_pool/README.md new file mode 100644 index 0000000000000..5b8714f5df48e --- /dev/null +++ b/compiler/rustc_thread_pool/README.md @@ -0,0 +1,13 @@ +Note: This is an unstable fork made for use in rustc + +Rayon-core represents the "core, stable" APIs of Rayon: join, scope, and so forth, as well as the ability to create custom thread-pools with ThreadPool. + +Maybe worth mentioning: users are not necessarily intended to directly access rayon-core; all its APIs are mirrored in the rayon crate. To that end, the examples in the docs use rayon::join and so forth rather than rayon_core::join. + +rayon-core aims to never, or almost never, have a breaking change to its API, because each revision of rayon-core also houses the global thread-pool (and hence if you have two simultaneous versions of rayon-core, you have two thread-pools). + +Please see [Rayon Docs] for details about using Rayon. + +[Rayon Docs]: https://docs.rs/rayon/ + +Rayon-core currently requires `rustc 1.63.0` or greater. diff --git a/compiler/rustc_thread_pool/src/broadcast/mod.rs b/compiler/rustc_thread_pool/src/broadcast/mod.rs new file mode 100644 index 0000000000000..442891f2d2876 --- /dev/null +++ b/compiler/rustc_thread_pool/src/broadcast/mod.rs @@ -0,0 +1,151 @@ +use crate::job::{ArcJob, StackJob}; +use crate::latch::{CountLatch, LatchRef}; +use crate::registry::{Registry, WorkerThread}; +use std::fmt; +use std::marker::PhantomData; +use std::sync::Arc; + +mod test; + +/// Executes `op` within every thread in the current threadpool. If this is +/// called from a non-Rayon thread, it will execute in the global threadpool. +/// Any attempts to use `join`, `scope`, or parallel iterators will then operate +/// within that threadpool. When the call has completed on each thread, returns +/// a vector containing all of their return values. +/// +/// For more information, see the [`ThreadPool::broadcast()`][m] method. +/// +/// [m]: struct.ThreadPool.html#method.broadcast +pub fn broadcast(op: OP) -> Vec +where + OP: Fn(BroadcastContext<'_>) -> R + Sync, + R: Send, +{ + // We assert that current registry has not terminated. + unsafe { broadcast_in(op, &Registry::current()) } +} + +/// Spawns an asynchronous task on every thread in this thread-pool. This task +/// will run in the implicit, global scope, which means that it may outlast the +/// current stack frame -- therefore, it cannot capture any references onto the +/// stack (you will likely need a `move` closure). +/// +/// For more information, see the [`ThreadPool::spawn_broadcast()`][m] method. +/// +/// [m]: struct.ThreadPool.html#method.spawn_broadcast +pub fn spawn_broadcast(op: OP) +where + OP: Fn(BroadcastContext<'_>) + Send + Sync + 'static, +{ + // We assert that current registry has not terminated. + unsafe { spawn_broadcast_in(op, &Registry::current()) } +} + +/// Provides context to a closure called by `broadcast`. +pub struct BroadcastContext<'a> { + worker: &'a WorkerThread, + + /// Make sure to prevent auto-traits like `Send` and `Sync`. + _marker: PhantomData<&'a mut dyn Fn()>, +} + +impl<'a> BroadcastContext<'a> { + pub(super) fn with(f: impl FnOnce(BroadcastContext<'_>) -> R) -> R { + let worker_thread = WorkerThread::current(); + assert!(!worker_thread.is_null()); + f(BroadcastContext { + worker: unsafe { &*worker_thread }, + _marker: PhantomData, + }) + } + + /// Our index amongst the broadcast threads (ranges from `0..self.num_threads()`). + #[inline] + pub fn index(&self) -> usize { + self.worker.index() + } + + /// The number of threads receiving the broadcast in the thread pool. + /// + /// # Future compatibility note + /// + /// Future versions of Rayon might vary the number of threads over time, but + /// this method will always return the number of threads which are actually + /// receiving your particular `broadcast` call. + #[inline] + pub fn num_threads(&self) -> usize { + self.worker.registry().num_threads() + } +} + +impl<'a> fmt::Debug for BroadcastContext<'a> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("BroadcastContext") + .field("index", &self.index()) + .field("num_threads", &self.num_threads()) + .field("pool_id", &self.worker.registry().id()) + .finish() + } +} + +/// Execute `op` on every thread in the pool. It will be executed on each +/// thread when they have nothing else to do locally, before they try to +/// steal work from other threads. This function will not return until all +/// threads have completed the `op`. +/// +/// Unsafe because `registry` must not yet have terminated. +pub(super) unsafe fn broadcast_in(op: OP, registry: &Arc) -> Vec +where + OP: Fn(BroadcastContext<'_>) -> R + Sync, + R: Send, +{ + let f = move |injected: bool| { + debug_assert!(injected); + BroadcastContext::with(&op) + }; + + let n_threads = registry.num_threads(); + let current_thread = WorkerThread::current().as_ref(); + let tlv = crate::tlv::get(); + let latch = CountLatch::with_count(n_threads, current_thread); + let jobs: Vec<_> = (0..n_threads) + .map(|_| StackJob::new(tlv, &f, LatchRef::new(&latch))) + .collect(); + let job_refs = jobs.iter().map(|job| job.as_job_ref()); + + registry.inject_broadcast(job_refs); + + // Wait for all jobs to complete, then collect the results, maybe propagating a panic. + latch.wait(current_thread); + jobs.into_iter().map(|job| job.into_result()).collect() +} + +/// Execute `op` on every thread in the pool. It will be executed on each +/// thread when they have nothing else to do locally, before they try to +/// steal work from other threads. This function returns immediately after +/// injecting the jobs. +/// +/// Unsafe because `registry` must not yet have terminated. +pub(super) unsafe fn spawn_broadcast_in(op: OP, registry: &Arc) +where + OP: Fn(BroadcastContext<'_>) + Send + Sync + 'static, +{ + let job = ArcJob::new({ + let registry = Arc::clone(registry); + move || { + registry.catch_unwind(|| BroadcastContext::with(&op)); + registry.terminate(); // (*) permit registry to terminate now + } + }); + + let n_threads = registry.num_threads(); + let job_refs = (0..n_threads).map(|_| { + // Ensure that registry cannot terminate until this job has executed + // on each thread. This ref is decremented at the (*) above. + registry.increment_terminate_count(); + + ArcJob::as_static_job_ref(&job) + }); + + registry.inject_broadcast(job_refs); +} diff --git a/compiler/rustc_thread_pool/src/broadcast/test.rs b/compiler/rustc_thread_pool/src/broadcast/test.rs new file mode 100644 index 0000000000000..00ab4ad7fe41f --- /dev/null +++ b/compiler/rustc_thread_pool/src/broadcast/test.rs @@ -0,0 +1,263 @@ +#![cfg(test)] + +use crate::ThreadPoolBuilder; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::channel; +use std::sync::Arc; +use std::{thread, time}; + +#[test] +fn broadcast_global() { + let v = crate::broadcast(|ctx| ctx.index()); + assert!(v.into_iter().eq(0..crate::current_num_threads())); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_global() { + let (tx, rx) = channel(); + crate::spawn_broadcast(move |ctx| tx.send(ctx.index()).unwrap()); + + let mut v: Vec<_> = rx.into_iter().collect(); + v.sort_unstable(); + assert!(v.into_iter().eq(0..crate::current_num_threads())); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_pool() { + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let v = pool.broadcast(|ctx| ctx.index()); + assert!(v.into_iter().eq(0..7)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_pool() { + let (tx, rx) = channel(); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool.spawn_broadcast(move |ctx| tx.send(ctx.index()).unwrap()); + + let mut v: Vec<_> = rx.into_iter().collect(); + v.sort_unstable(); + assert!(v.into_iter().eq(0..7)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_self() { + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let v = pool.install(|| crate::broadcast(|ctx| ctx.index())); + assert!(v.into_iter().eq(0..7)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_self() { + let (tx, rx) = channel(); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool.spawn(|| crate::spawn_broadcast(move |ctx| tx.send(ctx.index()).unwrap())); + + let mut v: Vec<_> = rx.into_iter().collect(); + v.sort_unstable(); + assert!(v.into_iter().eq(0..7)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_mutual() { + let count = AtomicUsize::new(0); + let pool1 = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool1.install(|| { + pool2.broadcast(|_| { + pool1.broadcast(|_| { + count.fetch_add(1, Ordering::Relaxed); + }) + }) + }); + assert_eq!(count.into_inner(), 3 * 7); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_mutual() { + let (tx, rx) = channel(); + let pool1 = Arc::new(ThreadPoolBuilder::new().num_threads(3).build().unwrap()); + let pool2 = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool1.spawn({ + let pool1 = Arc::clone(&pool1); + move || { + pool2.spawn_broadcast(move |_| { + let tx = tx.clone(); + pool1.spawn_broadcast(move |_| tx.send(()).unwrap()) + }) + } + }); + assert_eq!(rx.into_iter().count(), 3 * 7); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_mutual_sleepy() { + let count = AtomicUsize::new(0); + let pool1 = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool1.install(|| { + thread::sleep(time::Duration::from_secs(1)); + pool2.broadcast(|_| { + thread::sleep(time::Duration::from_secs(1)); + pool1.broadcast(|_| { + thread::sleep(time::Duration::from_millis(100)); + count.fetch_add(1, Ordering::Relaxed); + }) + }) + }); + assert_eq!(count.into_inner(), 3 * 7); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_mutual_sleepy() { + let (tx, rx) = channel(); + let pool1 = Arc::new(ThreadPoolBuilder::new().num_threads(3).build().unwrap()); + let pool2 = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool1.spawn({ + let pool1 = Arc::clone(&pool1); + move || { + thread::sleep(time::Duration::from_secs(1)); + pool2.spawn_broadcast(move |_| { + let tx = tx.clone(); + thread::sleep(time::Duration::from_secs(1)); + pool1.spawn_broadcast(move |_| { + thread::sleep(time::Duration::from_millis(100)); + tx.send(()).unwrap(); + }) + }) + } + }); + assert_eq!(rx.into_iter().count(), 3 * 7); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn broadcast_panic_one() { + let count = AtomicUsize::new(0); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let result = crate::unwind::halt_unwinding(|| { + pool.broadcast(|ctx| { + count.fetch_add(1, Ordering::Relaxed); + if ctx.index() == 3 { + panic!("Hello, world!"); + } + }) + }); + assert_eq!(count.into_inner(), 7); + assert!(result.is_err(), "broadcast panic should propagate!"); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn spawn_broadcast_panic_one() { + let (tx, rx) = channel(); + let (panic_tx, panic_rx) = channel(); + let pool = ThreadPoolBuilder::new() + .num_threads(7) + .panic_handler(move |e| panic_tx.send(e).unwrap()) + .build() + .unwrap(); + pool.spawn_broadcast(move |ctx| { + tx.send(()).unwrap(); + if ctx.index() == 3 { + panic!("Hello, world!"); + } + }); + drop(pool); // including panic_tx + assert_eq!(rx.into_iter().count(), 7); + assert_eq!(panic_rx.into_iter().count(), 1); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn broadcast_panic_many() { + let count = AtomicUsize::new(0); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let result = crate::unwind::halt_unwinding(|| { + pool.broadcast(|ctx| { + count.fetch_add(1, Ordering::Relaxed); + if ctx.index() % 2 == 0 { + panic!("Hello, world!"); + } + }) + }); + assert_eq!(count.into_inner(), 7); + assert!(result.is_err(), "broadcast panic should propagate!"); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn spawn_broadcast_panic_many() { + let (tx, rx) = channel(); + let (panic_tx, panic_rx) = channel(); + let pool = ThreadPoolBuilder::new() + .num_threads(7) + .panic_handler(move |e| panic_tx.send(e).unwrap()) + .build() + .unwrap(); + pool.spawn_broadcast(move |ctx| { + tx.send(()).unwrap(); + if ctx.index() % 2 == 0 { + panic!("Hello, world!"); + } + }); + drop(pool); // including panic_tx + assert_eq!(rx.into_iter().count(), 7); + assert_eq!(panic_rx.into_iter().count(), 4); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_sleep_race() { + let test_duration = time::Duration::from_secs(1); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let start = time::Instant::now(); + while start.elapsed() < test_duration { + pool.broadcast(|ctx| { + // A slight spread of sleep duration increases the chance that one + // of the threads will race in the pool's idle sleep afterward. + thread::sleep(time::Duration::from_micros(ctx.index() as u64)); + }); + } +} + +#[test] +fn broadcast_after_spawn_broadcast() { + let (tx, rx) = channel(); + + // Queue a non-blocking spawn_broadcast. + crate::spawn_broadcast(move |ctx| tx.send(ctx.index()).unwrap()); + + // This blocking broadcast runs after all prior broadcasts. + crate::broadcast(|_| {}); + + // The spawn_broadcast **must** have run by now on all threads. + let mut v: Vec<_> = rx.try_iter().collect(); + v.sort_unstable(); + assert!(v.into_iter().eq(0..crate::current_num_threads())); +} + +#[test] +fn broadcast_after_spawn() { + let (tx, rx) = channel(); + + // Queue a regular spawn on a thread-local deque. + crate::registry::in_worker(move |_, _| { + crate::spawn(move || tx.send(22).unwrap()); + }); + + // Broadcast runs after the local deque is empty. + crate::broadcast(|_| {}); + + // The spawn **must** have run by now. + assert_eq!(22, rx.try_recv().unwrap()); +} diff --git a/compiler/rustc_thread_pool/src/compile_fail/mod.rs b/compiler/rustc_thread_pool/src/compile_fail/mod.rs new file mode 100644 index 0000000000000..f2ec646a4d3d2 --- /dev/null +++ b/compiler/rustc_thread_pool/src/compile_fail/mod.rs @@ -0,0 +1,7 @@ +// These modules contain `compile_fail` doc tests. +mod quicksort_race1; +mod quicksort_race2; +mod quicksort_race3; +mod rc_return; +mod rc_upvar; +mod scope_join_bad; diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs new file mode 100644 index 0000000000000..5615033895a00 --- /dev/null +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs @@ -0,0 +1,28 @@ +/*! ```compile_fail,E0524 + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (lo, _hi) = v.split_at_mut(mid); + rayon_core::join(|| quick_sort(lo), || quick_sort(lo)); //~ ERROR +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn main() { } + +``` */ diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs new file mode 100644 index 0000000000000..020589c29a849 --- /dev/null +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs @@ -0,0 +1,28 @@ +/*! ```compile_fail,E0500 + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (lo, _hi) = v.split_at_mut(mid); + rayon_core::join(|| quick_sort(lo), || quick_sort(v)); //~ ERROR +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn main() { } + +``` */ diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs new file mode 100644 index 0000000000000..16fbf3b824d35 --- /dev/null +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs @@ -0,0 +1,28 @@ +/*! ```compile_fail,E0524 + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (_lo, hi) = v.split_at_mut(mid); + rayon_core::join(|| quick_sort(hi), || quick_sort(hi)); //~ ERROR +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn main() { } + +``` */ diff --git a/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs b/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs new file mode 100644 index 0000000000000..93e3a603849a1 --- /dev/null +++ b/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs @@ -0,0 +1,17 @@ +/** ```compile_fail,E0277 + +use std::rc::Rc; + +rayon_core::join(|| Rc::new(22), || ()); //~ ERROR + +``` */ +mod left {} + +/** ```compile_fail,E0277 + +use std::rc::Rc; + +rayon_core::join(|| (), || Rc::new(23)); //~ ERROR + +``` */ +mod right {} diff --git a/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs b/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs new file mode 100644 index 0000000000000..d8aebcfcbf24b --- /dev/null +++ b/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs @@ -0,0 +1,9 @@ +/*! ```compile_fail,E0277 + +use std::rc::Rc; + +let r = Rc::new(22); +rayon_core::join(|| r.clone(), || r.clone()); +//~^ ERROR + +``` */ diff --git a/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs b/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs new file mode 100644 index 0000000000000..75e4c5ca6c0d5 --- /dev/null +++ b/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs @@ -0,0 +1,24 @@ +/*! ```compile_fail,E0373 + +fn bad_scope(f: F) + where F: FnOnce(&i32) + Send, +{ + rayon_core::scope(|s| { + let x = 22; + s.spawn(|_| f(&x)); //~ ERROR `x` does not live long enough + }); +} + +fn good_scope(f: F) + where F: FnOnce(&i32) + Send, +{ + let x = 22; + rayon_core::scope(|s| { + s.spawn(|_| f(&x)); + }); +} + +fn main() { +} + +``` */ diff --git a/compiler/rustc_thread_pool/src/job.rs b/compiler/rustc_thread_pool/src/job.rs new file mode 100644 index 0000000000000..394c7576b2cb2 --- /dev/null +++ b/compiler/rustc_thread_pool/src/job.rs @@ -0,0 +1,277 @@ +use crate::latch::Latch; +use crate::tlv; +use crate::tlv::Tlv; +use crate::unwind; +use crossbeam_deque::{Injector, Steal}; +use std::any::Any; +use std::cell::UnsafeCell; +use std::mem; +use std::sync::Arc; + +pub(super) enum JobResult { + None, + Ok(T), + Panic(Box), +} + +/// A `Job` is used to advertise work for other threads that they may +/// want to steal. In accordance with time honored tradition, jobs are +/// arranged in a deque, so that thieves can take from the top of the +/// deque while the main worker manages the bottom of the deque. This +/// deque is managed by the `thread_pool` module. +pub(super) trait Job { + /// Unsafe: this may be called from a different thread than the one + /// which scheduled the job, so the implementer must ensure the + /// appropriate traits are met, whether `Send`, `Sync`, or both. + unsafe fn execute(this: *const ()); +} + +/// Effectively a Job trait object. Each JobRef **must** be executed +/// exactly once, or else data may leak. +/// +/// Internally, we store the job's data in a `*const ()` pointer. The +/// true type is something like `*const StackJob<...>`, but we hide +/// it. We also carry the "execute fn" from the `Job` trait. +pub(super) struct JobRef { + pointer: *const (), + execute_fn: unsafe fn(*const ()), +} + +unsafe impl Send for JobRef {} +unsafe impl Sync for JobRef {} + +impl JobRef { + /// Unsafe: caller asserts that `data` will remain valid until the + /// job is executed. + pub(super) unsafe fn new(data: *const T) -> JobRef + where + T: Job, + { + // erase types: + JobRef { + pointer: data as *const (), + execute_fn: ::execute, + } + } + + /// Returns an opaque handle that can be saved and compared, + /// without making `JobRef` itself `Copy + Eq`. + #[inline] + pub(super) fn id(&self) -> impl Eq { + (self.pointer, self.execute_fn) + } + + #[inline] + pub(super) unsafe fn execute(self) { + (self.execute_fn)(self.pointer) + } +} + +/// A job that will be owned by a stack slot. This means that when it +/// executes it need not free any heap data, the cleanup occurs when +/// the stack frame is later popped. The function parameter indicates +/// `true` if the job was stolen -- executed on a different thread. +pub(super) struct StackJob +where + L: Latch + Sync, + F: FnOnce(bool) -> R + Send, + R: Send, +{ + pub(super) latch: L, + func: UnsafeCell>, + result: UnsafeCell>, + tlv: Tlv, +} + +impl StackJob +where + L: Latch + Sync, + F: FnOnce(bool) -> R + Send, + R: Send, +{ + pub(super) fn new(tlv: Tlv, func: F, latch: L) -> StackJob { + StackJob { + latch, + func: UnsafeCell::new(Some(func)), + result: UnsafeCell::new(JobResult::None), + tlv, + } + } + + pub(super) unsafe fn as_job_ref(&self) -> JobRef { + JobRef::new(self) + } + + pub(super) unsafe fn run_inline(self, stolen: bool) -> R { + self.func.into_inner().unwrap()(stolen) + } + + pub(super) unsafe fn into_result(self) -> R { + self.result.into_inner().into_return_value() + } +} + +impl Job for StackJob +where + L: Latch + Sync, + F: FnOnce(bool) -> R + Send, + R: Send, +{ + unsafe fn execute(this: *const ()) { + let this = &*(this as *const Self); + tlv::set(this.tlv); + let abort = unwind::AbortIfPanic; + let func = (*this.func.get()).take().unwrap(); + (*this.result.get()) = JobResult::call(func); + Latch::set(&this.latch); + mem::forget(abort); + } +} + +/// Represents a job stored in the heap. Used to implement +/// `scope`. Unlike `StackJob`, when executed, `HeapJob` simply +/// invokes a closure, which then triggers the appropriate logic to +/// signal that the job executed. +/// +/// (Probably `StackJob` should be refactored in a similar fashion.) +pub(super) struct HeapJob +where + BODY: FnOnce() + Send, +{ + job: BODY, + tlv: Tlv, +} + +impl HeapJob +where + BODY: FnOnce() + Send, +{ + pub(super) fn new(tlv: Tlv, job: BODY) -> Box { + Box::new(HeapJob { job, tlv }) + } + + /// Creates a `JobRef` from this job -- note that this hides all + /// lifetimes, so it is up to you to ensure that this JobRef + /// doesn't outlive any data that it closes over. + pub(super) unsafe fn into_job_ref(self: Box) -> JobRef { + JobRef::new(Box::into_raw(self)) + } + + /// Creates a static `JobRef` from this job. + pub(super) fn into_static_job_ref(self: Box) -> JobRef + where + BODY: 'static, + { + unsafe { self.into_job_ref() } + } +} + +impl Job for HeapJob +where + BODY: FnOnce() + Send, +{ + unsafe fn execute(this: *const ()) { + let this = Box::from_raw(this as *mut Self); + tlv::set(this.tlv); + (this.job)(); + } +} + +/// Represents a job stored in an `Arc` -- like `HeapJob`, but may +/// be turned into multiple `JobRef`s and called multiple times. +pub(super) struct ArcJob +where + BODY: Fn() + Send + Sync, +{ + job: BODY, +} + +impl ArcJob +where + BODY: Fn() + Send + Sync, +{ + pub(super) fn new(job: BODY) -> Arc { + Arc::new(ArcJob { job }) + } + + /// Creates a `JobRef` from this job -- note that this hides all + /// lifetimes, so it is up to you to ensure that this JobRef + /// doesn't outlive any data that it closes over. + pub(super) unsafe fn as_job_ref(this: &Arc) -> JobRef { + JobRef::new(Arc::into_raw(Arc::clone(this))) + } + + /// Creates a static `JobRef` from this job. + pub(super) fn as_static_job_ref(this: &Arc) -> JobRef + where + BODY: 'static, + { + unsafe { Self::as_job_ref(this) } + } +} + +impl Job for ArcJob +where + BODY: Fn() + Send + Sync, +{ + unsafe fn execute(this: *const ()) { + let this = Arc::from_raw(this as *mut Self); + (this.job)(); + } +} + +impl JobResult { + fn call(func: impl FnOnce(bool) -> T) -> Self { + match unwind::halt_unwinding(|| func(true)) { + Ok(x) => JobResult::Ok(x), + Err(x) => JobResult::Panic(x), + } + } + + /// Convert the `JobResult` for a job that has finished (and hence + /// its JobResult is populated) into its return value. + /// + /// NB. This will panic if the job panicked. + pub(super) fn into_return_value(self) -> T { + match self { + JobResult::None => unreachable!(), + JobResult::Ok(x) => x, + JobResult::Panic(x) => unwind::resume_unwinding(x), + } + } +} + +/// Indirect queue to provide FIFO job priority. +pub(super) struct JobFifo { + inner: Injector, +} + +impl JobFifo { + pub(super) fn new() -> Self { + JobFifo { + inner: Injector::new(), + } + } + + pub(super) unsafe fn push(&self, job_ref: JobRef) -> JobRef { + // A little indirection ensures that spawns are always prioritized in FIFO order. The + // jobs in a thread's deque may be popped from the back (LIFO) or stolen from the front + // (FIFO), but either way they will end up popping from the front of this queue. + self.inner.push(job_ref); + JobRef::new(self) + } +} + +impl Job for JobFifo { + unsafe fn execute(this: *const ()) { + // We "execute" a queue by executing its first job, FIFO. + let this = &*(this as *const Self); + loop { + match this.inner.steal() { + Steal::Success(job_ref) => break job_ref.execute(), + Steal::Empty => panic!("FIFO is empty"), + Steal::Retry => {} + } + } + } +} diff --git a/compiler/rustc_thread_pool/src/join/mod.rs b/compiler/rustc_thread_pool/src/join/mod.rs new file mode 100644 index 0000000000000..032eec9c4c84d --- /dev/null +++ b/compiler/rustc_thread_pool/src/join/mod.rs @@ -0,0 +1,202 @@ +use crate::job::StackJob; +use crate::latch::SpinLatch; +use crate::registry::{self, WorkerThread}; +use crate::tlv::{self, Tlv}; +use crate::unwind; +use std::any::Any; + +use crate::FnContext; + +#[cfg(test)] +mod test; + +/// Takes two closures and *potentially* runs them in parallel. It +/// returns a pair of the results from those closures. +/// +/// Conceptually, calling `join()` is similar to spawning two threads, +/// one executing each of the two closures. However, the +/// implementation is quite different and incurs very low +/// overhead. The underlying technique is called "work stealing": the +/// Rayon runtime uses a fixed pool of worker threads and attempts to +/// only execute code in parallel when there are idle CPUs to handle +/// it. +/// +/// When `join` is called from outside the thread pool, the calling +/// thread will block while the closures execute in the pool. When +/// `join` is called within the pool, the calling thread still actively +/// participates in the thread pool. It will begin by executing closure +/// A (on the current thread). While it is doing that, it will advertise +/// closure B as being available for other threads to execute. Once closure A +/// has completed, the current thread will try to execute closure B; +/// if however closure B has been stolen, then it will look for other work +/// while waiting for the thief to fully execute closure B. (This is the +/// typical work-stealing strategy). +/// +/// # Examples +/// +/// This example uses join to perform a quick-sort (note this is not a +/// particularly optimized implementation: if you **actually** want to +/// sort for real, you should prefer [the `par_sort` method] offered +/// by Rayon). +/// +/// [the `par_sort` method]: ../rayon/slice/trait.ParallelSliceMut.html#method.par_sort +/// +/// ```rust +/// # use rayon_core as rayon; +/// let mut v = vec![5, 1, 8, 22, 0, 44]; +/// quick_sort(&mut v); +/// assert_eq!(v, vec![0, 1, 5, 8, 22, 44]); +/// +/// fn quick_sort(v: &mut [T]) { +/// if v.len() > 1 { +/// let mid = partition(v); +/// let (lo, hi) = v.split_at_mut(mid); +/// rayon::join(|| quick_sort(lo), +/// || quick_sort(hi)); +/// } +/// } +/// +/// // Partition rearranges all items `<=` to the pivot +/// // item (arbitrary selected to be the last item in the slice) +/// // to the first half of the slice. It then returns the +/// // "dividing point" where the pivot is placed. +/// fn partition(v: &mut [T]) -> usize { +/// let pivot = v.len() - 1; +/// let mut i = 0; +/// for j in 0..pivot { +/// if v[j] <= v[pivot] { +/// v.swap(i, j); +/// i += 1; +/// } +/// } +/// v.swap(i, pivot); +/// i +/// } +/// ``` +/// +/// # Warning about blocking I/O +/// +/// The assumption is that the closures given to `join()` are +/// CPU-bound tasks that do not perform I/O or other blocking +/// operations. If you do perform I/O, and that I/O should block +/// (e.g., waiting for a network request), the overall performance may +/// be poor. Moreover, if you cause one closure to be blocked waiting +/// on another (for example, using a channel), that could lead to a +/// deadlock. +/// +/// # Panics +/// +/// No matter what happens, both closures will always be executed. If +/// a single closure panics, whether it be the first or second +/// closure, that panic will be propagated and hence `join()` will +/// panic with the same panic value. If both closures panic, `join()` +/// will panic with the panic value from the first closure. +pub fn join(oper_a: A, oper_b: B) -> (RA, RB) +where + A: FnOnce() -> RA + Send, + B: FnOnce() -> RB + Send, + RA: Send, + RB: Send, +{ + #[inline] + fn call(f: impl FnOnce() -> R) -> impl FnOnce(FnContext) -> R { + move |_| f() + } + + join_context(call(oper_a), call(oper_b)) +} + +/// Identical to `join`, except that the closures have a parameter +/// that provides context for the way the closure has been called, +/// especially indicating whether they're executing on a different +/// thread than where `join_context` was called. This will occur if +/// the second job is stolen by a different thread, or if +/// `join_context` was called from outside the thread pool to begin +/// with. +pub fn join_context(oper_a: A, oper_b: B) -> (RA, RB) +where + A: FnOnce(FnContext) -> RA + Send, + B: FnOnce(FnContext) -> RB + Send, + RA: Send, + RB: Send, +{ + #[inline] + fn call_a(f: impl FnOnce(FnContext) -> R, injected: bool) -> impl FnOnce() -> R { + move || f(FnContext::new(injected)) + } + + #[inline] + fn call_b(f: impl FnOnce(FnContext) -> R) -> impl FnOnce(bool) -> R { + move |migrated| f(FnContext::new(migrated)) + } + + registry::in_worker(|worker_thread, injected| unsafe { + let tlv = tlv::get(); + // Create virtual wrapper for task b; this all has to be + // done here so that the stack frame can keep it all live + // long enough. + let job_b = StackJob::new(tlv, call_b(oper_b), SpinLatch::new(worker_thread)); + let job_b_ref = job_b.as_job_ref(); + let job_b_id = job_b_ref.id(); + worker_thread.push(job_b_ref); + + // Execute task a; hopefully b gets stolen in the meantime. + let status_a = unwind::halt_unwinding(call_a(oper_a, injected)); + let result_a = match status_a { + Ok(v) => v, + Err(err) => join_recover_from_panic(worker_thread, &job_b.latch, err, tlv), + }; + + // Now that task A has finished, try to pop job B from the + // local stack. It may already have been popped by job A; it + // may also have been stolen. There may also be some tasks + // pushed on top of it in the stack, and we will have to pop + // those off to get to it. + while !job_b.latch.probe() { + if let Some(job) = worker_thread.take_local_job() { + if job_b_id == job.id() { + // Found it! Let's run it. + // + // Note that this could panic, but it's ok if we unwind here. + + // Restore the TLV since we might have run some jobs overwriting it when waiting for job b. + tlv::set(tlv); + + let result_b = job_b.run_inline(injected); + return (result_a, result_b); + } else { + worker_thread.execute(job); + } + } else { + // Local deque is empty. Time to steal from other + // threads. + worker_thread.wait_until(&job_b.latch); + debug_assert!(job_b.latch.probe()); + break; + } + } + + // Restore the TLV since we might have run some jobs overwriting it when waiting for job b. + tlv::set(tlv); + + (result_a, job_b.into_result()) + }) +} + +/// If job A panics, we still cannot return until we are sure that job +/// B is complete. This is because it may contain references into the +/// enclosing stack frame(s). +#[cold] // cold path +unsafe fn join_recover_from_panic( + worker_thread: &WorkerThread, + job_b_latch: &SpinLatch<'_>, + err: Box, + tlv: Tlv, +) -> ! { + worker_thread.wait_until(job_b_latch); + + // Restore the TLV since we might have run some jobs overwriting it when waiting for job b. + tlv::set(tlv); + + unwind::resume_unwinding(err) +} diff --git a/compiler/rustc_thread_pool/src/join/test.rs b/compiler/rustc_thread_pool/src/join/test.rs new file mode 100644 index 0000000000000..03f4ab4478d40 --- /dev/null +++ b/compiler/rustc_thread_pool/src/join/test.rs @@ -0,0 +1,150 @@ +//! Tests for the join code. + +use super::*; +use crate::ThreadPoolBuilder; +use rand::distr::StandardUniform; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (lo, hi) = v.split_at_mut(mid); + join(|| quick_sort(lo), || quick_sort(hi)); +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn seeded_rng() -> XorShiftRng { + let mut seed = ::Seed::default(); + (0..).zip(seed.as_mut()).for_each(|(i, x)| *x = i); + XorShiftRng::from_seed(seed) +} + +#[test] +fn sort() { + let rng = seeded_rng(); + let mut data: Vec = rng.sample_iter(&StandardUniform).take(6 * 1024).collect(); + let mut sorted_data = data.clone(); + sorted_data.sort(); + quick_sort(&mut data); + assert_eq!(data, sorted_data); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn sort_in_pool() { + let rng = seeded_rng(); + let mut data: Vec = rng.sample_iter(&StandardUniform).take(12 * 1024).collect(); + + let pool = ThreadPoolBuilder::new().build().unwrap(); + let mut sorted_data = data.clone(); + sorted_data.sort(); + pool.install(|| quick_sort(&mut data)); + assert_eq!(data, sorted_data); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_a() { + join(|| panic!("Hello, world!"), || ()); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_b() { + join(|| (), || panic!("Hello, world!")); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_both() { + join(|| panic!("Hello, world!"), || panic!("Goodbye, world!")); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_b_still_executes() { + let mut x = false; + match unwind::halt_unwinding(|| join(|| panic!("Hello, world!"), || x = true)) { + Ok(_) => panic!("failed to propagate panic from closure A,"), + Err(_) => assert!(x, "closure b failed to execute"), + } +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn join_context_both() { + // If we're not in a pool, both should be marked stolen as they're injected. + let (a_migrated, b_migrated) = join_context(|a| a.migrated(), |b| b.migrated()); + assert!(a_migrated); + assert!(b_migrated); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn join_context_neither() { + // If we're already in a 1-thread pool, neither job should be stolen. + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let (a_migrated, b_migrated) = + pool.install(|| join_context(|a| a.migrated(), |b| b.migrated())); + assert!(!a_migrated); + assert!(!b_migrated); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn join_context_second() { + use std::sync::Barrier; + + // If we're already in a 2-thread pool, the second job should be stolen. + let barrier = Barrier::new(2); + let pool = ThreadPoolBuilder::new().num_threads(2).build().unwrap(); + let (a_migrated, b_migrated) = pool.install(|| { + join_context( + |a| { + barrier.wait(); + a.migrated() + }, + |b| { + barrier.wait(); + b.migrated() + }, + ) + }); + assert!(!a_migrated); + assert!(b_migrated); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn join_counter_overflow() { + const MAX: u32 = 500_000; + + let mut i = 0; + let mut j = 0; + let pool = ThreadPoolBuilder::new().num_threads(2).build().unwrap(); + + // Hammer on join a bunch of times -- used to hit overflow debug-assertions + // in JEC on 32-bit targets: https://github.com/rayon-rs/rayon/issues/797 + for _ in 0..MAX { + pool.join(|| i += 1, || j += 1); + } + + assert_eq!(i, MAX); + assert_eq!(j, MAX); +} diff --git a/compiler/rustc_thread_pool/src/latch.rs b/compiler/rustc_thread_pool/src/latch.rs new file mode 100644 index 0000000000000..8903942a8ce36 --- /dev/null +++ b/compiler/rustc_thread_pool/src/latch.rs @@ -0,0 +1,459 @@ +use std::marker::PhantomData; +use std::ops::Deref; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +use crate::registry::{Registry, WorkerThread}; + +/// We define various kinds of latches, which are all a primitive signaling +/// mechanism. A latch starts as false. Eventually someone calls `set()` and +/// it becomes true. You can test if it has been set by calling `probe()`. +/// +/// Some kinds of latches, but not all, support a `wait()` operation +/// that will wait until the latch is set, blocking efficiently. That +/// is not part of the trait since it is not possibly to do with all +/// latches. +/// +/// The intention is that `set()` is called once, but `probe()` may be +/// called any number of times. Once `probe()` returns true, the memory +/// effects that occurred before `set()` become visible. +/// +/// It'd probably be better to refactor the API into two paired types, +/// but that's a bit of work, and this is not a public API. +/// +/// ## Memory ordering +/// +/// Latches need to guarantee two things: +/// +/// - Once `probe()` returns true, all memory effects from the `set()` +/// are visible (in other words, the set should synchronize-with +/// the probe). +/// - Once `set()` occurs, the next `probe()` *will* observe it. This +/// typically requires a seq-cst ordering. See [the "tickle-then-get-sleepy" scenario in the sleep +/// README](/src/sleep/README.md#tickle-then-get-sleepy) for details. +pub(super) trait Latch { + /// Set the latch, signalling others. + /// + /// # WARNING + /// + /// Setting a latch triggers other threads to wake up and (in some + /// cases) complete. This may, in turn, cause memory to be + /// deallocated and so forth. One must be very careful about this, + /// and it's typically better to read all the fields you will need + /// to access *before* a latch is set! + /// + /// This function operates on `*const Self` instead of `&self` to allow it + /// to become dangling during this call. The caller must ensure that the + /// pointer is valid upon entry, and not invalidated during the call by any + /// actions other than `set` itself. + unsafe fn set(this: *const Self); +} + +pub(super) trait AsCoreLatch { + fn as_core_latch(&self) -> &CoreLatch; +} + +/// Latch is not set, owning thread is awake +const UNSET: usize = 0; + +/// Latch is not set, owning thread is going to sleep on this latch +/// (but has not yet fallen asleep). +const SLEEPY: usize = 1; + +/// Latch is not set, owning thread is asleep on this latch and +/// must be awoken. +const SLEEPING: usize = 2; + +/// Latch is set. +const SET: usize = 3; + +/// Spin latches are the simplest, most efficient kind, but they do +/// not support a `wait()` operation. They just have a boolean flag +/// that becomes true when `set()` is called. +#[derive(Debug)] +pub(super) struct CoreLatch { + state: AtomicUsize, +} + +impl CoreLatch { + #[inline] + fn new() -> Self { + Self { + state: AtomicUsize::new(0), + } + } + + /// Invoked by owning thread as it prepares to sleep. Returns true + /// if the owning thread may proceed to fall asleep, false if the + /// latch was set in the meantime. + #[inline] + pub(super) fn get_sleepy(&self) -> bool { + self.state + .compare_exchange(UNSET, SLEEPY, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + } + + /// Invoked by owning thread as it falls asleep sleep. Returns + /// true if the owning thread should block, or false if the latch + /// was set in the meantime. + #[inline] + pub(super) fn fall_asleep(&self) -> bool { + self.state + .compare_exchange(SLEEPY, SLEEPING, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + } + + /// Invoked by owning thread as it falls asleep sleep. Returns + /// true if the owning thread should block, or false if the latch + /// was set in the meantime. + #[inline] + pub(super) fn wake_up(&self) { + if !self.probe() { + let _ = + self.state + .compare_exchange(SLEEPING, UNSET, Ordering::SeqCst, Ordering::Relaxed); + } + } + + /// Set the latch. If this returns true, the owning thread was sleeping + /// and must be awoken. + /// + /// This is private because, typically, setting a latch involves + /// doing some wakeups; those are encapsulated in the surrounding + /// latch code. + #[inline] + unsafe fn set(this: *const Self) -> bool { + let old_state = (*this).state.swap(SET, Ordering::AcqRel); + old_state == SLEEPING + } + + /// Test if this latch has been set. + #[inline] + pub(super) fn probe(&self) -> bool { + self.state.load(Ordering::Acquire) == SET + } +} + +impl AsCoreLatch for CoreLatch { + #[inline] + fn as_core_latch(&self) -> &CoreLatch { + self + } +} + +/// Spin latches are the simplest, most efficient kind, but they do +/// not support a `wait()` operation. They just have a boolean flag +/// that becomes true when `set()` is called. +pub(super) struct SpinLatch<'r> { + core_latch: CoreLatch, + registry: &'r Arc, + target_worker_index: usize, + cross: bool, +} + +impl<'r> SpinLatch<'r> { + /// Creates a new spin latch that is owned by `thread`. This means + /// that `thread` is the only thread that should be blocking on + /// this latch -- it also means that when the latch is set, we + /// will wake `thread` if it is sleeping. + #[inline] + pub(super) fn new(thread: &'r WorkerThread) -> SpinLatch<'r> { + SpinLatch { + core_latch: CoreLatch::new(), + registry: thread.registry(), + target_worker_index: thread.index(), + cross: false, + } + } + + /// Creates a new spin latch for cross-threadpool blocking. Notably, we + /// need to make sure the registry is kept alive after setting, so we can + /// safely call the notification. + #[inline] + pub(super) fn cross(thread: &'r WorkerThread) -> SpinLatch<'r> { + SpinLatch { + cross: true, + ..SpinLatch::new(thread) + } + } + + #[inline] + pub(super) fn probe(&self) -> bool { + self.core_latch.probe() + } +} + +impl<'r> AsCoreLatch for SpinLatch<'r> { + #[inline] + fn as_core_latch(&self) -> &CoreLatch { + &self.core_latch + } +} + +impl<'r> Latch for SpinLatch<'r> { + #[inline] + unsafe fn set(this: *const Self) { + let cross_registry; + + let registry: &Registry = if (*this).cross { + // Ensure the registry stays alive while we notify it. + // Otherwise, it would be possible that we set the spin + // latch and the other thread sees it and exits, causing + // the registry to be deallocated, all before we get a + // chance to invoke `registry.notify_worker_latch_is_set`. + cross_registry = Arc::clone((*this).registry); + &cross_registry + } else { + // If this is not a "cross-registry" spin-latch, then the + // thread which is performing `set` is itself ensuring + // that the registry stays alive. However, that doesn't + // include this *particular* `Arc` handle if the waiting + // thread then exits, so we must completely dereference it. + (*this).registry + }; + let target_worker_index = (*this).target_worker_index; + + // NOTE: Once we `set`, the target may proceed and invalidate `this`! + if CoreLatch::set(&(*this).core_latch) { + // Subtle: at this point, we can no longer read from + // `self`, because the thread owning this spin latch may + // have awoken and deallocated the latch. Therefore, we + // only use fields whose values we already read. + registry.notify_worker_latch_is_set(target_worker_index); + } + } +} + +/// A Latch starts as false and eventually becomes true. You can block +/// until it becomes true. +#[derive(Debug)] +pub(super) struct LockLatch { + m: Mutex, + v: Condvar, +} + +impl LockLatch { + #[inline] + pub(super) fn new() -> LockLatch { + LockLatch { + m: Mutex::new(false), + v: Condvar::new(), + } + } + + /// Block until latch is set, then resets this lock latch so it can be reused again. + pub(super) fn wait_and_reset(&self) { + let mut guard = self.m.lock().unwrap(); + while !*guard { + guard = self.v.wait(guard).unwrap(); + } + *guard = false; + } + + /// Block until latch is set. + pub(super) fn wait(&self) { + let mut guard = self.m.lock().unwrap(); + while !*guard { + guard = self.v.wait(guard).unwrap(); + } + } +} + +impl Latch for LockLatch { + #[inline] + unsafe fn set(this: *const Self) { + let mut guard = (*this).m.lock().unwrap(); + *guard = true; + (*this).v.notify_all(); + } +} + +/// Once latches are used to implement one-time blocking, primarily +/// for the termination flag of the threads in the pool. +/// +/// Note: like a `SpinLatch`, once-latches are always associated with +/// some registry that is probing them, which must be tickled when +/// they are set. *Unlike* a `SpinLatch`, they don't themselves hold a +/// reference to that registry. This is because in some cases the +/// registry owns the once-latch, and that would create a cycle. So a +/// `OnceLatch` must be given a reference to its owning registry when +/// it is set. For this reason, it does not implement the `Latch` +/// trait (but it doesn't have to, as it is not used in those generic +/// contexts). +#[derive(Debug)] +pub(super) struct OnceLatch { + core_latch: CoreLatch, +} + +impl OnceLatch { + #[inline] + pub(super) fn new() -> OnceLatch { + Self { + core_latch: CoreLatch::new(), + } + } + + /// Set the latch, then tickle the specific worker thread, + /// which should be the one that owns this latch. + #[inline] + pub(super) unsafe fn set_and_tickle_one( + this: *const Self, + registry: &Registry, + target_worker_index: usize, + ) { + if CoreLatch::set(&(*this).core_latch) { + registry.notify_worker_latch_is_set(target_worker_index); + } + } +} + +impl AsCoreLatch for OnceLatch { + #[inline] + fn as_core_latch(&self) -> &CoreLatch { + &self.core_latch + } +} + +/// Counting latches are used to implement scopes. They track a +/// counter. Unlike other latches, calling `set()` does not +/// necessarily make the latch be considered `set()`; instead, it just +/// decrements the counter. The latch is only "set" (in the sense that +/// `probe()` returns true) once the counter reaches zero. +#[derive(Debug)] +pub(super) struct CountLatch { + counter: AtomicUsize, + kind: CountLatchKind, +} + +enum CountLatchKind { + /// A latch for scopes created on a rayon thread which will participate in work- + /// stealing while it waits for completion. This thread is not necessarily part + /// of the same registry as the scope itself! + Stealing { + latch: CoreLatch, + /// If a worker thread in registry A calls `in_place_scope` on a ThreadPool + /// with registry B, when a job completes in a thread of registry B, we may + /// need to call `notify_worker_latch_is_set()` to wake the thread in registry A. + /// That means we need a reference to registry A (since at that point we will + /// only have a reference to registry B), so we stash it here. + registry: Arc, + /// The index of the worker to wake in `registry` + worker_index: usize, + }, + + /// A latch for scopes created on a non-rayon thread which will block to wait. + Blocking { latch: LockLatch }, +} + +impl std::fmt::Debug for CountLatchKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CountLatchKind::Stealing { latch, .. } => { + f.debug_tuple("Stealing").field(latch).finish() + } + CountLatchKind::Blocking { latch, .. } => { + f.debug_tuple("Blocking").field(latch).finish() + } + } + } +} + +impl CountLatch { + pub(super) fn new(owner: Option<&WorkerThread>) -> Self { + Self::with_count(1, owner) + } + + pub(super) fn with_count(count: usize, owner: Option<&WorkerThread>) -> Self { + Self { + counter: AtomicUsize::new(count), + kind: match owner { + Some(owner) => CountLatchKind::Stealing { + latch: CoreLatch::new(), + registry: Arc::clone(owner.registry()), + worker_index: owner.index(), + }, + None => CountLatchKind::Blocking { + latch: LockLatch::new(), + }, + }, + } + } + + #[inline] + pub(super) fn increment(&self) { + let old_counter = self.counter.fetch_add(1, Ordering::Relaxed); + debug_assert!(old_counter != 0); + } + + pub(super) fn wait(&self, owner: Option<&WorkerThread>) { + match &self.kind { + CountLatchKind::Stealing { + latch, + registry, + worker_index, + } => unsafe { + let owner = owner.expect("owner thread"); + debug_assert_eq!(registry.id(), owner.registry().id()); + debug_assert_eq!(*worker_index, owner.index()); + owner.wait_until(latch); + }, + CountLatchKind::Blocking { latch } => latch.wait(), + } + } +} + +impl Latch for CountLatch { + #[inline] + unsafe fn set(this: *const Self) { + if (*this).counter.fetch_sub(1, Ordering::SeqCst) == 1 { + // NOTE: Once we call `set` on the internal `latch`, + // the target may proceed and invalidate `this`! + match (*this).kind { + CountLatchKind::Stealing { + ref latch, + ref registry, + worker_index, + } => { + let registry = Arc::clone(registry); + if CoreLatch::set(latch) { + registry.notify_worker_latch_is_set(worker_index); + } + } + CountLatchKind::Blocking { ref latch } => LockLatch::set(latch), + } + } + } +} + +/// `&L` without any implication of `dereferenceable` for `Latch::set` +pub(super) struct LatchRef<'a, L> { + inner: *const L, + marker: PhantomData<&'a L>, +} + +impl LatchRef<'_, L> { + pub(super) fn new(inner: &L) -> LatchRef<'_, L> { + LatchRef { + inner, + marker: PhantomData, + } + } +} + +unsafe impl Sync for LatchRef<'_, L> {} + +impl Deref for LatchRef<'_, L> { + type Target = L; + + fn deref(&self) -> &L { + // SAFETY: if we have &self, the inner latch is still alive + unsafe { &*self.inner } + } +} + +impl Latch for LatchRef<'_, L> { + #[inline] + unsafe fn set(this: *const Self) { + L::set((*this).inner); + } +} diff --git a/compiler/rustc_thread_pool/src/lib.rs b/compiler/rustc_thread_pool/src/lib.rs new file mode 100644 index 0000000000000..72064547e526e --- /dev/null +++ b/compiler/rustc_thread_pool/src/lib.rs @@ -0,0 +1,922 @@ +//! Rayon-core houses the core stable APIs of Rayon. +//! +//! These APIs have been mirrored in the Rayon crate and it is recommended to use these from there. +//! +//! [`join`] is used to take two closures and potentially run them in parallel. +//! - It will run in parallel if task B gets stolen before task A can finish. +//! - It will run sequentially if task A finishes before task B is stolen and can continue on task B. +//! +//! [`scope`] creates a scope in which you can run any number of parallel tasks. +//! These tasks can spawn nested tasks and scopes, but given the nature of work stealing, the order of execution can not be guaranteed. +//! The scope will exist until all tasks spawned within the scope have been completed. +//! +//! [`spawn`] add a task into the 'static' or 'global' scope, or a local scope created by the [`scope()`] function. +//! +//! [`ThreadPool`] can be used to create your own thread pools (using [`ThreadPoolBuilder`]) or to customize the global one. +//! Tasks spawned within the pool (using [`install()`], [`join()`], etc.) will be added to a deque, +//! where it becomes available for work stealing from other threads in the local threadpool. +//! +//! [`join`]: fn.join.html +//! [`scope`]: fn.scope.html +//! [`scope()`]: fn.scope.html +//! [`spawn`]: fn.spawn.html +//! [`ThreadPool`]: struct.threadpool.html +//! [`install()`]: struct.ThreadPool.html#method.install +//! [`spawn()`]: struct.ThreadPool.html#method.spawn +//! [`join()`]: struct.ThreadPool.html#method.join +//! [`ThreadPoolBuilder`]: struct.ThreadPoolBuilder.html +//! +//! # Global fallback when threading is unsupported +//! +//! Rayon uses `std` APIs for threading, but some targets have incomplete implementations that +//! always return `Unsupported` errors. The WebAssembly `wasm32-unknown-unknown` and `wasm32-wasi` +//! targets are notable examples of this. Rather than panicking on the unsupported error when +//! creating the implicit global threadpool, Rayon configures a fallback mode instead. +//! +//! This fallback mode mostly functions as if it were using a single-threaded "pool", like setting +//! `RAYON_NUM_THREADS=1`. For example, `join` will execute its two closures sequentially, since +//! there is no other thread to share the work. However, since the pool is not running independent +//! of the main thread, non-blocking calls like `spawn` may not execute at all, unless a lower- +//! priority call like `broadcast` gives them an opening. The fallback mode does not try to emulate +//! anything like thread preemption or `async` task switching, but `yield_now` or `yield_local` +//! can also volunteer execution time. +//! +//! Explicit `ThreadPoolBuilder` methods always report their error without any fallback. +//! +//! # Restricting multiple versions +//! +//! In order to ensure proper coordination between threadpools, and especially +//! to make sure there's only one global threadpool, `rayon-core` is actively +//! restricted from building multiple versions of itself into a single target. +//! You may see a build error like this in violation: +//! +//! ```text +//! error: native library `rayon-core` is being linked to by more +//! than one package, and can only be linked to by one package +//! ``` +//! +//! While we strive to keep `rayon-core` semver-compatible, it's still +//! possible to arrive at this situation if different crates have overly +//! restrictive tilde or inequality requirements for `rayon-core`. The +//! conflicting requirements will need to be resolved before the build will +//! succeed. + +#![warn(rust_2018_idioms)] + +use std::any::Any; +use std::env; +use std::error::Error; +use std::fmt; +use std::io; +use std::marker::PhantomData; +use std::str::FromStr; +use std::thread; + +#[macro_use] +mod private; + +mod broadcast; +mod job; +mod join; +mod latch; +mod registry; +mod scope; +mod sleep; +mod spawn; +mod thread_pool; +mod unwind; +mod worker_local; + +mod compile_fail; +mod test; + +pub mod tlv; + +pub use self::broadcast::{broadcast, spawn_broadcast, BroadcastContext}; +pub use self::join::{join, join_context}; +pub use self::registry::ThreadBuilder; +pub use self::registry::{mark_blocked, mark_unblocked, Registry}; +pub use self::scope::{in_place_scope, scope, Scope}; +pub use self::scope::{in_place_scope_fifo, scope_fifo, ScopeFifo}; +pub use self::spawn::{spawn, spawn_fifo}; +pub use self::thread_pool::current_thread_has_pending_tasks; +pub use self::thread_pool::current_thread_index; +pub use self::thread_pool::ThreadPool; +pub use self::thread_pool::{yield_local, yield_now, Yield}; +pub use worker_local::WorkerLocal; + +use self::registry::{CustomSpawn, DefaultSpawn, ThreadSpawn}; + +/// Returns the maximum number of threads that Rayon supports in a single thread-pool. +/// +/// If a higher thread count is requested by calling `ThreadPoolBuilder::num_threads` or by setting +/// the `RAYON_NUM_THREADS` environment variable, then it will be reduced to this maximum. +/// +/// The value may vary between different targets, and is subject to change in new Rayon versions. +pub fn max_num_threads() -> usize { + // We are limited by the bits available in the sleep counter's `AtomicUsize`. + crate::sleep::THREADS_MAX +} + +/// Returns the number of threads in the current registry. If this +/// code is executing within a Rayon thread-pool, then this will be +/// the number of threads for the thread-pool of the current +/// thread. Otherwise, it will be the number of threads for the global +/// thread-pool. +/// +/// This can be useful when trying to judge how many times to split +/// parallel work (the parallel iterator traits use this value +/// internally for this purpose). +/// +/// # Future compatibility note +/// +/// Note that unless this thread-pool was created with a +/// builder that specifies the number of threads, then this +/// number may vary over time in future versions (see [the +/// `num_threads()` method for details][snt]). +/// +/// [snt]: struct.ThreadPoolBuilder.html#method.num_threads +pub fn current_num_threads() -> usize { + crate::registry::Registry::current_num_threads() +} + +/// Error when initializing a thread pool. +#[derive(Debug)] +pub struct ThreadPoolBuildError { + kind: ErrorKind, +} + +#[derive(Debug)] +enum ErrorKind { + GlobalPoolAlreadyInitialized, + IOError(io::Error), +} + +/// Used to create a new [`ThreadPool`] or to configure the global rayon thread pool. +/// ## Creating a ThreadPool +/// The following creates a thread pool with 22 threads. +/// +/// ```rust +/// # use rayon_core as rayon; +/// let pool = rayon::ThreadPoolBuilder::new().num_threads(22).build().unwrap(); +/// ``` +/// +/// To instead configure the global thread pool, use [`build_global()`]: +/// +/// ```rust +/// # use rayon_core as rayon; +/// rayon::ThreadPoolBuilder::new().num_threads(22).build_global().unwrap(); +/// ``` +/// +/// [`ThreadPool`]: struct.ThreadPool.html +/// [`build_global()`]: struct.ThreadPoolBuilder.html#method.build_global +pub struct ThreadPoolBuilder { + /// The number of threads in the rayon thread pool. + /// If zero will use the RAYON_NUM_THREADS environment variable. + /// If RAYON_NUM_THREADS is invalid or zero will use the default. + num_threads: usize, + + /// Custom closure, if any, to handle a panic that we cannot propagate + /// anywhere else. + panic_handler: Option>, + + /// Closure to compute the name of a thread. + get_thread_name: Option String>>, + + /// The stack size for the created worker threads + stack_size: Option, + + /// Closure invoked on deadlock. + deadlock_handler: Option>, + + /// Closure invoked on worker thread start. + start_handler: Option>, + + /// Closure invoked on worker thread exit. + exit_handler: Option>, + + /// Closure invoked to spawn threads. + spawn_handler: S, + + /// Closure invoked when starting computations in a thread. + acquire_thread_handler: Option>, + + /// Closure invoked when blocking in a thread. + release_thread_handler: Option>, + + /// If false, worker threads will execute spawned jobs in a + /// "depth-first" fashion. If true, they will do a "breadth-first" + /// fashion. Depth-first is the default. + breadth_first: bool, +} + +/// Contains the rayon thread pool configuration. Use [`ThreadPoolBuilder`] instead. +/// +/// [`ThreadPoolBuilder`]: struct.ThreadPoolBuilder.html +#[deprecated(note = "Use `ThreadPoolBuilder`")] +#[derive(Default)] +pub struct Configuration { + builder: ThreadPoolBuilder, +} + +/// The type for a panic handling closure. Note that this same closure +/// may be invoked multiple times in parallel. +type PanicHandler = dyn Fn(Box) + Send + Sync; + +/// The type for a closure that gets invoked when the Rayon thread pool deadlocks +type DeadlockHandler = dyn Fn() + Send + Sync; + +/// The type for a closure that gets invoked when a thread starts. The +/// closure is passed the index of the thread on which it is invoked. +/// Note that this same closure may be invoked multiple times in parallel. +type StartHandler = dyn Fn(usize) + Send + Sync; + +/// The type for a closure that gets invoked when a thread exits. The +/// closure is passed the index of the thread on which it is invoked. +/// Note that this same closure may be invoked multiple times in parallel. +type ExitHandler = dyn Fn(usize) + Send + Sync; + +// NB: We can't `#[derive(Default)]` because `S` is left ambiguous. +impl Default for ThreadPoolBuilder { + fn default() -> Self { + ThreadPoolBuilder { + num_threads: 0, + panic_handler: None, + get_thread_name: None, + stack_size: None, + start_handler: None, + exit_handler: None, + deadlock_handler: None, + acquire_thread_handler: None, + release_thread_handler: None, + spawn_handler: DefaultSpawn, + breadth_first: false, + } + } +} + +/// The type for a closure that gets invoked before starting computations in a thread. +/// Note that this same closure may be invoked multiple times in parallel. +type AcquireThreadHandler = dyn Fn() + Send + Sync; + +/// The type for a closure that gets invoked before blocking in a thread. +/// Note that this same closure may be invoked multiple times in parallel. +type ReleaseThreadHandler = dyn Fn() + Send + Sync; + +impl ThreadPoolBuilder { + /// Creates and returns a valid rayon thread pool builder, but does not initialize it. + pub fn new() -> Self { + Self::default() + } +} + +/// Note: the `S: ThreadSpawn` constraint is an internal implementation detail for the +/// default spawn and those set by [`spawn_handler`](#method.spawn_handler). +impl ThreadPoolBuilder +where + S: ThreadSpawn, +{ + /// Creates a new `ThreadPool` initialized using this configuration. + pub fn build(self) -> Result { + ThreadPool::build(self) + } + + /// Initializes the global thread pool. This initialization is + /// **optional**. If you do not call this function, the thread pool + /// will be automatically initialized with the default + /// configuration. Calling `build_global` is not recommended, except + /// in two scenarios: + /// + /// - You wish to change the default configuration. + /// - You are running a benchmark, in which case initializing may + /// yield slightly more consistent results, since the worker threads + /// will already be ready to go even in the first iteration. But + /// this cost is minimal. + /// + /// Initialization of the global thread pool happens exactly + /// once. Once started, the configuration cannot be + /// changed. Therefore, if you call `build_global` a second time, it + /// will return an error. An `Ok` result indicates that this + /// is the first initialization of the thread pool. + pub fn build_global(self) -> Result<(), ThreadPoolBuildError> { + let registry = registry::init_global_registry(self)?; + registry.wait_until_primed(); + Ok(()) + } +} + +impl ThreadPoolBuilder { + /// Creates a scoped `ThreadPool` initialized using this configuration. + /// + /// This is a convenience function for building a pool using [`std::thread::scope`] + /// to spawn threads in a [`spawn_handler`](#method.spawn_handler). + /// The threads in this pool will start by calling `wrapper`, which should + /// do initialization and continue by calling `ThreadBuilder::run()`. + /// + /// [`std::thread::scope`]: https://doc.rust-lang.org/std/thread/fn.scope.html + /// + /// # Examples + /// + /// A scoped pool may be useful in combination with scoped thread-local variables. + /// + /// ``` + /// # use rayon_core as rayon; + /// + /// scoped_tls::scoped_thread_local!(static POOL_DATA: Vec); + /// + /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { + /// let pool_data = vec![1, 2, 3]; + /// + /// // We haven't assigned any TLS data yet. + /// assert!(!POOL_DATA.is_set()); + /// + /// rayon::ThreadPoolBuilder::new() + /// .build_scoped( + /// // Borrow `pool_data` in TLS for each thread. + /// |thread| POOL_DATA.set(&pool_data, || thread.run()), + /// // Do some work that needs the TLS data. + /// |pool| pool.install(|| assert!(POOL_DATA.is_set())), + /// )?; + /// + /// // Once we've returned, `pool_data` is no longer borrowed. + /// drop(pool_data); + /// Ok(()) + /// } + /// ``` + pub fn build_scoped(self, wrapper: W, with_pool: F) -> Result + where + W: Fn(ThreadBuilder) + Sync, // expected to call `run()` + F: FnOnce(&ThreadPool) -> R, + { + std::thread::scope(|scope| { + let pool = self + .spawn_handler(|thread| { + let mut builder = std::thread::Builder::new(); + if let Some(name) = thread.name() { + builder = builder.name(name.to_string()); + } + if let Some(size) = thread.stack_size() { + builder = builder.stack_size(size); + } + builder.spawn_scoped(scope, || wrapper(thread))?; + Ok(()) + }) + .build()?; + let result = unwind::halt_unwinding(|| with_pool(&pool)); + pool.wait_until_stopped(); + match result { + Ok(result) => Ok(result), + Err(err) => unwind::resume_unwinding(err), + } + }) + } +} + +impl ThreadPoolBuilder { + /// Sets a custom function for spawning threads. + /// + /// Note that the threads will not exit until after the pool is dropped. It + /// is up to the caller to wait for thread termination if that is important + /// for any invariants. For instance, threads created in [`std::thread::scope`] + /// will be joined before that scope returns, and this will block indefinitely + /// if the pool is leaked. Furthermore, the global thread pool doesn't terminate + /// until the entire process exits! + /// + /// # Examples + /// + /// A minimal spawn handler just needs to call `run()` from an independent thread. + /// + /// ``` + /// # use rayon_core as rayon; + /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { + /// let pool = rayon::ThreadPoolBuilder::new() + /// .spawn_handler(|thread| { + /// std::thread::spawn(|| thread.run()); + /// Ok(()) + /// }) + /// .build()?; + /// + /// pool.install(|| println!("Hello from my custom thread!")); + /// Ok(()) + /// } + /// ``` + /// + /// The default spawn handler sets the name and stack size if given, and propagates + /// any errors from the thread builder. + /// + /// ``` + /// # use rayon_core as rayon; + /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { + /// let pool = rayon::ThreadPoolBuilder::new() + /// .spawn_handler(|thread| { + /// let mut b = std::thread::Builder::new(); + /// if let Some(name) = thread.name() { + /// b = b.name(name.to_owned()); + /// } + /// if let Some(stack_size) = thread.stack_size() { + /// b = b.stack_size(stack_size); + /// } + /// b.spawn(|| thread.run())?; + /// Ok(()) + /// }) + /// .build()?; + /// + /// pool.install(|| println!("Hello from my fully custom thread!")); + /// Ok(()) + /// } + /// ``` + /// + /// This can also be used for a pool of scoped threads like [`crossbeam::scope`], + /// or [`std::thread::scope`] introduced in Rust 1.63, which is encapsulated in + /// [`build_scoped`](#method.build_scoped). + /// + /// [`crossbeam::scope`]: https://docs.rs/crossbeam/0.8/crossbeam/fn.scope.html + /// [`std::thread::scope`]: https://doc.rust-lang.org/std/thread/fn.scope.html + /// + /// ``` + /// # use rayon_core as rayon; + /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { + /// std::thread::scope(|scope| { + /// let pool = rayon::ThreadPoolBuilder::new() + /// .spawn_handler(|thread| { + /// let mut builder = std::thread::Builder::new(); + /// if let Some(name) = thread.name() { + /// builder = builder.name(name.to_string()); + /// } + /// if let Some(size) = thread.stack_size() { + /// builder = builder.stack_size(size); + /// } + /// builder.spawn_scoped(scope, || { + /// // Add any scoped initialization here, then run! + /// thread.run() + /// })?; + /// Ok(()) + /// }) + /// .build()?; + /// + /// pool.install(|| println!("Hello from my custom scoped thread!")); + /// Ok(()) + /// }) + /// } + /// ``` + pub fn spawn_handler(self, spawn: F) -> ThreadPoolBuilder> + where + F: FnMut(ThreadBuilder) -> io::Result<()>, + { + ThreadPoolBuilder { + spawn_handler: CustomSpawn::new(spawn), + // ..self + num_threads: self.num_threads, + panic_handler: self.panic_handler, + get_thread_name: self.get_thread_name, + stack_size: self.stack_size, + start_handler: self.start_handler, + exit_handler: self.exit_handler, + deadlock_handler: self.deadlock_handler, + acquire_thread_handler: self.acquire_thread_handler, + release_thread_handler: self.release_thread_handler, + breadth_first: self.breadth_first, + } + } + + /// Returns a reference to the current spawn handler. + fn get_spawn_handler(&mut self) -> &mut S { + &mut self.spawn_handler + } + + /// Get the number of threads that will be used for the thread + /// pool. See `num_threads()` for more information. + fn get_num_threads(&self) -> usize { + if self.num_threads > 0 { + self.num_threads + } else { + let default = || { + thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + }; + + match env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|s| usize::from_str(&s).ok()) + { + Some(x @ 1..) => return x, + Some(0) => return default(), + _ => {} + } + + // Support for deprecated `RAYON_RS_NUM_CPUS`. + match env::var("RAYON_RS_NUM_CPUS") + .ok() + .and_then(|s| usize::from_str(&s).ok()) + { + Some(x @ 1..) => x, + _ => default(), + } + } + } + + /// Get the thread name for the thread with the given index. + fn get_thread_name(&mut self, index: usize) -> Option { + let f = self.get_thread_name.as_mut()?; + Some(f(index)) + } + + /// Sets a closure which takes a thread index and returns + /// the thread's name. + pub fn thread_name(mut self, closure: F) -> Self + where + F: FnMut(usize) -> String + 'static, + { + self.get_thread_name = Some(Box::new(closure)); + self + } + + /// Sets the number of threads to be used in the rayon threadpool. + /// + /// If you specify a non-zero number of threads using this + /// function, then the resulting thread-pools are guaranteed to + /// start at most this number of threads. + /// + /// If `num_threads` is 0, or you do not call this function, then + /// the Rayon runtime will select the number of threads + /// automatically. At present, this is based on the + /// `RAYON_NUM_THREADS` environment variable (if set), + /// or the number of logical CPUs (otherwise). + /// In the future, however, the default behavior may + /// change to dynamically add or remove threads as needed. + /// + /// **Future compatibility warning:** Given the default behavior + /// may change in the future, if you wish to rely on a fixed + /// number of threads, you should use this function to specify + /// that number. To reproduce the current default behavior, you + /// may wish to use [`std::thread::available_parallelism`] + /// to query the number of CPUs dynamically. + /// + /// **Old environment variable:** `RAYON_NUM_THREADS` is a one-to-one + /// replacement of the now deprecated `RAYON_RS_NUM_CPUS` environment + /// variable. If both variables are specified, `RAYON_NUM_THREADS` will + /// be preferred. + pub fn num_threads(mut self, num_threads: usize) -> Self { + self.num_threads = num_threads; + self + } + + /// Returns a copy of the current panic handler. + fn take_panic_handler(&mut self) -> Option> { + self.panic_handler.take() + } + + /// Normally, whenever Rayon catches a panic, it tries to + /// propagate it to someplace sensible, to try and reflect the + /// semantics of sequential execution. But in some cases, + /// particularly with the `spawn()` APIs, there is no + /// obvious place where we should propagate the panic to. + /// In that case, this panic handler is invoked. + /// + /// If no panic handler is set, the default is to abort the + /// process, under the principle that panics should not go + /// unobserved. + /// + /// If the panic handler itself panics, this will abort the + /// process. To prevent this, wrap the body of your panic handler + /// in a call to `std::panic::catch_unwind()`. + pub fn panic_handler(mut self, panic_handler: H) -> Self + where + H: Fn(Box) + Send + Sync + 'static, + { + self.panic_handler = Some(Box::new(panic_handler)); + self + } + + /// Get the stack size of the worker threads + fn get_stack_size(&self) -> Option { + self.stack_size + } + + /// Sets the stack size of the worker threads + pub fn stack_size(mut self, stack_size: usize) -> Self { + self.stack_size = Some(stack_size); + self + } + + /// **(DEPRECATED)** Suggest to worker threads that they execute + /// spawned jobs in a "breadth-first" fashion. + /// + /// Typically, when a worker thread is idle or blocked, it will + /// attempt to execute the job from the *top* of its local deque of + /// work (i.e., the job most recently spawned). If this flag is set + /// to true, however, workers will prefer to execute in a + /// *breadth-first* fashion -- that is, they will search for jobs at + /// the *bottom* of their local deque. (At present, workers *always* + /// steal from the bottom of other workers' deques, regardless of + /// the setting of this flag.) + /// + /// If you think of the tasks as a tree, where a parent task + /// spawns its children in the tree, then this flag loosely + /// corresponds to doing a breadth-first traversal of the tree, + /// whereas the default would be to do a depth-first traversal. + /// + /// **Note that this is an "execution hint".** Rayon's task + /// execution is highly dynamic and the precise order in which + /// independent tasks are executed is not intended to be + /// guaranteed. + /// + /// This `breadth_first()` method is now deprecated per [RFC #1], + /// and in the future its effect may be removed. Consider using + /// [`scope_fifo()`] for a similar effect. + /// + /// [RFC #1]: https://github.com/rayon-rs/rfcs/blob/master/accepted/rfc0001-scope-scheduling.md + /// [`scope_fifo()`]: fn.scope_fifo.html + #[deprecated(note = "use `scope_fifo` and `spawn_fifo` for similar effect")] + pub fn breadth_first(mut self) -> Self { + self.breadth_first = true; + self + } + + fn get_breadth_first(&self) -> bool { + self.breadth_first + } + + /// Takes the current acquire thread callback, leaving `None`. + fn take_acquire_thread_handler(&mut self) -> Option> { + self.acquire_thread_handler.take() + } + + /// Set a callback to be invoked when starting computations in a thread. + pub fn acquire_thread_handler(mut self, acquire_thread_handler: H) -> Self + where + H: Fn() + Send + Sync + 'static, + { + self.acquire_thread_handler = Some(Box::new(acquire_thread_handler)); + self + } + + /// Takes the current release thread callback, leaving `None`. + fn take_release_thread_handler(&mut self) -> Option> { + self.release_thread_handler.take() + } + + /// Set a callback to be invoked when blocking in thread. + pub fn release_thread_handler(mut self, release_thread_handler: H) -> Self + where + H: Fn() + Send + Sync + 'static, + { + self.release_thread_handler = Some(Box::new(release_thread_handler)); + self + } + + /// Takes the current deadlock callback, leaving `None`. + fn take_deadlock_handler(&mut self) -> Option> { + self.deadlock_handler.take() + } + + /// Set a callback to be invoked on current deadlock. + pub fn deadlock_handler(mut self, deadlock_handler: H) -> Self + where + H: Fn() + Send + Sync + 'static, + { + self.deadlock_handler = Some(Box::new(deadlock_handler)); + self + } + + /// Takes the current thread start callback, leaving `None`. + fn take_start_handler(&mut self) -> Option> { + self.start_handler.take() + } + + /// Sets a callback to be invoked on thread start. + /// + /// The closure is passed the index of the thread on which it is invoked. + /// Note that this same closure may be invoked multiple times in parallel. + /// If this closure panics, the panic will be passed to the panic handler. + /// If that handler returns, then startup will continue normally. + pub fn start_handler(mut self, start_handler: H) -> Self + where + H: Fn(usize) + Send + Sync + 'static, + { + self.start_handler = Some(Box::new(start_handler)); + self + } + + /// Returns a current thread exit callback, leaving `None`. + fn take_exit_handler(&mut self) -> Option> { + self.exit_handler.take() + } + + /// Sets a callback to be invoked on thread exit. + /// + /// The closure is passed the index of the thread on which it is invoked. + /// Note that this same closure may be invoked multiple times in parallel. + /// If this closure panics, the panic will be passed to the panic handler. + /// If that handler returns, then the thread will exit normally. + pub fn exit_handler(mut self, exit_handler: H) -> Self + where + H: Fn(usize) + Send + Sync + 'static, + { + self.exit_handler = Some(Box::new(exit_handler)); + self + } +} + +#[allow(deprecated)] +impl Configuration { + /// Creates and return a valid rayon thread pool configuration, but does not initialize it. + pub fn new() -> Configuration { + Configuration { + builder: ThreadPoolBuilder::new(), + } + } + + /// Deprecated in favor of `ThreadPoolBuilder::build`. + pub fn build(self) -> Result> { + self.builder.build().map_err(Box::from) + } + + /// Deprecated in favor of `ThreadPoolBuilder::thread_name`. + pub fn thread_name(mut self, closure: F) -> Self + where + F: FnMut(usize) -> String + 'static, + { + self.builder = self.builder.thread_name(closure); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::num_threads`. + pub fn num_threads(mut self, num_threads: usize) -> Configuration { + self.builder = self.builder.num_threads(num_threads); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::panic_handler`. + pub fn panic_handler(mut self, panic_handler: H) -> Configuration + where + H: Fn(Box) + Send + Sync + 'static, + { + self.builder = self.builder.panic_handler(panic_handler); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::stack_size`. + pub fn stack_size(mut self, stack_size: usize) -> Self { + self.builder = self.builder.stack_size(stack_size); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::breadth_first`. + pub fn breadth_first(mut self) -> Self { + self.builder = self.builder.breadth_first(); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::start_handler`. + pub fn start_handler(mut self, start_handler: H) -> Configuration + where + H: Fn(usize) + Send + Sync + 'static, + { + self.builder = self.builder.start_handler(start_handler); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::exit_handler`. + pub fn exit_handler(mut self, exit_handler: H) -> Configuration + where + H: Fn(usize) + Send + Sync + 'static, + { + self.builder = self.builder.exit_handler(exit_handler); + self + } + + /// Returns a ThreadPoolBuilder with identical parameters. + fn into_builder(self) -> ThreadPoolBuilder { + self.builder + } +} + +impl ThreadPoolBuildError { + fn new(kind: ErrorKind) -> ThreadPoolBuildError { + ThreadPoolBuildError { kind } + } + + fn is_unsupported(&self) -> bool { + matches!(&self.kind, ErrorKind::IOError(e) if e.kind() == io::ErrorKind::Unsupported) + } +} + +const GLOBAL_POOL_ALREADY_INITIALIZED: &str = + "The global thread pool has already been initialized."; + +impl Error for ThreadPoolBuildError { + #[allow(deprecated)] + fn description(&self) -> &str { + match self.kind { + ErrorKind::GlobalPoolAlreadyInitialized => GLOBAL_POOL_ALREADY_INITIALIZED, + ErrorKind::IOError(ref e) => e.description(), + } + } + + fn source(&self) -> Option<&(dyn Error + 'static)> { + match &self.kind { + ErrorKind::GlobalPoolAlreadyInitialized => None, + ErrorKind::IOError(e) => Some(e), + } + } +} + +impl fmt::Display for ThreadPoolBuildError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.kind { + ErrorKind::GlobalPoolAlreadyInitialized => GLOBAL_POOL_ALREADY_INITIALIZED.fmt(f), + ErrorKind::IOError(e) => e.fmt(f), + } + } +} + +/// Deprecated in favor of `ThreadPoolBuilder::build_global`. +#[deprecated(note = "use `ThreadPoolBuilder::build_global`")] +#[allow(deprecated)] +pub fn initialize(config: Configuration) -> Result<(), Box> { + config.into_builder().build_global().map_err(Box::from) +} + +impl fmt::Debug for ThreadPoolBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let ThreadPoolBuilder { + ref num_threads, + ref get_thread_name, + ref panic_handler, + ref stack_size, + ref deadlock_handler, + ref start_handler, + ref exit_handler, + ref acquire_thread_handler, + ref release_thread_handler, + spawn_handler: _, + ref breadth_first, + } = *self; + + // Just print `Some()` or `None` to the debug + // output. + struct ClosurePlaceholder; + impl fmt::Debug for ClosurePlaceholder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } + } + let get_thread_name = get_thread_name.as_ref().map(|_| ClosurePlaceholder); + let panic_handler = panic_handler.as_ref().map(|_| ClosurePlaceholder); + let deadlock_handler = deadlock_handler.as_ref().map(|_| ClosurePlaceholder); + let start_handler = start_handler.as_ref().map(|_| ClosurePlaceholder); + let exit_handler = exit_handler.as_ref().map(|_| ClosurePlaceholder); + let acquire_thread_handler = acquire_thread_handler.as_ref().map(|_| ClosurePlaceholder); + let release_thread_handler = release_thread_handler.as_ref().map(|_| ClosurePlaceholder); + + f.debug_struct("ThreadPoolBuilder") + .field("num_threads", num_threads) + .field("get_thread_name", &get_thread_name) + .field("panic_handler", &panic_handler) + .field("stack_size", &stack_size) + .field("deadlock_handler", &deadlock_handler) + .field("start_handler", &start_handler) + .field("exit_handler", &exit_handler) + .field("acquire_thread_handler", &acquire_thread_handler) + .field("release_thread_handler", &release_thread_handler) + .field("breadth_first", &breadth_first) + .finish() + } +} + +#[allow(deprecated)] +impl fmt::Debug for Configuration { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.builder.fmt(f) + } +} + +/// Provides the calling context to a closure called by `join_context`. +#[derive(Debug)] +pub struct FnContext { + migrated: bool, + + /// disable `Send` and `Sync`, just for a little future-proofing. + _marker: PhantomData<*mut ()>, +} + +impl FnContext { + #[inline] + fn new(migrated: bool) -> Self { + FnContext { + migrated, + _marker: PhantomData, + } + } +} + +impl FnContext { + /// Returns `true` if the closure was called from a different thread + /// than it was provided from. + #[inline] + pub fn migrated(&self) -> bool { + self.migrated + } +} diff --git a/compiler/rustc_thread_pool/src/private.rs b/compiler/rustc_thread_pool/src/private.rs new file mode 100644 index 0000000000000..c85e77b9cbb73 --- /dev/null +++ b/compiler/rustc_thread_pool/src/private.rs @@ -0,0 +1,26 @@ +//! The public parts of this private module are used to create traits +//! that cannot be implemented outside of our own crate. This way we +//! can feel free to extend those traits without worrying about it +//! being a breaking change for other implementations. + +/// If this type is pub but not publicly reachable, third parties +/// can't name it and can't implement traits using it. +#[allow(missing_debug_implementations)] +pub struct PrivateMarker; + +macro_rules! private_decl { + () => { + /// This trait is private; this method exists to make it + /// impossible to implement outside the crate. + #[doc(hidden)] + fn __rayon_private__(&self) -> crate::private::PrivateMarker; + }; +} + +macro_rules! private_impl { + () => { + fn __rayon_private__(&self) -> crate::private::PrivateMarker { + crate::private::PrivateMarker + } + }; +} diff --git a/compiler/rustc_thread_pool/src/registry.rs b/compiler/rustc_thread_pool/src/registry.rs new file mode 100644 index 0000000000000..781b6827b829e --- /dev/null +++ b/compiler/rustc_thread_pool/src/registry.rs @@ -0,0 +1,1048 @@ +use crate::job::{JobFifo, JobRef, StackJob}; +use crate::latch::{AsCoreLatch, CoreLatch, Latch, LatchRef, LockLatch, OnceLatch, SpinLatch}; +use crate::sleep::Sleep; +use crate::tlv::Tlv; +use crate::unwind; +use crate::{ + AcquireThreadHandler, DeadlockHandler, ErrorKind, ExitHandler, PanicHandler, + ReleaseThreadHandler, StartHandler, ThreadPoolBuildError, ThreadPoolBuilder, Yield, +}; +use crossbeam_deque::{Injector, Steal, Stealer, Worker}; +use std::cell::Cell; +use std::collections::hash_map::DefaultHasher; +use std::fmt; +use std::hash::Hasher; +use std::io; +use std::mem; +use std::ptr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, Once}; +use std::thread; + +/// Thread builder used for customization via +/// [`ThreadPoolBuilder::spawn_handler`](struct.ThreadPoolBuilder.html#method.spawn_handler). +pub struct ThreadBuilder { + name: Option, + stack_size: Option, + worker: Worker, + stealer: Stealer, + registry: Arc, + index: usize, +} + +impl ThreadBuilder { + /// Gets the index of this thread in the pool, within `0..num_threads`. + pub fn index(&self) -> usize { + self.index + } + + /// Gets the string that was specified by `ThreadPoolBuilder::name()`. + pub fn name(&self) -> Option<&str> { + self.name.as_deref() + } + + /// Gets the value that was specified by `ThreadPoolBuilder::stack_size()`. + pub fn stack_size(&self) -> Option { + self.stack_size + } + + /// Executes the main loop for this thread. This will not return until the + /// thread pool is dropped. + pub fn run(self) { + unsafe { main_loop(self) } + } +} + +impl fmt::Debug for ThreadBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ThreadBuilder") + .field("pool", &self.registry.id()) + .field("index", &self.index) + .field("name", &self.name) + .field("stack_size", &self.stack_size) + .finish() + } +} + +/// Generalized trait for spawning a thread in the `Registry`. +/// +/// This trait is pub-in-private -- E0445 forces us to make it public, +/// but we don't actually want to expose these details in the API. +pub trait ThreadSpawn { + private_decl! {} + + /// Spawn a thread with the `ThreadBuilder` parameters, and then + /// call `ThreadBuilder::run()`. + fn spawn(&mut self, thread: ThreadBuilder) -> io::Result<()>; +} + +/// Spawns a thread in the "normal" way with `std::thread::Builder`. +/// +/// This type is pub-in-private -- E0445 forces us to make it public, +/// but we don't actually want to expose these details in the API. +#[derive(Debug, Default)] +pub struct DefaultSpawn; + +impl ThreadSpawn for DefaultSpawn { + private_impl! {} + + fn spawn(&mut self, thread: ThreadBuilder) -> io::Result<()> { + let mut b = thread::Builder::new(); + if let Some(name) = thread.name() { + b = b.name(name.to_owned()); + } + if let Some(stack_size) = thread.stack_size() { + b = b.stack_size(stack_size); + } + b.spawn(|| thread.run())?; + Ok(()) + } +} + +/// Spawns a thread with a user's custom callback. +/// +/// This type is pub-in-private -- E0445 forces us to make it public, +/// but we don't actually want to expose these details in the API. +#[derive(Debug)] +pub struct CustomSpawn(F); + +impl CustomSpawn +where + F: FnMut(ThreadBuilder) -> io::Result<()>, +{ + pub(super) fn new(spawn: F) -> Self { + CustomSpawn(spawn) + } +} + +impl ThreadSpawn for CustomSpawn +where + F: FnMut(ThreadBuilder) -> io::Result<()>, +{ + private_impl! {} + + #[inline] + fn spawn(&mut self, thread: ThreadBuilder) -> io::Result<()> { + (self.0)(thread) + } +} + +pub struct Registry { + thread_infos: Vec, + sleep: Sleep, + injected_jobs: Injector, + broadcasts: Mutex>>, + panic_handler: Option>, + pub(crate) deadlock_handler: Option>, + start_handler: Option>, + exit_handler: Option>, + pub(crate) acquire_thread_handler: Option>, + pub(crate) release_thread_handler: Option>, + + // When this latch reaches 0, it means that all work on this + // registry must be complete. This is ensured in the following ways: + // + // - if this is the global registry, there is a ref-count that never + // gets released. + // - if this is a user-created thread-pool, then so long as the thread-pool + // exists, it holds a reference. + // - when we inject a "blocking job" into the registry with `ThreadPool::install()`, + // no adjustment is needed; the `ThreadPool` holds the reference, and since we won't + // return until the blocking job is complete, that ref will continue to be held. + // - when `join()` or `scope()` is invoked, similarly, no adjustments are needed. + // These are always owned by some other job (e.g., one injected by `ThreadPool::install()`) + // and that job will keep the pool alive. + terminate_count: AtomicUsize, +} + +/// //////////////////////////////////////////////////////////////////////// +/// Initialization + +static mut THE_REGISTRY: Option> = None; +static THE_REGISTRY_SET: Once = Once::new(); + +/// Starts the worker threads (if that has not already happened). If +/// initialization has not already occurred, use the default +/// configuration. +pub(super) fn global_registry() -> &'static Arc { + set_global_registry(default_global_registry) + .or_else(|err| { + // SAFETY: we only create a shared reference to `THE_REGISTRY` after the `call_once` + // that initializes it, and there will be no more mutable accesses at all. + debug_assert!(THE_REGISTRY_SET.is_completed()); + let the_registry = unsafe { &*ptr::addr_of!(THE_REGISTRY) }; + the_registry.as_ref().ok_or(err) + }) + .expect("The global thread pool has not been initialized.") +} + +/// Starts the worker threads (if that has not already happened) with +/// the given builder. +pub(super) fn init_global_registry( + builder: ThreadPoolBuilder, +) -> Result<&'static Arc, ThreadPoolBuildError> +where + S: ThreadSpawn, +{ + set_global_registry(|| Registry::new(builder)) +} + +/// Starts the worker threads (if that has not already happened) +/// by creating a registry with the given callback. +fn set_global_registry(registry: F) -> Result<&'static Arc, ThreadPoolBuildError> +where + F: FnOnce() -> Result, ThreadPoolBuildError>, +{ + let mut result = Err(ThreadPoolBuildError::new( + ErrorKind::GlobalPoolAlreadyInitialized, + )); + + THE_REGISTRY_SET.call_once(|| { + result = registry().map(|registry: Arc| { + // SAFETY: this is the only mutable access to `THE_REGISTRY`, thanks to `Once`, and + // `global_registry()` only takes a shared reference **after** this `call_once`. + unsafe { + ptr::addr_of_mut!(THE_REGISTRY).write(Some(registry)); + (*ptr::addr_of!(THE_REGISTRY)).as_ref().unwrap_unchecked() + } + }) + }); + + result +} + +fn default_global_registry() -> Result, ThreadPoolBuildError> { + let result = Registry::new(ThreadPoolBuilder::new()); + + // If we're running in an environment that doesn't support threads at all, we can fall back to + // using the current thread alone. This is crude, and probably won't work for non-blocking + // calls like `spawn` or `broadcast_spawn`, but a lot of stuff does work fine. + // + // Notably, this allows current WebAssembly targets to work even though their threading support + // is stubbed out, and we won't have to change anything if they do add real threading. + let unsupported = matches!(&result, Err(e) if e.is_unsupported()); + if unsupported && WorkerThread::current().is_null() { + let builder = ThreadPoolBuilder::new() + .num_threads(1) + .spawn_handler(|thread| { + // Rather than starting a new thread, we're just taking over the current thread + // *without* running the main loop, so we can still return from here. + // The WorkerThread is leaked, but we never shutdown the global pool anyway. + let worker_thread = Box::leak(Box::new(WorkerThread::from(thread))); + let registry = &*worker_thread.registry; + let index = worker_thread.index; + + unsafe { + WorkerThread::set_current(worker_thread); + + // let registry know we are ready to do work + Latch::set(®istry.thread_infos[index].primed); + } + + Ok(()) + }); + + let fallback_result = Registry::new(builder); + if fallback_result.is_ok() { + return fallback_result; + } + } + + result +} + +struct Terminator<'a>(&'a Arc); + +impl<'a> Drop for Terminator<'a> { + fn drop(&mut self) { + self.0.terminate() + } +} + +impl Registry { + pub(super) fn new( + mut builder: ThreadPoolBuilder, + ) -> Result, ThreadPoolBuildError> + where + S: ThreadSpawn, + { + // Soft-limit the number of threads that we can actually support. + let n_threads = Ord::min(builder.get_num_threads(), crate::max_num_threads()); + + let breadth_first = builder.get_breadth_first(); + + let (workers, stealers): (Vec<_>, Vec<_>) = (0..n_threads) + .map(|_| { + let worker = if breadth_first { + Worker::new_fifo() + } else { + Worker::new_lifo() + }; + + let stealer = worker.stealer(); + (worker, stealer) + }) + .unzip(); + + let (broadcasts, broadcast_stealers): (Vec<_>, Vec<_>) = (0..n_threads) + .map(|_| { + let worker = Worker::new_fifo(); + let stealer = worker.stealer(); + (worker, stealer) + }) + .unzip(); + + let registry = Arc::new(Registry { + thread_infos: stealers.into_iter().map(ThreadInfo::new).collect(), + sleep: Sleep::new(n_threads), + injected_jobs: Injector::new(), + broadcasts: Mutex::new(broadcasts), + terminate_count: AtomicUsize::new(1), + panic_handler: builder.take_panic_handler(), + deadlock_handler: builder.take_deadlock_handler(), + start_handler: builder.take_start_handler(), + exit_handler: builder.take_exit_handler(), + acquire_thread_handler: builder.take_acquire_thread_handler(), + release_thread_handler: builder.take_release_thread_handler(), + }); + + // If we return early or panic, make sure to terminate existing threads. + let t1000 = Terminator(®istry); + + for (index, (worker, stealer)) in workers.into_iter().zip(broadcast_stealers).enumerate() { + let thread = ThreadBuilder { + name: builder.get_thread_name(index), + stack_size: builder.get_stack_size(), + registry: Arc::clone(®istry), + worker, + stealer, + index, + }; + if let Err(e) = builder.get_spawn_handler().spawn(thread) { + return Err(ThreadPoolBuildError::new(ErrorKind::IOError(e))); + } + } + + // Returning normally now, without termination. + mem::forget(t1000); + + Ok(registry) + } + + pub fn current() -> Arc { + unsafe { + let worker_thread = WorkerThread::current(); + let registry = if worker_thread.is_null() { + global_registry() + } else { + &(*worker_thread).registry + }; + Arc::clone(registry) + } + } + + /// Returns the number of threads in the current registry. This + /// is better than `Registry::current().num_threads()` because it + /// avoids incrementing the `Arc`. + pub(super) fn current_num_threads() -> usize { + unsafe { + let worker_thread = WorkerThread::current(); + if worker_thread.is_null() { + global_registry().num_threads() + } else { + (*worker_thread).registry.num_threads() + } + } + } + + /// Returns the current `WorkerThread` if it's part of this `Registry`. + pub(super) fn current_thread(&self) -> Option<&WorkerThread> { + unsafe { + let worker = WorkerThread::current().as_ref()?; + if worker.registry().id() == self.id() { + Some(worker) + } else { + None + } + } + } + + /// Returns an opaque identifier for this registry. + pub(super) fn id(&self) -> RegistryId { + // We can rely on `self` not to change since we only ever create + // registries that are boxed up in an `Arc` (see `new()` above). + RegistryId { + addr: self as *const Self as usize, + } + } + + pub(super) fn num_threads(&self) -> usize { + self.thread_infos.len() + } + + pub(super) fn catch_unwind(&self, f: impl FnOnce()) { + if let Err(err) = unwind::halt_unwinding(f) { + // If there is no handler, or if that handler itself panics, then we abort. + let abort_guard = unwind::AbortIfPanic; + if let Some(ref handler) = self.panic_handler { + handler(err); + mem::forget(abort_guard); + } + } + } + + /// Waits for the worker threads to get up and running. This is + /// meant to be used for benchmarking purposes, primarily, so that + /// you can get more consistent numbers by having everything + /// "ready to go". + pub(super) fn wait_until_primed(&self) { + for info in &self.thread_infos { + info.primed.wait(); + } + } + + /// Waits for the worker threads to stop. This is used for testing + /// -- so we can check that termination actually works. + pub(super) fn wait_until_stopped(&self) { + self.release_thread(); + for info in &self.thread_infos { + info.stopped.wait(); + } + self.acquire_thread(); + } + + pub(crate) fn acquire_thread(&self) { + if let Some(ref acquire_thread_handler) = self.acquire_thread_handler { + acquire_thread_handler(); + } + } + + pub(crate) fn release_thread(&self) { + if let Some(ref release_thread_handler) = self.release_thread_handler { + release_thread_handler(); + } + } + + /// //////////////////////////////////////////////////////////////////////// + /// MAIN LOOP + /// + /// So long as all of the worker threads are hanging out in their + /// top-level loop, there is no work to be done. + + /// Push a job into the given `registry`. If we are running on a + /// worker thread for the registry, this will push onto the + /// deque. Else, it will inject from the outside (which is slower). + pub(super) fn inject_or_push(&self, job_ref: JobRef) { + let worker_thread = WorkerThread::current(); + unsafe { + if !worker_thread.is_null() && (*worker_thread).registry().id() == self.id() { + (*worker_thread).push(job_ref); + } else { + self.inject(job_ref); + } + } + } + + /// Push a job into the "external jobs" queue; it will be taken by + /// whatever worker has nothing to do. Use this if you know that + /// you are not on a worker of this registry. + pub(super) fn inject(&self, injected_job: JobRef) { + // It should not be possible for `state.terminate` to be true + // here. It is only set to true when the user creates (and + // drops) a `ThreadPool`; and, in that case, they cannot be + // calling `inject()` later, since they dropped their + // `ThreadPool`. + debug_assert_ne!( + self.terminate_count.load(Ordering::Acquire), + 0, + "inject() sees state.terminate as true" + ); + + let queue_was_empty = self.injected_jobs.is_empty(); + + self.injected_jobs.push(injected_job); + self.sleep.new_injected_jobs(1, queue_was_empty); + } + + pub(crate) fn has_injected_job(&self) -> bool { + !self.injected_jobs.is_empty() + } + + fn pop_injected_job(&self) -> Option { + loop { + match self.injected_jobs.steal() { + Steal::Success(job) => return Some(job), + Steal::Empty => return None, + Steal::Retry => {} + } + } + } + + /// Push a job into each thread's own "external jobs" queue; it will be + /// executed only on that thread, when it has nothing else to do locally, + /// before it tries to steal other work. + /// + /// **Panics** if not given exactly as many jobs as there are threads. + pub(super) fn inject_broadcast(&self, injected_jobs: impl ExactSizeIterator) { + assert_eq!(self.num_threads(), injected_jobs.len()); + { + let broadcasts = self.broadcasts.lock().unwrap(); + + // It should not be possible for `state.terminate` to be true + // here. It is only set to true when the user creates (and + // drops) a `ThreadPool`; and, in that case, they cannot be + // calling `inject_broadcast()` later, since they dropped their + // `ThreadPool`. + debug_assert_ne!( + self.terminate_count.load(Ordering::Acquire), + 0, + "inject_broadcast() sees state.terminate as true" + ); + + assert_eq!(broadcasts.len(), injected_jobs.len()); + for (worker, job_ref) in broadcasts.iter().zip(injected_jobs) { + worker.push(job_ref); + } + } + for i in 0..self.num_threads() { + self.sleep.notify_worker_latch_is_set(i); + } + } + + /// If already in a worker-thread of this registry, just execute `op`. + /// Otherwise, inject `op` in this thread-pool. Either way, block until `op` + /// completes and return its return value. If `op` panics, that panic will + /// be propagated as well. The second argument indicates `true` if injection + /// was performed, `false` if executed directly. + pub(super) fn in_worker(&self, op: OP) -> R + where + OP: FnOnce(&WorkerThread, bool) -> R + Send, + R: Send, + { + unsafe { + let worker_thread = WorkerThread::current(); + if worker_thread.is_null() { + self.in_worker_cold(op) + } else if (*worker_thread).registry().id() != self.id() { + self.in_worker_cross(&*worker_thread, op) + } else { + // Perfectly valid to give them a `&T`: this is the + // current thread, so we know the data structure won't be + // invalidated until we return. + op(&*worker_thread, false) + } + } + } + + #[cold] + unsafe fn in_worker_cold(&self, op: OP) -> R + where + OP: FnOnce(&WorkerThread, bool) -> R + Send, + R: Send, + { + thread_local!(static LOCK_LATCH: LockLatch = LockLatch::new()); + + LOCK_LATCH.with(|l| { + // This thread isn't a member of *any* thread pool, so just block. + debug_assert!(WorkerThread::current().is_null()); + let job = StackJob::new( + Tlv::null(), + |injected| { + let worker_thread = WorkerThread::current(); + assert!(injected && !worker_thread.is_null()); + op(&*worker_thread, true) + }, + LatchRef::new(l), + ); + self.inject(job.as_job_ref()); + self.release_thread(); + job.latch.wait_and_reset(); // Make sure we can use the same latch again next time. + self.acquire_thread(); + + job.into_result() + }) + } + + #[cold] + unsafe fn in_worker_cross(&self, current_thread: &WorkerThread, op: OP) -> R + where + OP: FnOnce(&WorkerThread, bool) -> R + Send, + R: Send, + { + // This thread is a member of a different pool, so let it process + // other work while waiting for this `op` to complete. + debug_assert!(current_thread.registry().id() != self.id()); + let latch = SpinLatch::cross(current_thread); + let job = StackJob::new( + Tlv::null(), + |injected| { + let worker_thread = WorkerThread::current(); + assert!(injected && !worker_thread.is_null()); + op(&*worker_thread, true) + }, + latch, + ); + self.inject(job.as_job_ref()); + current_thread.wait_until(&job.latch); + job.into_result() + } + + /// Increments the terminate counter. This increment should be + /// balanced by a call to `terminate`, which will decrement. This + /// is used when spawning asynchronous work, which needs to + /// prevent the registry from terminating so long as it is active. + /// + /// Note that blocking functions such as `join` and `scope` do not + /// need to concern themselves with this fn; their context is + /// responsible for ensuring the current thread-pool will not + /// terminate until they return. + /// + /// The global thread-pool always has an outstanding reference + /// (the initial one). Custom thread-pools have one outstanding + /// reference that is dropped when the `ThreadPool` is dropped: + /// since installing the thread-pool blocks until any joins/scopes + /// complete, this ensures that joins/scopes are covered. + /// + /// The exception is `::spawn()`, which can create a job outside + /// of any blocking scope. In that case, the job itself holds a + /// terminate count and is responsible for invoking `terminate()` + /// when finished. + pub(super) fn increment_terminate_count(&self) { + let previous = self.terminate_count.fetch_add(1, Ordering::AcqRel); + debug_assert!(previous != 0, "registry ref count incremented from zero"); + assert!(previous != usize::MAX, "overflow in registry ref count"); + } + + /// Signals that the thread-pool which owns this registry has been + /// dropped. The worker threads will gradually terminate, once any + /// extant work is completed. + pub(super) fn terminate(&self) { + if self.terminate_count.fetch_sub(1, Ordering::AcqRel) == 1 { + for (i, thread_info) in self.thread_infos.iter().enumerate() { + unsafe { OnceLatch::set_and_tickle_one(&thread_info.terminate, self, i) }; + } + } + } + + /// Notify the worker that the latch they are sleeping on has been "set". + pub(super) fn notify_worker_latch_is_set(&self, target_worker_index: usize) { + self.sleep.notify_worker_latch_is_set(target_worker_index); + } +} + +/// Mark a Rayon worker thread as blocked. This triggers the deadlock handler +/// if no other worker thread is active +#[inline] +pub fn mark_blocked() { + let worker_thread = WorkerThread::current(); + assert!(!worker_thread.is_null()); + unsafe { + let registry = &(*worker_thread).registry; + registry.sleep.mark_blocked(®istry.deadlock_handler) + } +} + +/// Mark a previously blocked Rayon worker thread as unblocked +#[inline] +pub fn mark_unblocked(registry: &Registry) { + registry.sleep.mark_unblocked() +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct RegistryId { + addr: usize, +} + +struct ThreadInfo { + /// Latch set once thread has started and we are entering into the + /// main loop. Used to wait for worker threads to become primed, + /// primarily of interest for benchmarking. + primed: LockLatch, + + /// Latch is set once worker thread has completed. Used to wait + /// until workers have stopped; only used for tests. + stopped: LockLatch, + + /// The latch used to signal that terminated has been requested. + /// This latch is *set* by the `terminate` method on the + /// `Registry`, once the registry's main "terminate" counter + /// reaches zero. + terminate: OnceLatch, + + /// the "stealer" half of the worker's deque + stealer: Stealer, +} + +impl ThreadInfo { + fn new(stealer: Stealer) -> ThreadInfo { + ThreadInfo { + primed: LockLatch::new(), + stopped: LockLatch::new(), + terminate: OnceLatch::new(), + stealer, + } + } +} + +/// //////////////////////////////////////////////////////////////////////// +/// WorkerThread identifiers + +pub(super) struct WorkerThread { + /// the "worker" half of our local deque + worker: Worker, + + /// the "stealer" half of the worker's broadcast deque + stealer: Stealer, + + /// local queue used for `spawn_fifo` indirection + fifo: JobFifo, + + pub(crate) index: usize, + + /// A weak random number generator. + rng: XorShift64Star, + + pub(crate) registry: Arc, +} + +// This is a bit sketchy, but basically: the WorkerThread is +// allocated on the stack of the worker on entry and stored into this +// thread local variable. So it will remain valid at least until the +// worker is fully unwound. Using an unsafe pointer avoids the need +// for a RefCell etc. +thread_local! { + static WORKER_THREAD_STATE: Cell<*const WorkerThread> = const { Cell::new(ptr::null()) }; +} + +impl From for WorkerThread { + fn from(thread: ThreadBuilder) -> Self { + Self { + worker: thread.worker, + stealer: thread.stealer, + fifo: JobFifo::new(), + index: thread.index, + rng: XorShift64Star::new(), + registry: thread.registry, + } + } +} + +impl Drop for WorkerThread { + fn drop(&mut self) { + // Undo `set_current` + WORKER_THREAD_STATE.with(|t| { + assert!(t.get().eq(&(self as *const _))); + t.set(ptr::null()); + }); + } +} + +impl WorkerThread { + /// Gets the `WorkerThread` index for the current thread; returns + /// NULL if this is not a worker thread. This pointer is valid + /// anywhere on the current thread. + #[inline] + pub(super) fn current() -> *const WorkerThread { + WORKER_THREAD_STATE.with(Cell::get) + } + + /// Sets `self` as the worker thread index for the current thread. + /// This is done during worker thread startup. + unsafe fn set_current(thread: *const WorkerThread) { + WORKER_THREAD_STATE.with(|t| { + assert!(t.get().is_null()); + t.set(thread); + }); + } + + /// Returns the registry that owns this worker thread. + #[inline] + pub(super) fn registry(&self) -> &Arc { + &self.registry + } + + /// Our index amongst the worker threads (ranges from `0..self.num_threads()`). + #[inline] + pub(super) fn index(&self) -> usize { + self.index + } + + #[inline] + pub(super) unsafe fn push(&self, job: JobRef) { + let queue_was_empty = self.worker.is_empty(); + self.worker.push(job); + self.registry.sleep.new_internal_jobs(1, queue_was_empty); + } + + #[inline] + pub(super) unsafe fn push_fifo(&self, job: JobRef) { + self.push(self.fifo.push(job)); + } + + #[inline] + pub(super) fn local_deque_is_empty(&self) -> bool { + self.worker.is_empty() + } + + /// Attempts to obtain a "local" job -- typically this means + /// popping from the top of the stack, though if we are configured + /// for breadth-first execution, it would mean dequeuing from the + /// bottom. + #[inline] + pub(super) fn take_local_job(&self) -> Option { + let popped_job = self.worker.pop(); + + if popped_job.is_some() { + return popped_job; + } + + loop { + match self.stealer.steal() { + Steal::Success(job) => return Some(job), + Steal::Empty => return None, + Steal::Retry => {} + } + } + } + + pub(super) fn has_injected_job(&self) -> bool { + !self.stealer.is_empty() || self.registry.has_injected_job() + } + + /// Wait until the latch is set. Try to keep busy by popping and + /// stealing tasks as necessary. + #[inline] + pub(super) unsafe fn wait_until(&self, latch: &L) { + let latch = latch.as_core_latch(); + if !latch.probe() { + self.wait_until_cold(latch); + } + } + + #[cold] + unsafe fn wait_until_cold(&self, latch: &CoreLatch) { + // the code below should swallow all panics and hence never + // unwind; but if something does wrong, we want to abort, + // because otherwise other code in rayon may assume that the + // latch has been signaled, and that can lead to random memory + // accesses, which would be *very bad* + let abort_guard = unwind::AbortIfPanic; + + 'outer: while !latch.probe() { + // Check for local work *before* we start marking ourself idle, + // especially to avoid modifying shared sleep state. + if let Some(job) = self.take_local_job() { + self.execute(job); + continue; + } + + let mut idle_state = self.registry.sleep.start_looking(self.index); + while !latch.probe() { + if let Some(job) = self.find_work() { + self.registry.sleep.work_found(); + self.execute(job); + // The job might have injected local work, so go back to the outer loop. + continue 'outer; + } else { + self.registry + .sleep + .no_work_found(&mut idle_state, latch, &self) + } + } + + // If we were sleepy, we are not anymore. We "found work" -- + // whatever the surrounding thread was doing before it had to wait. + self.registry.sleep.work_found(); + break; + } + + mem::forget(abort_guard); // successful execution, do not abort + } + + unsafe fn wait_until_out_of_work(&self) { + debug_assert_eq!(self as *const _, WorkerThread::current()); + let registry = &*self.registry; + let index = self.index; + + registry.acquire_thread(); + self.wait_until(®istry.thread_infos[index].terminate); + + // Should not be any work left in our queue. + debug_assert!(self.take_local_job().is_none()); + + // Let registry know we are done + Latch::set(®istry.thread_infos[index].stopped); + } + + fn find_work(&self) -> Option { + // Try to find some work to do. We give preference first + // to things in our local deque, then in other workers + // deques, and finally to injected jobs from the + // outside. The idea is to finish what we started before + // we take on something new. + self.take_local_job() + .or_else(|| self.steal()) + .or_else(|| self.registry.pop_injected_job()) + } + + pub(super) fn yield_now(&self) -> Yield { + match self.find_work() { + Some(job) => unsafe { + self.execute(job); + Yield::Executed + }, + None => Yield::Idle, + } + } + + pub(super) fn yield_local(&self) -> Yield { + match self.take_local_job() { + Some(job) => unsafe { + self.execute(job); + Yield::Executed + }, + None => Yield::Idle, + } + } + + #[inline] + pub(super) unsafe fn execute(&self, job: JobRef) { + job.execute(); + } + + /// Try to steal a single job and return it. + /// + /// This should only be done as a last resort, when there is no + /// local work to do. + fn steal(&self) -> Option { + // we only steal when we don't have any work to do locally + debug_assert!(self.local_deque_is_empty()); + + // otherwise, try to steal + let thread_infos = &self.registry.thread_infos.as_slice(); + let num_threads = thread_infos.len(); + if num_threads <= 1 { + return None; + } + + loop { + let mut retry = false; + let start = self.rng.next_usize(num_threads); + let job = (start..num_threads) + .chain(0..start) + .filter(move |&i| i != self.index) + .find_map(|victim_index| { + let victim = &thread_infos[victim_index]; + match victim.stealer.steal() { + Steal::Success(job) => Some(job), + Steal::Empty => None, + Steal::Retry => { + retry = true; + None + } + } + }); + if job.is_some() || !retry { + return job; + } + } + } +} + +/// //////////////////////////////////////////////////////////////////////// + +unsafe fn main_loop(thread: ThreadBuilder) { + let worker_thread = &WorkerThread::from(thread); + WorkerThread::set_current(worker_thread); + let registry = &*worker_thread.registry; + let index = worker_thread.index; + + // let registry know we are ready to do work + Latch::set(®istry.thread_infos[index].primed); + + // Worker threads should not panic. If they do, just abort, as the + // internal state of the threadpool is corrupted. Note that if + // **user code** panics, we should catch that and redirect. + let abort_guard = unwind::AbortIfPanic; + + // Inform a user callback that we started a thread. + if let Some(ref handler) = registry.start_handler { + registry.catch_unwind(|| handler(index)); + } + + worker_thread.wait_until_out_of_work(); + + // Normal termination, do not abort. + mem::forget(abort_guard); + + // Inform a user callback that we exited a thread. + if let Some(ref handler) = registry.exit_handler { + registry.catch_unwind(|| handler(index)); + // We're already exiting the thread, there's nothing else to do. + } + + registry.release_thread(); +} + +/// If already in a worker-thread, just execute `op`. Otherwise, +/// execute `op` in the default thread-pool. Either way, block until +/// `op` completes and return its return value. If `op` panics, that +/// panic will be propagated as well. The second argument indicates +/// `true` if injection was performed, `false` if executed directly. +pub(super) fn in_worker(op: OP) -> R +where + OP: FnOnce(&WorkerThread, bool) -> R + Send, + R: Send, +{ + unsafe { + let owner_thread = WorkerThread::current(); + if !owner_thread.is_null() { + // Perfectly valid to give them a `&T`: this is the + // current thread, so we know the data structure won't be + // invalidated until we return. + op(&*owner_thread, false) + } else { + global_registry().in_worker(op) + } + } +} + +/// [xorshift*] is a fast pseudorandom number generator which will +/// even tolerate weak seeding, as long as it's not zero. +/// +/// [xorshift*]: https://en.wikipedia.org/wiki/Xorshift#xorshift* +struct XorShift64Star { + state: Cell, +} + +impl XorShift64Star { + fn new() -> Self { + // Any non-zero seed will do -- this uses the hash of a global counter. + let mut seed = 0; + while seed == 0 { + let mut hasher = DefaultHasher::new(); + static COUNTER: AtomicUsize = AtomicUsize::new(0); + hasher.write_usize(COUNTER.fetch_add(1, Ordering::Relaxed)); + seed = hasher.finish(); + } + + XorShift64Star { + state: Cell::new(seed), + } + } + + fn next(&self) -> u64 { + let mut x = self.state.get(); + debug_assert_ne!(x, 0); + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.state.set(x); + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// Return a value from `0..n`. + fn next_usize(&self, n: usize) -> usize { + (self.next() % n as u64) as usize + } +} diff --git a/compiler/rustc_thread_pool/src/scope/mod.rs b/compiler/rustc_thread_pool/src/scope/mod.rs new file mode 100644 index 0000000000000..364b322baadf4 --- /dev/null +++ b/compiler/rustc_thread_pool/src/scope/mod.rs @@ -0,0 +1,783 @@ +//! Methods for custom fork-join scopes, created by the [`scope()`] +//! and [`in_place_scope()`] functions. These are a more flexible alternative to [`join()`]. +//! +//! [`scope()`]: fn.scope.html +//! [`in_place_scope()`]: fn.in_place_scope.html +//! [`join()`]: ../join/join.fn.html + +use crate::broadcast::BroadcastContext; +use crate::job::{ArcJob, HeapJob, JobFifo, JobRef}; +use crate::latch::{CountLatch, Latch}; +use crate::registry::{global_registry, in_worker, Registry, WorkerThread}; +use crate::tlv::{self, Tlv}; +use crate::unwind; +use std::any::Any; +use std::fmt; +use std::marker::PhantomData; +use std::mem::ManuallyDrop; +use std::ptr; +use std::sync::atomic::{AtomicPtr, Ordering}; +use std::sync::Arc; + +#[cfg(test)] +mod test; + +/// Represents a fork-join scope which can be used to spawn any number of tasks. +/// See [`scope()`] for more information. +/// +///[`scope()`]: fn.scope.html +pub struct Scope<'scope> { + base: ScopeBase<'scope>, +} + +/// Represents a fork-join scope which can be used to spawn any number of tasks. +/// Those spawned from the same thread are prioritized in relative FIFO order. +/// See [`scope_fifo()`] for more information. +/// +///[`scope_fifo()`]: fn.scope_fifo.html +pub struct ScopeFifo<'scope> { + base: ScopeBase<'scope>, + fifos: Vec, +} + +struct ScopeBase<'scope> { + /// thread registry where `scope()` was executed or where `in_place_scope()` + /// should spawn jobs. + registry: Arc, + + /// if some job panicked, the error is stored here; it will be + /// propagated to the one who created the scope + panic: AtomicPtr>, + + /// latch to track job counts + job_completed_latch: CountLatch, + + /// You can think of a scope as containing a list of closures to execute, + /// all of which outlive `'scope`. They're not actually required to be + /// `Sync`, but it's still safe to let the `Scope` implement `Sync` because + /// the closures are only *moved* across threads to be executed. + #[allow(clippy::type_complexity)] + marker: PhantomData) + Send + Sync + 'scope>>, + + /// The TLV at the scope's creation. Used to set the TLV for spawned jobs. + tlv: Tlv, +} + +/// Creates a "fork-join" scope `s` and invokes the closure with a +/// reference to `s`. This closure can then spawn asynchronous tasks +/// into `s`. Those tasks may run asynchronously with respect to the +/// closure; they may themselves spawn additional tasks into `s`. When +/// the closure returns, it will block until all tasks that have been +/// spawned into `s` complete. +/// +/// `scope()` is a more flexible building block compared to `join()`, +/// since a loop can be used to spawn any number of tasks without +/// recursing. However, that flexibility comes at a performance price: +/// tasks spawned using `scope()` must be allocated onto the heap, +/// whereas `join()` can make exclusive use of the stack. **Prefer +/// `join()` (or, even better, parallel iterators) where possible.** +/// +/// # Example +/// +/// The Rayon `join()` function launches two closures and waits for them +/// to stop. One could implement `join()` using a scope like so, although +/// it would be less efficient than the real implementation: +/// +/// ```rust +/// # use rayon_core as rayon; +/// pub fn join(oper_a: A, oper_b: B) -> (RA, RB) +/// where A: FnOnce() -> RA + Send, +/// B: FnOnce() -> RB + Send, +/// RA: Send, +/// RB: Send, +/// { +/// let mut result_a: Option = None; +/// let mut result_b: Option = None; +/// rayon::scope(|s| { +/// s.spawn(|_| result_a = Some(oper_a())); +/// s.spawn(|_| result_b = Some(oper_b())); +/// }); +/// (result_a.unwrap(), result_b.unwrap()) +/// } +/// ``` +/// +/// # A note on threading +/// +/// The closure given to `scope()` executes in the Rayon thread-pool, +/// as do those given to `spawn()`. This means that you can't access +/// thread-local variables (well, you can, but they may have +/// unexpected values). +/// +/// # Task execution +/// +/// Task execution potentially starts as soon as `spawn()` is called. +/// The task will end sometime before `scope()` returns. Note that the +/// *closure* given to scope may return much earlier. In general +/// the lifetime of a scope created like `scope(body)` goes something like this: +/// +/// - Scope begins when `scope(body)` is called +/// - Scope body `body()` is invoked +/// - Scope tasks may be spawned +/// - Scope body returns +/// - Scope tasks execute, possibly spawning more tasks +/// - Once all tasks are done, scope ends and `scope()` returns +/// +/// To see how and when tasks are joined, consider this example: +/// +/// ```rust +/// # use rayon_core as rayon; +/// // point start +/// rayon::scope(|s| { +/// s.spawn(|s| { // task s.1 +/// s.spawn(|s| { // task s.1.1 +/// rayon::scope(|t| { +/// t.spawn(|_| ()); // task t.1 +/// t.spawn(|_| ()); // task t.2 +/// }); +/// }); +/// }); +/// s.spawn(|s| { // task s.2 +/// }); +/// // point mid +/// }); +/// // point end +/// ``` +/// +/// The various tasks that are run will execute roughly like so: +/// +/// ```notrust +/// | (start) +/// | +/// | (scope `s` created) +/// +-----------------------------------------------+ (task s.2) +/// +-------+ (task s.1) | +/// | | | +/// | +---+ (task s.1.1) | +/// | | | | +/// | | | (scope `t` created) | +/// | | +----------------+ (task t.2) | +/// | | +---+ (task t.1) | | +/// | (mid) | | | | | +/// : | + <-+------------+ (scope `t` ends) | +/// : | | | +/// |<------+---+-----------------------------------+ (scope `s` ends) +/// | +/// | (end) +/// ``` +/// +/// The point here is that everything spawned into scope `s` will +/// terminate (at latest) at the same point -- right before the +/// original call to `rayon::scope` returns. This includes new +/// subtasks created by other subtasks (e.g., task `s.1.1`). If a new +/// scope is created (such as `t`), the things spawned into that scope +/// will be joined before that scope returns, which in turn occurs +/// before the creating task (task `s.1.1` in this case) finishes. +/// +/// There is no guaranteed order of execution for spawns in a scope, +/// given that other threads may steal tasks at any time. However, they +/// are generally prioritized in a LIFO order on the thread from which +/// they were spawned. So in this example, absent any stealing, we can +/// expect `s.2` to execute before `s.1`, and `t.2` before `t.1`. Other +/// threads always steal from the other end of the deque, like FIFO +/// order. The idea is that "recent" tasks are most likely to be fresh +/// in the local CPU's cache, while other threads can steal older +/// "stale" tasks. For an alternate approach, consider +/// [`scope_fifo()`] instead. +/// +/// [`scope_fifo()`]: fn.scope_fifo.html +/// +/// # Accessing stack data +/// +/// In general, spawned tasks may access stack data in place that +/// outlives the scope itself. Other data must be fully owned by the +/// spawned task. +/// +/// ```rust +/// # use rayon_core as rayon; +/// let ok: Vec = vec![1, 2, 3]; +/// rayon::scope(|s| { +/// let bad: Vec = vec![4, 5, 6]; +/// s.spawn(|_| { +/// // We can access `ok` because outlives the scope `s`. +/// println!("ok: {:?}", ok); +/// +/// // If we just try to use `bad` here, the closure will borrow `bad` +/// // (because we are just printing it out, and that only requires a +/// // borrow), which will result in a compilation error. Read on +/// // for options. +/// // println!("bad: {:?}", bad); +/// }); +/// }); +/// ``` +/// +/// As the comments example above suggest, to reference `bad` we must +/// take ownership of it. One way to do this is to detach the closure +/// from the surrounding stack frame, using the `move` keyword. This +/// will cause it to take ownership of *all* the variables it touches, +/// in this case including both `ok` *and* `bad`: +/// +/// ```rust +/// # use rayon_core as rayon; +/// let ok: Vec = vec![1, 2, 3]; +/// rayon::scope(|s| { +/// let bad: Vec = vec![4, 5, 6]; +/// s.spawn(move |_| { +/// println!("ok: {:?}", ok); +/// println!("bad: {:?}", bad); +/// }); +/// +/// // That closure is fine, but now we can't use `ok` anywhere else, +/// // since it is owned by the previous task: +/// // s.spawn(|_| println!("ok: {:?}", ok)); +/// }); +/// ``` +/// +/// While this works, it could be a problem if we want to use `ok` elsewhere. +/// There are two choices. We can keep the closure as a `move` closure, but +/// instead of referencing the variable `ok`, we create a shadowed variable that +/// is a borrow of `ok` and capture *that*: +/// +/// ```rust +/// # use rayon_core as rayon; +/// let ok: Vec = vec![1, 2, 3]; +/// rayon::scope(|s| { +/// let bad: Vec = vec![4, 5, 6]; +/// let ok: &Vec = &ok; // shadow the original `ok` +/// s.spawn(move |_| { +/// println!("ok: {:?}", ok); // captures the shadowed version +/// println!("bad: {:?}", bad); +/// }); +/// +/// // Now we too can use the shadowed `ok`, since `&Vec` references +/// // can be shared freely. Note that we need a `move` closure here though, +/// // because otherwise we'd be trying to borrow the shadowed `ok`, +/// // and that doesn't outlive `scope`. +/// s.spawn(move |_| println!("ok: {:?}", ok)); +/// }); +/// ``` +/// +/// Another option is not to use the `move` keyword but instead to take ownership +/// of individual variables: +/// +/// ```rust +/// # use rayon_core as rayon; +/// let ok: Vec = vec![1, 2, 3]; +/// rayon::scope(|s| { +/// let bad: Vec = vec![4, 5, 6]; +/// s.spawn(|_| { +/// // Transfer ownership of `bad` into a local variable (also named `bad`). +/// // This will force the closure to take ownership of `bad` from the environment. +/// let bad = bad; +/// println!("ok: {:?}", ok); // `ok` is only borrowed. +/// println!("bad: {:?}", bad); // refers to our local variable, above. +/// }); +/// +/// s.spawn(|_| println!("ok: {:?}", ok)); // we too can borrow `ok` +/// }); +/// ``` +/// +/// # Panics +/// +/// If a panic occurs, either in the closure given to `scope()` or in +/// any of the spawned jobs, that panic will be propagated and the +/// call to `scope()` will panic. If multiple panics occurs, it is +/// non-deterministic which of their panic values will propagate. +/// Regardless, once a task is spawned using `scope.spawn()`, it will +/// execute, even if the spawning task should later panic. `scope()` +/// returns once all spawned jobs have completed, and any panics are +/// propagated at that point. +pub fn scope<'scope, OP, R>(op: OP) -> R +where + OP: FnOnce(&Scope<'scope>) -> R + Send, + R: Send, +{ + in_worker(|owner_thread, _| { + let scope = Scope::<'scope>::new(Some(owner_thread), None); + scope.base.complete(Some(owner_thread), || op(&scope)) + }) +} + +/// Creates a "fork-join" scope `s` with FIFO order, and invokes the +/// closure with a reference to `s`. This closure can then spawn +/// asynchronous tasks into `s`. Those tasks may run asynchronously with +/// respect to the closure; they may themselves spawn additional tasks +/// into `s`. When the closure returns, it will block until all tasks +/// that have been spawned into `s` complete. +/// +/// # Task execution +/// +/// Tasks in a `scope_fifo()` run similarly to [`scope()`], but there's a +/// difference in the order of execution. Consider a similar example: +/// +/// [`scope()`]: fn.scope.html +/// +/// ```rust +/// # use rayon_core as rayon; +/// // point start +/// rayon::scope_fifo(|s| { +/// s.spawn_fifo(|s| { // task s.1 +/// s.spawn_fifo(|s| { // task s.1.1 +/// rayon::scope_fifo(|t| { +/// t.spawn_fifo(|_| ()); // task t.1 +/// t.spawn_fifo(|_| ()); // task t.2 +/// }); +/// }); +/// }); +/// s.spawn_fifo(|s| { // task s.2 +/// }); +/// // point mid +/// }); +/// // point end +/// ``` +/// +/// The various tasks that are run will execute roughly like so: +/// +/// ```notrust +/// | (start) +/// | +/// | (FIFO scope `s` created) +/// +--------------------+ (task s.1) +/// +-------+ (task s.2) | +/// | | +---+ (task s.1.1) +/// | | | | +/// | | | | (FIFO scope `t` created) +/// | | | +----------------+ (task t.1) +/// | | | +---+ (task t.2) | +/// | (mid) | | | | | +/// : | | + <-+------------+ (scope `t` ends) +/// : | | | +/// |<------+------------+---+ (scope `s` ends) +/// | +/// | (end) +/// ``` +/// +/// Under `scope_fifo()`, the spawns are prioritized in a FIFO order on +/// the thread from which they were spawned, as opposed to `scope()`'s +/// LIFO. So in this example, we can expect `s.1` to execute before +/// `s.2`, and `t.1` before `t.2`. Other threads also steal tasks in +/// FIFO order, as usual. Overall, this has roughly the same order as +/// the now-deprecated [`breadth_first`] option, except the effect is +/// isolated to a particular scope. If spawns are intermingled from any +/// combination of `scope()` and `scope_fifo()`, or from different +/// threads, their order is only specified with respect to spawns in the +/// same scope and thread. +/// +/// For more details on this design, see Rayon [RFC #1]. +/// +/// [`breadth_first`]: struct.ThreadPoolBuilder.html#method.breadth_first +/// [RFC #1]: https://github.com/rayon-rs/rfcs/blob/master/accepted/rfc0001-scope-scheduling.md +/// +/// # Panics +/// +/// If a panic occurs, either in the closure given to `scope_fifo()` or +/// in any of the spawned jobs, that panic will be propagated and the +/// call to `scope_fifo()` will panic. If multiple panics occurs, it is +/// non-deterministic which of their panic values will propagate. +/// Regardless, once a task is spawned using `scope.spawn_fifo()`, it +/// will execute, even if the spawning task should later panic. +/// `scope_fifo()` returns once all spawned jobs have completed, and any +/// panics are propagated at that point. +pub fn scope_fifo<'scope, OP, R>(op: OP) -> R +where + OP: FnOnce(&ScopeFifo<'scope>) -> R + Send, + R: Send, +{ + in_worker(|owner_thread, _| { + let scope = ScopeFifo::<'scope>::new(Some(owner_thread), None); + scope.base.complete(Some(owner_thread), || op(&scope)) + }) +} + +/// Creates a "fork-join" scope `s` and invokes the closure with a +/// reference to `s`. This closure can then spawn asynchronous tasks +/// into `s`. Those tasks may run asynchronously with respect to the +/// closure; they may themselves spawn additional tasks into `s`. When +/// the closure returns, it will block until all tasks that have been +/// spawned into `s` complete. +/// +/// This is just like `scope()` except the closure runs on the same thread +/// that calls `in_place_scope()`. Only work that it spawns runs in the +/// thread pool. +/// +/// # Panics +/// +/// If a panic occurs, either in the closure given to `in_place_scope()` or in +/// any of the spawned jobs, that panic will be propagated and the +/// call to `in_place_scope()` will panic. If multiple panics occurs, it is +/// non-deterministic which of their panic values will propagate. +/// Regardless, once a task is spawned using `scope.spawn()`, it will +/// execute, even if the spawning task should later panic. `in_place_scope()` +/// returns once all spawned jobs have completed, and any panics are +/// propagated at that point. +pub fn in_place_scope<'scope, OP, R>(op: OP) -> R +where + OP: FnOnce(&Scope<'scope>) -> R, +{ + do_in_place_scope(None, op) +} + +pub(crate) fn do_in_place_scope<'scope, OP, R>(registry: Option<&Arc>, op: OP) -> R +where + OP: FnOnce(&Scope<'scope>) -> R, +{ + let thread = unsafe { WorkerThread::current().as_ref() }; + let scope = Scope::<'scope>::new(thread, registry); + scope.base.complete(thread, || op(&scope)) +} + +/// Creates a "fork-join" scope `s` with FIFO order, and invokes the +/// closure with a reference to `s`. This closure can then spawn +/// asynchronous tasks into `s`. Those tasks may run asynchronously with +/// respect to the closure; they may themselves spawn additional tasks +/// into `s`. When the closure returns, it will block until all tasks +/// that have been spawned into `s` complete. +/// +/// This is just like `scope_fifo()` except the closure runs on the same thread +/// that calls `in_place_scope_fifo()`. Only work that it spawns runs in the +/// thread pool. +/// +/// # Panics +/// +/// If a panic occurs, either in the closure given to `in_place_scope_fifo()` or in +/// any of the spawned jobs, that panic will be propagated and the +/// call to `in_place_scope_fifo()` will panic. If multiple panics occurs, it is +/// non-deterministic which of their panic values will propagate. +/// Regardless, once a task is spawned using `scope.spawn_fifo()`, it will +/// execute, even if the spawning task should later panic. `in_place_scope_fifo()` +/// returns once all spawned jobs have completed, and any panics are +/// propagated at that point. +pub fn in_place_scope_fifo<'scope, OP, R>(op: OP) -> R +where + OP: FnOnce(&ScopeFifo<'scope>) -> R, +{ + do_in_place_scope_fifo(None, op) +} + +pub(crate) fn do_in_place_scope_fifo<'scope, OP, R>(registry: Option<&Arc>, op: OP) -> R +where + OP: FnOnce(&ScopeFifo<'scope>) -> R, +{ + let thread = unsafe { WorkerThread::current().as_ref() }; + let scope = ScopeFifo::<'scope>::new(thread, registry); + scope.base.complete(thread, || op(&scope)) +} + +impl<'scope> Scope<'scope> { + fn new(owner: Option<&WorkerThread>, registry: Option<&Arc>) -> Self { + let base = ScopeBase::new(owner, registry); + Scope { base } + } + + /// Spawns a job into the fork-join scope `self`. This job will + /// execute sometime before the fork-join scope completes. The + /// job is specified as a closure, and this closure receives its + /// own reference to the scope `self` as argument. This can be + /// used to inject new jobs into `self`. + /// + /// # Returns + /// + /// Nothing. The spawned closures cannot pass back values to the + /// caller directly, though they can write to local variables on + /// the stack (if those variables outlive the scope) or + /// communicate through shared channels. + /// + /// (The intention is to eventually integrate with Rust futures to + /// support spawns of functions that compute a value.) + /// + /// # Examples + /// + /// ```rust + /// # use rayon_core as rayon; + /// let mut value_a = None; + /// let mut value_b = None; + /// let mut value_c = None; + /// rayon::scope(|s| { + /// s.spawn(|s1| { + /// // ^ this is the same scope as `s`; this handle `s1` + /// // is intended for use by the spawned task, + /// // since scope handles cannot cross thread boundaries. + /// + /// value_a = Some(22); + /// + /// // the scope `s` will not end until all these tasks are done + /// s1.spawn(|_| { + /// value_b = Some(44); + /// }); + /// }); + /// + /// s.spawn(|_| { + /// value_c = Some(66); + /// }); + /// }); + /// assert_eq!(value_a, Some(22)); + /// assert_eq!(value_b, Some(44)); + /// assert_eq!(value_c, Some(66)); + /// ``` + /// + /// # See also + /// + /// The [`scope` function] has more extensive documentation about + /// task spawning. + /// + /// [`scope` function]: fn.scope.html + pub fn spawn(&self, body: BODY) + where + BODY: FnOnce(&Scope<'scope>) + Send + 'scope, + { + let scope_ptr = ScopePtr(self); + let job = HeapJob::new(self.base.tlv, move || unsafe { + // SAFETY: this job will execute before the scope ends. + let scope = scope_ptr.as_ref(); + ScopeBase::execute_job(&scope.base, move || body(scope)) + }); + let job_ref = self.base.heap_job_ref(job); + + // Since `Scope` implements `Sync`, we can't be sure that we're still in a + // thread of this pool, so we can't just push to the local worker thread. + // Also, this might be an in-place scope. + self.base.registry.inject_or_push(job_ref); + } + + /// Spawns a job into every thread of the fork-join scope `self`. This job will + /// execute on each thread sometime before the fork-join scope completes. The + /// job is specified as a closure, and this closure receives its own reference + /// to the scope `self` as argument, as well as a `BroadcastContext`. + pub fn spawn_broadcast(&self, body: BODY) + where + BODY: Fn(&Scope<'scope>, BroadcastContext<'_>) + Send + Sync + 'scope, + { + let scope_ptr = ScopePtr(self); + let job = ArcJob::new(move || unsafe { + // SAFETY: this job will execute before the scope ends. + let scope = scope_ptr.as_ref(); + let body = &body; + let func = move || BroadcastContext::with(move |ctx| body(scope, ctx)); + ScopeBase::execute_job(&scope.base, func) + }); + self.base.inject_broadcast(job) + } +} + +impl<'scope> ScopeFifo<'scope> { + fn new(owner: Option<&WorkerThread>, registry: Option<&Arc>) -> Self { + let base = ScopeBase::new(owner, registry); + let num_threads = base.registry.num_threads(); + let fifos = (0..num_threads).map(|_| JobFifo::new()).collect(); + ScopeFifo { base, fifos } + } + + /// Spawns a job into the fork-join scope `self`. This job will + /// execute sometime before the fork-join scope completes. The + /// job is specified as a closure, and this closure receives its + /// own reference to the scope `self` as argument. This can be + /// used to inject new jobs into `self`. + /// + /// # See also + /// + /// This method is akin to [`Scope::spawn()`], but with a FIFO + /// priority. The [`scope_fifo` function] has more details about + /// this distinction. + /// + /// [`Scope::spawn()`]: struct.Scope.html#method.spawn + /// [`scope_fifo` function]: fn.scope_fifo.html + pub fn spawn_fifo(&self, body: BODY) + where + BODY: FnOnce(&ScopeFifo<'scope>) + Send + 'scope, + { + let scope_ptr = ScopePtr(self); + let job = HeapJob::new(self.base.tlv, move || unsafe { + // SAFETY: this job will execute before the scope ends. + let scope = scope_ptr.as_ref(); + ScopeBase::execute_job(&scope.base, move || body(scope)) + }); + let job_ref = self.base.heap_job_ref(job); + + // If we're in the pool, use our scope's private fifo for this thread to execute + // in a locally-FIFO order. Otherwise, just use the pool's global injector. + match self.base.registry.current_thread() { + Some(worker) => { + let fifo = &self.fifos[worker.index()]; + // SAFETY: this job will execute before the scope ends. + unsafe { worker.push(fifo.push(job_ref)) }; + } + None => self.base.registry.inject(job_ref), + } + } + + /// Spawns a job into every thread of the fork-join scope `self`. This job will + /// execute on each thread sometime before the fork-join scope completes. The + /// job is specified as a closure, and this closure receives its own reference + /// to the scope `self` as argument, as well as a `BroadcastContext`. + pub fn spawn_broadcast(&self, body: BODY) + where + BODY: Fn(&ScopeFifo<'scope>, BroadcastContext<'_>) + Send + Sync + 'scope, + { + let scope_ptr = ScopePtr(self); + let job = ArcJob::new(move || unsafe { + // SAFETY: this job will execute before the scope ends. + let scope = scope_ptr.as_ref(); + let body = &body; + let func = move || BroadcastContext::with(move |ctx| body(scope, ctx)); + ScopeBase::execute_job(&scope.base, func) + }); + self.base.inject_broadcast(job) + } +} + +impl<'scope> ScopeBase<'scope> { + /// Creates the base of a new scope for the given registry + fn new(owner: Option<&WorkerThread>, registry: Option<&Arc>) -> Self { + let registry = registry.unwrap_or_else(|| match owner { + Some(owner) => owner.registry(), + None => global_registry(), + }); + + ScopeBase { + registry: Arc::clone(registry), + panic: AtomicPtr::new(ptr::null_mut()), + job_completed_latch: CountLatch::new(owner), + marker: PhantomData, + tlv: tlv::get(), + } + } + + fn heap_job_ref(&self, job: Box>) -> JobRef + where + FUNC: FnOnce() + Send + 'scope, + { + unsafe { + self.job_completed_latch.increment(); + job.into_job_ref() + } + } + + fn inject_broadcast(&self, job: Arc>) + where + FUNC: Fn() + Send + Sync + 'scope, + { + let n_threads = self.registry.num_threads(); + let job_refs = (0..n_threads).map(|_| unsafe { + self.job_completed_latch.increment(); + ArcJob::as_job_ref(&job) + }); + + self.registry.inject_broadcast(job_refs); + } + + /// Executes `func` as a job, either aborting or executing as + /// appropriate. + fn complete(&self, owner: Option<&WorkerThread>, func: FUNC) -> R + where + FUNC: FnOnce() -> R, + { + let result = unsafe { Self::execute_job_closure(self, func) }; + self.job_completed_latch.wait(owner); + + // Restore the TLV if we ran some jobs while waiting + tlv::set(self.tlv); + + self.maybe_propagate_panic(); + result.unwrap() // only None if `op` panicked, and that would have been propagated + } + + /// Executes `func` as a job, either aborting or executing as + /// appropriate. + unsafe fn execute_job(this: *const Self, func: FUNC) + where + FUNC: FnOnce(), + { + let _: Option<()> = Self::execute_job_closure(this, func); + } + + /// Executes `func` as a job in scope. Adjusts the "job completed" + /// counters and also catches any panic and stores it into + /// `scope`. + unsafe fn execute_job_closure(this: *const Self, func: FUNC) -> Option + where + FUNC: FnOnce() -> R, + { + let result = match unwind::halt_unwinding(func) { + Ok(r) => Some(r), + Err(err) => { + (*this).job_panicked(err); + None + } + }; + Latch::set(&(*this).job_completed_latch); + result + } + + fn job_panicked(&self, err: Box) { + // capture the first error we see, free the rest + if self.panic.load(Ordering::Relaxed).is_null() { + let nil = ptr::null_mut(); + let mut err = ManuallyDrop::new(Box::new(err)); // box up the fat ptr + let err_ptr: *mut Box = &mut **err; + if self + .panic + .compare_exchange(nil, err_ptr, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + // ownership now transferred into self.panic + } else { + // another panic raced in ahead of us, so drop ours + let _: Box> = ManuallyDrop::into_inner(err); + } + } + } + + fn maybe_propagate_panic(&self) { + // propagate panic, if any occurred; at this point, all + // outstanding jobs have completed, so we can use a relaxed + // ordering: + let panic = self.panic.swap(ptr::null_mut(), Ordering::Relaxed); + if !panic.is_null() { + let value = unsafe { Box::from_raw(panic) }; + + // Restore the TLV if we ran some jobs while waiting + tlv::set(self.tlv); + + unwind::resume_unwinding(*value); + } + } +} + +impl<'scope> fmt::Debug for Scope<'scope> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Scope") + .field("pool_id", &self.base.registry.id()) + .field("panic", &self.base.panic) + .field("job_completed_latch", &self.base.job_completed_latch) + .finish() + } +} + +impl<'scope> fmt::Debug for ScopeFifo<'scope> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("ScopeFifo") + .field("num_fifos", &self.fifos.len()) + .field("pool_id", &self.base.registry.id()) + .field("panic", &self.base.panic) + .field("job_completed_latch", &self.base.job_completed_latch) + .finish() + } +} + +/// Used to capture a scope `&Self` pointer in jobs, without faking a lifetime. +/// +/// Unsafe code is still required to dereference the pointer, but that's fine in +/// scope jobs that are guaranteed to execute before the scope ends. +struct ScopePtr(*const T); + +// SAFETY: !Send for raw pointers is not for safety, just as a lint +unsafe impl Send for ScopePtr {} + +// SAFETY: !Sync for raw pointers is not for safety, just as a lint +unsafe impl Sync for ScopePtr {} + +impl ScopePtr { + // Helper to avoid disjoint captures of `scope_ptr.0` + unsafe fn as_ref(&self) -> &T { + &*self.0 + } +} diff --git a/compiler/rustc_thread_pool/src/scope/test.rs b/compiler/rustc_thread_pool/src/scope/test.rs new file mode 100644 index 0000000000000..4505ba7c4fba5 --- /dev/null +++ b/compiler/rustc_thread_pool/src/scope/test.rs @@ -0,0 +1,622 @@ +use crate::unwind; +use crate::ThreadPoolBuilder; +use crate::{scope, scope_fifo, Scope, ScopeFifo}; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; +use std::iter::once; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Barrier, Mutex}; +use std::vec; + +#[test] +fn scope_empty() { + scope(|_| {}); +} + +#[test] +fn scope_result() { + let x = scope(|_| 22); + assert_eq!(x, 22); +} + +#[test] +fn scope_two() { + let counter = &AtomicUsize::new(0); + scope(|s| { + s.spawn(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + }); + s.spawn(move |_| { + counter.fetch_add(10, Ordering::SeqCst); + }); + }); + + let v = counter.load(Ordering::SeqCst); + assert_eq!(v, 11); +} + +#[test] +fn scope_divide_and_conquer() { + let counter_p = &AtomicUsize::new(0); + scope(|s| s.spawn(move |s| divide_and_conquer(s, counter_p, 1024))); + + let counter_s = &AtomicUsize::new(0); + divide_and_conquer_seq(counter_s, 1024); + + let p = counter_p.load(Ordering::SeqCst); + let s = counter_s.load(Ordering::SeqCst); + assert_eq!(p, s); +} + +fn divide_and_conquer<'scope>(scope: &Scope<'scope>, counter: &'scope AtomicUsize, size: usize) { + if size > 1 { + scope.spawn(move |scope| divide_and_conquer(scope, counter, size / 2)); + scope.spawn(move |scope| divide_and_conquer(scope, counter, size / 2)); + } else { + // count the leaves + counter.fetch_add(1, Ordering::SeqCst); + } +} + +fn divide_and_conquer_seq(counter: &AtomicUsize, size: usize) { + if size > 1 { + divide_and_conquer_seq(counter, size / 2); + divide_and_conquer_seq(counter, size / 2); + } else { + // count the leaves + counter.fetch_add(1, Ordering::SeqCst); + } +} + +struct Tree { + value: T, + children: Vec>, +} + +impl Tree { + fn iter(&self) -> vec::IntoIter<&T> { + once(&self.value) + .chain(self.children.iter().flat_map(Tree::iter)) + .collect::>() // seems like it shouldn't be needed... but prevents overflow + .into_iter() + } + + fn update(&mut self, op: OP) + where + OP: Fn(&mut T) + Sync, + T: Send, + { + scope(|s| self.update_in_scope(&op, s)); + } + + fn update_in_scope<'scope, OP>(&'scope mut self, op: &'scope OP, scope: &Scope<'scope>) + where + OP: Fn(&mut T) + Sync, + { + let Tree { + ref mut value, + ref mut children, + } = *self; + scope.spawn(move |scope| { + for child in children { + scope.spawn(move |scope| child.update_in_scope(op, scope)); + } + }); + + op(value); + } +} + +fn random_tree(depth: usize) -> Tree { + assert!(depth > 0); + let mut seed = ::Seed::default(); + (0..).zip(seed.as_mut()).for_each(|(i, x)| *x = i); + let mut rng = XorShiftRng::from_seed(seed); + random_tree1(depth, &mut rng) +} + +fn random_tree1(depth: usize, rng: &mut XorShiftRng) -> Tree { + let children = if depth == 0 { + vec![] + } else { + (0..rng.random_range(0..4)) // somewhere between 0 and 3 children at each level + .map(|_| random_tree1(depth - 1, rng)) + .collect() + }; + + Tree { + value: rng.random_range(0..1_000_000), + children, + } +} + +#[test] +fn update_tree() { + let mut tree: Tree = random_tree(10); + let values: Vec = tree.iter().cloned().collect(); + tree.update(|v| *v += 1); + let new_values: Vec = tree.iter().cloned().collect(); + assert_eq!(values.len(), new_values.len()); + for (&i, &j) in values.iter().zip(&new_values) { + assert_eq!(i + 1, j); + } +} + +/// Check that if you have a chain of scoped tasks where T0 spawns T1 +/// spawns T2 and so forth down to Tn, the stack space should not grow +/// linearly with N. We test this by some unsafe hackery and +/// permitting an approx 10% change with a 10x input change. +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn linear_stack_growth() { + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let mut max_diff = Mutex::new(0); + let bottom_of_stack = 0; + scope(|s| the_final_countdown(s, &bottom_of_stack, &max_diff, 5)); + let diff_when_5 = *max_diff.get_mut().unwrap() as f64; + + scope(|s| the_final_countdown(s, &bottom_of_stack, &max_diff, 500)); + let diff_when_500 = *max_diff.get_mut().unwrap() as f64; + + let ratio = diff_when_5 / diff_when_500; + assert!( + ratio > 0.9 && ratio < 1.1, + "stack usage ratio out of bounds: {}", + ratio + ); + }); +} + +fn the_final_countdown<'scope>( + s: &Scope<'scope>, + bottom_of_stack: &'scope i32, + max: &'scope Mutex, + n: usize, +) { + let top_of_stack = 0; + let p = bottom_of_stack as *const i32 as usize; + let q = &top_of_stack as *const i32 as usize; + let diff = if p > q { p - q } else { q - p }; + + let mut data = max.lock().unwrap(); + *data = Ord::max(diff, *data); + + if n > 0 { + s.spawn(move |s| the_final_countdown(s, bottom_of_stack, max, n - 1)); + } +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_scope() { + scope(|_| panic!("Hello, world!")); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_spawn() { + scope(|s| s.spawn(|_| panic!("Hello, world!"))); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_nested_spawn() { + scope(|s| s.spawn(|s| s.spawn(|s| s.spawn(|_| panic!("Hello, world!"))))); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_nested_scope_spawn() { + scope(|s| s.spawn(|_| scope(|s| s.spawn(|_| panic!("Hello, world!"))))); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_propagate_still_execute_1() { + let mut x = false; + let result = unwind::halt_unwinding(|| { + scope(|s| { + s.spawn(|_| panic!("Hello, world!")); // job A + s.spawn(|_| x = true); // job B, should still execute even though A panics + }); + }); + match result { + Ok(_) => panic!("failed to propagate panic"), + Err(_) => assert!(x, "job b failed to execute"), + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_propagate_still_execute_2() { + let mut x = false; + let result = unwind::halt_unwinding(|| { + scope(|s| { + s.spawn(|_| x = true); // job B, should still execute even though A panics + s.spawn(|_| panic!("Hello, world!")); // job A + }); + }); + match result { + Ok(_) => panic!("failed to propagate panic"), + Err(_) => assert!(x, "job b failed to execute"), + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_propagate_still_execute_3() { + let mut x = false; + let result = unwind::halt_unwinding(|| { + scope(|s| { + s.spawn(|_| x = true); // spawned job should still execute despite later panic + panic!("Hello, world!"); + }); + }); + match result { + Ok(_) => panic!("failed to propagate panic"), + Err(_) => assert!(x, "panic after spawn, spawn failed to execute"), + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_propagate_still_execute_4() { + let mut x = false; + let result = unwind::halt_unwinding(|| { + scope(|s| { + s.spawn(|_| panic!("Hello, world!")); + x = true; + }); + }); + match result { + Ok(_) => panic!("failed to propagate panic"), + Err(_) => assert!(x, "panic in spawn tainted scope"), + } +} + +macro_rules! test_order { + ($scope:ident => $spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let vec = Mutex::new(vec![]); + $scope(|scope| { + let vec = &vec; + for i in 0..10 { + scope.$spawn(move |scope| { + for j in 0..10 { + scope.$spawn(move |_| { + vec.lock().unwrap().push(i * 10 + j); + }); + } + }); + } + }); + vec.into_inner().unwrap() + }) + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn lifo_order() { + // In the absence of stealing, `scope()` runs its `spawn()` jobs in LIFO order. + let vec = test_order!(scope => spawn); + let expected: Vec = (0..100).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn fifo_order() { + // In the absence of stealing, `scope_fifo()` runs its `spawn_fifo()` jobs in FIFO order. + let vec = test_order!(scope_fifo => spawn_fifo); + let expected: Vec = (0..100).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +macro_rules! test_nested_order { + ($outer_scope:ident => $outer_spawn:ident, + $inner_scope:ident => $inner_spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let vec = Mutex::new(vec![]); + $outer_scope(|scope| { + let vec = &vec; + for i in 0..10 { + scope.$outer_spawn(move |_| { + $inner_scope(|scope| { + for j in 0..10 { + scope.$inner_spawn(move |_| { + vec.lock().unwrap().push(i * 10 + j); + }); + } + }); + }); + } + }); + vec.into_inner().unwrap() + }) + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_lifo_order() { + // In the absence of stealing, `scope()` runs its `spawn()` jobs in LIFO order. + let vec = test_nested_order!(scope => spawn, scope => spawn); + let expected: Vec = (0..100).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_fifo_order() { + // In the absence of stealing, `scope_fifo()` runs its `spawn_fifo()` jobs in FIFO order. + let vec = test_nested_order!(scope_fifo => spawn_fifo, scope_fifo => spawn_fifo); + let expected: Vec = (0..100).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_lifo_fifo_order() { + // LIFO on the outside, FIFO on the inside + let vec = test_nested_order!(scope => spawn, scope_fifo => spawn_fifo); + let expected: Vec = (0..10) + .rev() + .flat_map(|i| (0..10).map(move |j| i * 10 + j)) + .collect(); + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_fifo_lifo_order() { + // FIFO on the outside, LIFO on the inside + let vec = test_nested_order!(scope_fifo => spawn_fifo, scope => spawn); + let expected: Vec = (0..10) + .flat_map(|i| (0..10).rev().map(move |j| i * 10 + j)) + .collect(); + assert_eq!(vec, expected); +} + +macro_rules! spawn_push { + ($scope:ident . $spawn:ident, $vec:ident, $i:expr) => {{ + $scope.$spawn(move |_| $vec.lock().unwrap().push($i)); + }}; +} + +/// Test spawns pushing a series of numbers, interleaved +/// such that negative values are using an inner scope. +macro_rules! test_mixed_order { + ($outer_scope:ident => $outer_spawn:ident, + $inner_scope:ident => $inner_spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let vec = Mutex::new(vec![]); + $outer_scope(|outer_scope| { + let vec = &vec; + spawn_push!(outer_scope.$outer_spawn, vec, 0); + $inner_scope(|inner_scope| { + spawn_push!(inner_scope.$inner_spawn, vec, -1); + spawn_push!(outer_scope.$outer_spawn, vec, 1); + spawn_push!(inner_scope.$inner_spawn, vec, -2); + spawn_push!(outer_scope.$outer_spawn, vec, 2); + spawn_push!(inner_scope.$inner_spawn, vec, -3); + }); + spawn_push!(outer_scope.$outer_spawn, vec, 3); + }); + vec.into_inner().unwrap() + }) + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_lifo_order() { + // NB: the end of the inner scope makes us execute some of the outer scope + // before they've all been spawned, so they're not perfectly LIFO. + let vec = test_mixed_order!(scope => spawn, scope => spawn); + let expected = vec![-3, 2, -2, 1, -1, 3, 0]; + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_fifo_order() { + let vec = test_mixed_order!(scope_fifo => spawn_fifo, scope_fifo => spawn_fifo); + let expected = vec![-1, 0, -2, 1, -3, 2, 3]; + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_lifo_fifo_order() { + // NB: the end of the inner scope makes us execute some of the outer scope + // before they've all been spawned, so they're not perfectly LIFO. + let vec = test_mixed_order!(scope => spawn, scope_fifo => spawn_fifo); + let expected = vec![-1, 2, -2, 1, -3, 3, 0]; + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_fifo_lifo_order() { + let vec = test_mixed_order!(scope_fifo => spawn_fifo, scope => spawn); + let expected = vec![-3, 0, -2, 1, -1, 2, 3]; + assert_eq!(vec, expected); +} + +#[test] +fn static_scope() { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + let mut range = 0..100; + let sum = range.clone().sum(); + let iter = &mut range; + + COUNTER.store(0, Ordering::Relaxed); + scope(|s: &Scope<'static>| { + // While we're allowed the locally borrowed iterator, + // the spawns must be static. + for i in iter { + s.spawn(move |_| { + COUNTER.fetch_add(i, Ordering::Relaxed); + }); + } + }); + + assert_eq!(COUNTER.load(Ordering::Relaxed), sum); +} + +#[test] +fn static_scope_fifo() { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + let mut range = 0..100; + let sum = range.clone().sum(); + let iter = &mut range; + + COUNTER.store(0, Ordering::Relaxed); + scope_fifo(|s: &ScopeFifo<'static>| { + // While we're allowed the locally borrowed iterator, + // the spawns must be static. + for i in iter { + s.spawn_fifo(move |_| { + COUNTER.fetch_add(i, Ordering::Relaxed); + }); + } + }); + + assert_eq!(COUNTER.load(Ordering::Relaxed), sum); +} + +#[test] +fn mixed_lifetime_scope() { + fn increment<'slice, 'counter>(counters: &'slice [&'counter AtomicUsize]) { + scope(move |s: &Scope<'counter>| { + // We can borrow 'slice here, but the spawns can only borrow 'counter. + for &c in counters { + s.spawn(move |_| { + c.fetch_add(1, Ordering::Relaxed); + }); + } + }); + } + + let counter = AtomicUsize::new(0); + increment(&[&counter; 100]); + assert_eq!(counter.into_inner(), 100); +} + +#[test] +fn mixed_lifetime_scope_fifo() { + fn increment<'slice, 'counter>(counters: &'slice [&'counter AtomicUsize]) { + scope_fifo(move |s: &ScopeFifo<'counter>| { + // We can borrow 'slice here, but the spawns can only borrow 'counter. + for &c in counters { + s.spawn_fifo(move |_| { + c.fetch_add(1, Ordering::Relaxed); + }); + } + }); + } + + let counter = AtomicUsize::new(0); + increment(&[&counter; 100]); + assert_eq!(counter.into_inner(), 100); +} + +#[test] +fn scope_spawn_broadcast() { + let sum = AtomicUsize::new(0); + let n = scope(|s| { + s.spawn_broadcast(|_, ctx| { + sum.fetch_add(ctx.index(), Ordering::Relaxed); + }); + crate::current_num_threads() + }); + assert_eq!(sum.into_inner(), n * (n - 1) / 2); +} + +#[test] +fn scope_fifo_spawn_broadcast() { + let sum = AtomicUsize::new(0); + let n = scope_fifo(|s| { + s.spawn_broadcast(|_, ctx| { + sum.fetch_add(ctx.index(), Ordering::Relaxed); + }); + crate::current_num_threads() + }); + assert_eq!(sum.into_inner(), n * (n - 1) / 2); +} + +#[test] +fn scope_spawn_broadcast_nested() { + let sum = AtomicUsize::new(0); + let n = scope(|s| { + s.spawn_broadcast(|s, _| { + s.spawn_broadcast(|_, ctx| { + sum.fetch_add(ctx.index(), Ordering::Relaxed); + }); + }); + crate::current_num_threads() + }); + assert_eq!(sum.into_inner(), n * n * (n - 1) / 2); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_spawn_broadcast_barrier() { + let barrier = Barrier::new(8); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool.in_place_scope(|s| { + s.spawn_broadcast(|_, _| { + barrier.wait(); + }); + barrier.wait(); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_spawn_broadcast_panic_one() { + let count = AtomicUsize::new(0); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let result = crate::unwind::halt_unwinding(|| { + pool.scope(|s| { + s.spawn_broadcast(|_, ctx| { + count.fetch_add(1, Ordering::Relaxed); + if ctx.index() == 3 { + panic!("Hello, world!"); + } + }); + }); + }); + assert_eq!(count.into_inner(), 7); + assert!(result.is_err(), "broadcast panic should propagate!"); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_spawn_broadcast_panic_many() { + let count = AtomicUsize::new(0); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let result = crate::unwind::halt_unwinding(|| { + pool.scope(|s| { + s.spawn_broadcast(|_, ctx| { + count.fetch_add(1, Ordering::Relaxed); + if ctx.index() % 2 == 0 { + panic!("Hello, world!"); + } + }); + }); + }); + assert_eq!(count.into_inner(), 7); + assert!(result.is_err(), "broadcast panic should propagate!"); +} diff --git a/compiler/rustc_thread_pool/src/sleep/README.md b/compiler/rustc_thread_pool/src/sleep/README.md new file mode 100644 index 0000000000000..e79efd15ca9b0 --- /dev/null +++ b/compiler/rustc_thread_pool/src/sleep/README.md @@ -0,0 +1,252 @@ +# Introduction: the sleep module + +The code in this module governs when worker threads should go to +sleep. The system used in this code was introduced in [Rayon RFC #5]. +There is also a [video walkthrough] available. Both of those may be +valuable resources to understanding the code, though naturally they +will also grow stale over time. The comments in this file are +extracted from the RFC and meant to be kept up to date. + +[Rayon RFC #5]: https://github.com/rayon-rs/rfcs/pull/5 +[video walkthrough]: https://youtu.be/HvmQsE5M4cY + +# The `Sleep` struct + +The `Sleep` struct is embedded into each registry. It performs several functions: + +* It tracks when workers are awake or asleep. +* It decides how long a worker should look for work before it goes to sleep, + via a callback that is invoked periodically from the worker's search loop. +* It is notified when latches are set, jobs are published, or other + events occur, and it will go and wake the appropriate threads if + they are sleeping. + +# Thread states + +There are three main thread states: + +* An **active** thread is one that is actively executing a job. +* An **idle** thread is one that is searching for work to do. It will be + trying to steal work or pop work from the global injector queue. +* A **sleeping** thread is one that is blocked on a condition variable, + waiting to be awoken. + +We sometimes refer to the final two states collectively as **inactive**. +Threads begin as idle but transition to idle and finally sleeping when +they're unable to find work to do. + +## Sleepy threads + +There is one other special state worth mentioning. During the idle state, +threads can get **sleepy**. A sleepy thread is still idle, in that it is still +searching for work, but it is *about* to go to sleep after it does one more +search (or some other number, potentially). When a thread enters the sleepy +state, it signals (via the **jobs event counter**, described below) that it is +about to go to sleep. If new work is published, this will lead to the counter +being adjusted. When the thread actually goes to sleep, it will (hopefully, but +not guaranteed) see that the counter has changed and elect not to sleep, but +instead to search again. See the section on the **jobs event counter** for more +details. + +# The counters + +One of the key structs in the sleep module is `AtomicCounters`, found in +`counters.rs`. It packs three counters into one atomically managed value: + +* Two **thread counters**, which track the number of threads in a particular state. +* The **jobs event counter**, which is used to signal when new work is available. + It (sort of) tracks the number of jobs posted, but not quite, and it can rollover. + +## Thread counters + +There are two thread counters, one that tracks **inactive** threads and one that +tracks **sleeping** threads. From this, one can deduce the number of threads +that are idle by subtracting sleeping threads from inactive threads. We track +the counters in this way because it permits simpler atomic operations. One can +increment the number of sleeping threads (and thus decrease the number of idle +threads) simply by doing one atomic increment, for example. Similarly, one can +decrease the number of sleeping threads (and increase the number of idle +threads) through one atomic decrement. + +These counters are adjusted as follows: + +* When a thread enters the idle state: increment the inactive thread counter. +* When a thread enters the sleeping state: increment the sleeping thread counter. +* When a thread awakens a sleeping thread: decrement the sleeping thread counter. + * Subtle point: the thread that *awakens* the sleeping thread decrements the + counter, not the thread that is *sleeping*. This is because there is a delay + between signaling a thread to wake and the thread actually waking: + decrementing the counter when awakening the thread means that other threads + that may be posting work will see the up-to-date value that much faster. +* When a thread finds work, exiting the idle state: decrement the inactive + thread counter. + +## Jobs event counter + +The final counter is the **jobs event counter**. The role of this counter is to +help sleepy threads detect when new work is posted in a lightweight fashion. In +its simplest form, we would simply have a counter that gets incremented each +time a new job is posted. This way, when a thread gets sleepy, it could read the +counter, and then compare to see if the value has changed before it actually +goes to sleep. But this [turns out to be too expensive] in practice, so we use a +somewhat more complex scheme. + +[turns out to be too expensive]: https://github.com/rayon-rs/rayon/pull/746#issuecomment-624802747 + +The idea is that the counter toggles between two states, depending on whether +its value is even or odd (or, equivalently, on the value of its low bit): + +* Even -- If the low bit is zero, then it means that there has been no new work + since the last thread got sleepy. +* Odd -- If the low bit is one, then it means that new work was posted since + the last thread got sleepy. + +### New work is posted + +When new work is posted, we check the value of the counter: if it is even, +then we increment it by one, so that it becomes odd. + +### Worker thread gets sleepy + +When a worker thread gets sleepy, it will read the value of the counter. If the +counter is odd, it will increment the counter so that it is even. Either way, it +remembers the final value of the counter. The final value will be used later, +when the thread is going to sleep. If at that time the counter has not changed, +then we can assume no new jobs have been posted (though note the remote +possibility of rollover, discussed in detail below). + +# Protocol for a worker thread to post work + +The full protocol for a thread to post work is as follows + +* If the work is posted into the injection queue, then execute a seq-cst fence (see below). +* Load the counters, incrementing the JEC if it is even so that it is odd. +* Check if there are idle threads available to handle this new job. If not, + and there are sleeping threads, then wake one or more threads. + +# Protocol for a worker thread to fall asleep + +The full protocol for a thread to fall asleep is as follows: + +* After completing all its jobs, the worker goes idle and begins to + search for work. As it searches, it counts "rounds". In each round, + it searches all other work threads' queues, plus the 'injector queue' for + work injected from the outside. If work is found in this search, the thread + becomes active again and hence restarts this protocol from the top. +* After a certain number of rounds, the thread "gets sleepy" and executes `get_sleepy` + above, remembering the `final_value` of the JEC. It does one more search for work. +* If no work is found, the thread atomically: + * Checks the JEC to see that it has not changed from `final_value`. + * If it has, then the thread goes back to searching for work. We reset to + just before we got sleepy, so that we will do one more search + before attempting to sleep again (rather than searching for many rounds). + * Increments the number of sleeping threads by 1. +* The thread then executes a seq-cst fence operation (see below). +* The thread then does one final check for injected jobs (see below). If any + are available, it returns to the 'pre-sleepy' state as if the JEC had changed. +* The thread waits to be signaled. Once signaled, it returns to the idle state. + +# The jobs event counter and deadlock + +As described in the section on the JEC, the main concern around going to sleep +is avoiding a race condition wherein: + +* Thread A looks for work, finds none. +* Thread B posts work but sees no sleeping threads. +* Thread A goes to sleep. + +The JEC protocol largely prevents this, but due to rollover, this prevention is +not complete. It is possible -- if unlikely -- that enough activity occurs for +Thread A to observe the same JEC value that it saw when getting sleepy. If the +new work being published came from *inside* the thread-pool, then this race +condition isn't too harmful. It means that we have fewer workers processing the +work then we should, but we won't deadlock. This seems like an acceptable risk +given that this is unlikely in practice. + +However, if the work was posted as an *external* job, that is a problem. In that +case, it's possible that all of our workers could go to sleep, and the external +job would never get processed. To prevent that, the sleeping protocol includes +one final check to see if the injector queue is empty before fully falling +asleep. Note that this final check occurs **after** the number of sleeping +threads has been incremented. We are not concerned therefore with races against +injections that occur after that increment, only before. + +Unfortunately, there is one rather subtle point concerning this final check: +we wish to avoid the possibility that: + +* work is pushed into the injection queue by an outside thread X, +* the sleepy thread S sees the JEC but it has rolled over and is equal +* the sleepy thread S reads the injection queue but does not see the work posted by X. + +This is possible because the C++ memory model typically offers guarantees of the +form "if you see the access A, then you must see those other accesses" -- but it +doesn't guarantee that you will see the access A (i.e., if you think of +processors with independent caches, you may be operating on very out of date +cache state). + +## Using seq-cst fences to prevent deadlock + +To overcome this problem, we have inserted two sequentially consistent fence +operations into the protocols above: + +* One fence occurs after work is posted into the injection queue, but before the + counters are read (including the number of sleeping threads). + * Note that no fence is needed for work posted to internal queues, since it is ok + to overlook work in that case. +* One fence occurs after the number of sleeping threads is incremented, but + before the injection queue is read. + +### Proof sketch + +What follows is a "proof sketch" that the protocol is deadlock free. We model +two relevant bits of memory, the job injector queue J and the atomic counters C. + +Consider the actions of the injecting thread: + +* PushJob: Job is injected, which can be modeled as an atomic write to J with release semantics. +* PushFence: A sequentially consistent fence is executed. +* ReadSleepers: The counters C are read (they may also be incremented, but we just consider the read that comes first). + +Meanwhile, the sleepy thread does the following: + +* IncSleepers: The number of sleeping threads is incremented, which is atomic exchange to C. +* SleepFence: A sequentially consistent fence is executed. +* ReadJob: We look to see if the queue is empty, which is a read of J with acquire semantics. + +Either PushFence or SleepFence must come first: + +* If PushFence comes first, then PushJob must be visible to ReadJob. +* If SleepFence comes first, then IncSleepers is visible to ReadSleepers. + +# Deadlock detection + +This module tracks a number of variables in order to detect deadlocks due to user code blocking. +These variables are stored in the `SleepData` struct which itself is kept behind a mutex. +It contains the following fields: +- `worker_count` - The number of threads in the thread pool. +- `active_threads` - The number of threads in the thread pool which are running + and aren't blocked in user code or sleeping. +- `blocked_threads` - The number of threads which are blocked in user code. + This doesn't include threads blocked by Rayon. + +User code can indicate blocking by calling `mark_blocked` before blocking and +calling `mark_unblocked` before unblocking a thread. +This will adjust `active_threads` and `blocked_threads` accordingly. + +When we tickle the thread pool in `Sleep::tickle_cold`, we set `active_threads` to +`worker_count` - `blocked_threads` since we wake up all Rayon threads, but not thread blocked +by user code. + +A deadlock is detected by checking if `active_threads` is 0 and `blocked_threads` is above 0. +If we ignored `blocked_threads` we would have a deadlock +immediately when creating the thread pool. +We would also deadlock once the thread pool ran out of work. +It is not possible for Rayon itself to deadlock. +Deadlocks can only be caused by user code blocking, so this condition doesn't miss any deadlocks. + +We check for the deadlock condition when +threads fall asleep in `mark_unblocked` and in `Sleep::sleep`. +If there's a deadlock detected we call the user provided deadlock handler while we hold the +lock to `SleepData`. This means the deadlock handler cannot call `mark_blocked` and +`mark_unblocked`. The user is expected to handle the deadlock in some non-Rayon thread. +Once the deadlock handler returns, the thread which called the deadlock handler will go to sleep. diff --git a/compiler/rustc_thread_pool/src/sleep/counters.rs b/compiler/rustc_thread_pool/src/sleep/counters.rs new file mode 100644 index 0000000000000..05941becd1c62 --- /dev/null +++ b/compiler/rustc_thread_pool/src/sleep/counters.rs @@ -0,0 +1,277 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +pub(super) struct AtomicCounters { + /// Packs together a number of counters. The counters are ordered as + /// follows, from least to most significant bits (here, we assuming + /// that [`THREADS_BITS`] is equal to 10): + /// + /// * Bits 0..10: Stores the number of **sleeping threads** + /// * Bits 10..20: Stores the number of **inactive threads** + /// * Bits 20..: Stores the **job event counter** (JEC) + /// + /// This uses 10 bits ([`THREADS_BITS`]) to encode the number of threads. Note + /// that the total number of bits (and hence the number of bits used for the + /// JEC) will depend on whether we are using a 32- or 64-bit architecture. + value: AtomicUsize, +} + +#[derive(Copy, Clone)] +pub(super) struct Counters { + word: usize, +} + +/// A value read from the **Jobs Event Counter**. +/// See the [`README.md`](README.md) for more +/// coverage of how the jobs event counter works. +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] +pub(super) struct JobsEventCounter(usize); + +impl JobsEventCounter { + pub(super) const DUMMY: JobsEventCounter = JobsEventCounter(usize::MAX); + + #[inline] + pub(super) fn as_usize(self) -> usize { + self.0 + } + + /// The JEC "is sleepy" if the last thread to increment it was in the + /// process of becoming sleepy. This is indicated by its value being *even*. + /// When new jobs are posted, they check if the JEC is sleepy, and if so + /// they incremented it. + #[inline] + pub(super) fn is_sleepy(self) -> bool { + (self.as_usize() & 1) == 0 + } + + /// The JEC "is active" if the last thread to increment it was posting new + /// work. This is indicated by its value being *odd*. When threads get + /// sleepy, they will check if the JEC is active, and increment it. + #[inline] + pub(super) fn is_active(self) -> bool { + !self.is_sleepy() + } +} + +/// Number of bits used for the thread counters. +#[cfg(target_pointer_width = "64")] +const THREADS_BITS: usize = 16; + +#[cfg(target_pointer_width = "32")] +const THREADS_BITS: usize = 8; + +/// Bits to shift to select the sleeping threads +/// (used with `select_bits`). +#[allow(clippy::erasing_op)] +const SLEEPING_SHIFT: usize = 0 * THREADS_BITS; + +/// Bits to shift to select the inactive threads +/// (used with `select_bits`). +#[allow(clippy::identity_op)] +const INACTIVE_SHIFT: usize = 1 * THREADS_BITS; + +/// Bits to shift to select the JEC +/// (use JOBS_BITS). +const JEC_SHIFT: usize = 2 * THREADS_BITS; + +/// Max value for the thread counters. +pub(crate) const THREADS_MAX: usize = (1 << THREADS_BITS) - 1; + +/// Constant that can be added to add one sleeping thread. +const ONE_SLEEPING: usize = 1; + +/// Constant that can be added to add one inactive thread. +/// An inactive thread is either idle, sleepy, or sleeping. +const ONE_INACTIVE: usize = 1 << INACTIVE_SHIFT; + +/// Constant that can be added to add one to the JEC. +const ONE_JEC: usize = 1 << JEC_SHIFT; + +impl AtomicCounters { + #[inline] + pub(super) fn new() -> AtomicCounters { + AtomicCounters { + value: AtomicUsize::new(0), + } + } + + /// Load and return the current value of the various counters. + /// This value can then be given to other method which will + /// attempt to update the counters via compare-and-swap. + #[inline] + pub(super) fn load(&self, ordering: Ordering) -> Counters { + Counters::new(self.value.load(ordering)) + } + + #[inline] + fn try_exchange(&self, old_value: Counters, new_value: Counters, ordering: Ordering) -> bool { + self.value + .compare_exchange(old_value.word, new_value.word, ordering, Ordering::Relaxed) + .is_ok() + } + + /// Adds an inactive thread. This cannot fail. + /// + /// This should be invoked when a thread enters its idle loop looking + /// for work. It is decremented when work is found. Note that it is + /// not decremented if the thread transitions from idle to sleepy or sleeping; + /// so the number of inactive threads is always greater-than-or-equal + /// to the number of sleeping threads. + #[inline] + pub(super) fn add_inactive_thread(&self) { + self.value.fetch_add(ONE_INACTIVE, Ordering::SeqCst); + } + + /// Increments the jobs event counter if `increment_when`, when applied to + /// the current value, is true. Used to toggle the JEC from even (sleepy) to + /// odd (active) or vice versa. Returns the final value of the counters, for + /// which `increment_when` is guaranteed to return false. + pub(super) fn increment_jobs_event_counter_if( + &self, + increment_when: impl Fn(JobsEventCounter) -> bool, + ) -> Counters { + loop { + let old_value = self.load(Ordering::SeqCst); + if increment_when(old_value.jobs_counter()) { + let new_value = old_value.increment_jobs_counter(); + if self.try_exchange(old_value, new_value, Ordering::SeqCst) { + return new_value; + } + } else { + return old_value; + } + } + } + + /// Subtracts an inactive thread. This cannot fail. It is invoked + /// when a thread finds work and hence becomes active. It returns the + /// number of sleeping threads to wake up (if any). + /// + /// See `add_inactive_thread`. + #[inline] + pub(super) fn sub_inactive_thread(&self) -> usize { + let old_value = Counters::new(self.value.fetch_sub(ONE_INACTIVE, Ordering::SeqCst)); + debug_assert!( + old_value.inactive_threads() > 0, + "sub_inactive_thread: old_value {:?} has no inactive threads", + old_value, + ); + debug_assert!( + old_value.sleeping_threads() <= old_value.inactive_threads(), + "sub_inactive_thread: old_value {:?} had {} sleeping threads and {} inactive threads", + old_value, + old_value.sleeping_threads(), + old_value.inactive_threads(), + ); + + // Current heuristic: whenever an inactive thread goes away, if + // there are any sleeping threads, wake 'em up. + let sleeping_threads = old_value.sleeping_threads(); + Ord::min(sleeping_threads, 2) + } + + /// Subtracts a sleeping thread. This cannot fail, but it is only + /// safe to do if you you know the number of sleeping threads is + /// non-zero (i.e., because you have just awoken a sleeping + /// thread). + #[inline] + pub(super) fn sub_sleeping_thread(&self) { + let old_value = Counters::new(self.value.fetch_sub(ONE_SLEEPING, Ordering::SeqCst)); + debug_assert!( + old_value.sleeping_threads() > 0, + "sub_sleeping_thread: old_value {:?} had no sleeping threads", + old_value, + ); + debug_assert!( + old_value.sleeping_threads() <= old_value.inactive_threads(), + "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} inactive threads", + old_value, + old_value.sleeping_threads(), + old_value.inactive_threads(), + ); + } + + #[inline] + pub(super) fn try_add_sleeping_thread(&self, old_value: Counters) -> bool { + debug_assert!( + old_value.inactive_threads() > 0, + "try_add_sleeping_thread: old_value {:?} has no inactive threads", + old_value, + ); + debug_assert!( + old_value.sleeping_threads() < THREADS_MAX, + "try_add_sleeping_thread: old_value {:?} has too many sleeping threads", + old_value, + ); + + let mut new_value = old_value; + new_value.word += ONE_SLEEPING; + + self.try_exchange(old_value, new_value, Ordering::SeqCst) + } +} + +#[inline] +fn select_thread(word: usize, shift: usize) -> usize { + (word >> shift) & THREADS_MAX +} + +#[inline] +fn select_jec(word: usize) -> usize { + word >> JEC_SHIFT +} + +impl Counters { + #[inline] + fn new(word: usize) -> Counters { + Counters { word } + } + + #[inline] + fn increment_jobs_counter(self) -> Counters { + // We can freely add to JEC because it occupies the most significant bits. + // Thus it doesn't overflow into the other counters, just wraps itself. + Counters { + word: self.word.wrapping_add(ONE_JEC), + } + } + + #[inline] + pub(super) fn jobs_counter(self) -> JobsEventCounter { + JobsEventCounter(select_jec(self.word)) + } + + /// The number of threads that are not actively + /// executing work. They may be idle, sleepy, or asleep. + #[inline] + pub(super) fn inactive_threads(self) -> usize { + select_thread(self.word, INACTIVE_SHIFT) + } + + #[inline] + pub(super) fn awake_but_idle_threads(self) -> usize { + debug_assert!( + self.sleeping_threads() <= self.inactive_threads(), + "sleeping threads: {} > raw idle threads {}", + self.sleeping_threads(), + self.inactive_threads() + ); + self.inactive_threads() - self.sleeping_threads() + } + + #[inline] + pub(super) fn sleeping_threads(self) -> usize { + select_thread(self.word, SLEEPING_SHIFT) + } +} + +impl std::fmt::Debug for Counters { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let word = format!("{:016x}", self.word); + fmt.debug_struct("Counters") + .field("word", &word) + .field("jobs", &self.jobs_counter().0) + .field("inactive", &self.inactive_threads()) + .field("sleeping", &self.sleeping_threads()) + .finish() + } +} diff --git a/compiler/rustc_thread_pool/src/sleep/mod.rs b/compiler/rustc_thread_pool/src/sleep/mod.rs new file mode 100644 index 0000000000000..7d88ece210769 --- /dev/null +++ b/compiler/rustc_thread_pool/src/sleep/mod.rs @@ -0,0 +1,392 @@ +//! Code that decides when workers should go to sleep. See README.md +//! for an overview. + +use crate::latch::CoreLatch; +use crate::registry::WorkerThread; +use crate::DeadlockHandler; +use crossbeam_utils::CachePadded; +use std::sync::atomic::Ordering; +use std::sync::{Condvar, Mutex}; +use std::thread; + +mod counters; +pub(crate) use self::counters::THREADS_MAX; +use self::counters::{AtomicCounters, JobsEventCounter}; + +struct SleepData { + /// The number of threads in the thread pool. + worker_count: usize, + + /// The number of threads in the thread pool which are running and + /// aren't blocked in user code or sleeping. + active_threads: usize, + + /// The number of threads which are blocked in user code. + /// This doesn't include threads blocked by this module. + blocked_threads: usize, +} + +impl SleepData { + /// Checks if the conditions for a deadlock holds and if so calls the deadlock handler + #[inline] + pub fn deadlock_check(&self, deadlock_handler: &Option>) { + if self.active_threads == 0 && self.blocked_threads > 0 { + (deadlock_handler.as_ref().unwrap())(); + } + } +} + +/// The `Sleep` struct is embedded into each registry. It governs the waking and sleeping +/// of workers. It has callbacks that are invoked periodically at significant events, +/// such as when workers are looping and looking for work, when latches are set, or when +/// jobs are published, and it either blocks threads or wakes them in response to these +/// events. See the [`README.md`] in this module for more details. +/// +/// [`README.md`] README.md +pub(super) struct Sleep { + /// One "sleep state" per worker. Used to track if a worker is sleeping and to have + /// them block. + worker_sleep_states: Vec>, + + counters: AtomicCounters, + + data: Mutex, +} + +/// An instance of this struct is created when a thread becomes idle. +/// It is consumed when the thread finds work, and passed by `&mut` +/// reference for operations that preserve the idle state. (In other +/// words, producing one of these structs is evidence the thread is +/// idle.) It tracks state such as how long the thread has been idle. +pub(super) struct IdleState { + /// What is worker index of the idle thread? + worker_index: usize, + + /// How many rounds have we been circling without sleeping? + rounds: u32, + + /// Once we become sleepy, what was the sleepy counter value? + /// Set to `INVALID_SLEEPY_COUNTER` otherwise. + jobs_counter: JobsEventCounter, +} + +/// The "sleep state" for an individual worker. +#[derive(Default)] +struct WorkerSleepState { + /// Set to true when the worker goes to sleep; set to false when + /// the worker is notified or when it wakes. + is_blocked: Mutex, + + condvar: Condvar, +} + +const ROUNDS_UNTIL_SLEEPY: u32 = 32; +const ROUNDS_UNTIL_SLEEPING: u32 = ROUNDS_UNTIL_SLEEPY + 1; + +impl Sleep { + pub(super) fn new(n_threads: usize) -> Sleep { + assert!(n_threads <= THREADS_MAX); + Sleep { + worker_sleep_states: (0..n_threads).map(|_| Default::default()).collect(), + counters: AtomicCounters::new(), + data: Mutex::new(SleepData { + worker_count: n_threads, + active_threads: n_threads, + blocked_threads: 0, + }), + } + } + + /// Mark a Rayon worker thread as blocked. This triggers the deadlock handler + /// if no other worker thread is active + #[inline] + pub fn mark_blocked(&self, deadlock_handler: &Option>) { + let mut data = self.data.lock().unwrap(); + debug_assert!(data.active_threads > 0); + debug_assert!(data.blocked_threads < data.worker_count); + debug_assert!(data.active_threads > 0); + data.active_threads -= 1; + data.blocked_threads += 1; + + data.deadlock_check(deadlock_handler); + } + + /// Mark a previously blocked Rayon worker thread as unblocked + #[inline] + pub fn mark_unblocked(&self) { + let mut data = self.data.lock().unwrap(); + debug_assert!(data.active_threads < data.worker_count); + debug_assert!(data.blocked_threads > 0); + data.active_threads += 1; + data.blocked_threads -= 1; + } + + #[inline] + pub(super) fn start_looking(&self, worker_index: usize) -> IdleState { + self.counters.add_inactive_thread(); + + IdleState { + worker_index, + rounds: 0, + jobs_counter: JobsEventCounter::DUMMY, + } + } + + #[inline] + pub(super) fn work_found(&self) { + // If we were the last idle thread and other threads are still sleeping, + // then we should wake up another thread. + let threads_to_wake = self.counters.sub_inactive_thread(); + self.wake_any_threads(threads_to_wake as u32); + } + + #[inline] + pub(super) fn no_work_found( + &self, + idle_state: &mut IdleState, + latch: &CoreLatch, + thread: &WorkerThread, + ) { + if idle_state.rounds < ROUNDS_UNTIL_SLEEPY { + thread::yield_now(); + idle_state.rounds += 1; + } else if idle_state.rounds == ROUNDS_UNTIL_SLEEPY { + idle_state.jobs_counter = self.announce_sleepy(); + idle_state.rounds += 1; + thread::yield_now(); + } else if idle_state.rounds < ROUNDS_UNTIL_SLEEPING { + idle_state.rounds += 1; + thread::yield_now(); + } else { + debug_assert_eq!(idle_state.rounds, ROUNDS_UNTIL_SLEEPING); + self.sleep(idle_state, latch, thread); + } + } + + #[cold] + fn announce_sleepy(&self) -> JobsEventCounter { + self.counters + .increment_jobs_event_counter_if(JobsEventCounter::is_active) + .jobs_counter() + } + + #[cold] + fn sleep(&self, idle_state: &mut IdleState, latch: &CoreLatch, thread: &WorkerThread) { + let worker_index = idle_state.worker_index; + + if !latch.get_sleepy() { + return; + } + + let sleep_state = &self.worker_sleep_states[worker_index]; + let mut is_blocked = sleep_state.is_blocked.lock().unwrap(); + debug_assert!(!*is_blocked); + + // Our latch was signalled. We should wake back up fully as we + // will have some stuff to do. + if !latch.fall_asleep() { + idle_state.wake_fully(); + return; + } + + loop { + let counters = self.counters.load(Ordering::SeqCst); + + // Check if the JEC has changed since we got sleepy. + debug_assert!(idle_state.jobs_counter.is_sleepy()); + if counters.jobs_counter() != idle_state.jobs_counter { + // JEC has changed, so a new job was posted, but for some reason + // we didn't see it. We should return to just before the SLEEPY + // state so we can do another search and (if we fail to find + // work) go back to sleep. + idle_state.wake_partly(); + latch.wake_up(); + return; + } + + // Otherwise, let's move from IDLE to SLEEPING. + if self.counters.try_add_sleeping_thread(counters) { + break; + } + } + + // Successfully registered as asleep. + + // We have one last check for injected jobs to do. This protects against + // deadlock in the very unlikely event that + // + // - an external job is being injected while we are sleepy + // - that job triggers the rollover over the JEC such that we don't see it + // - we are the last active worker thread + std::sync::atomic::fence(Ordering::SeqCst); + if thread.has_injected_job() { + // If we see an externally injected job, then we have to 'wake + // ourselves up'. (Ordinarily, `sub_sleeping_thread` is invoked by + // the one that wakes us.) + self.counters.sub_sleeping_thread(); + } else { + { + // Decrement the number of active threads and check for a deadlock + let mut data = self.data.lock().unwrap(); + data.active_threads -= 1; + data.deadlock_check(&thread.registry.deadlock_handler); + } + + // If we don't see an injected job (the normal case), then flag + // ourselves as asleep and wait till we are notified. + // + // (Note that `is_blocked` is held under a mutex and the mutex was + // acquired *before* we incremented the "sleepy counter". This means + // that whomever is coming to wake us will have to wait until we + // release the mutex in the call to `wait`, so they will see this + // boolean as true.) + thread.registry.release_thread(); + *is_blocked = true; + while *is_blocked { + is_blocked = sleep_state.condvar.wait(is_blocked).unwrap(); + } + + // Drop `is_blocked` now in case `acquire_thread` blocks + drop(is_blocked); + + thread.registry.acquire_thread(); + } + + // Update other state: + idle_state.wake_fully(); + latch.wake_up(); + } + + /// Notify the given thread that it should wake up (if it is + /// sleeping). When this method is invoked, we typically know the + /// thread is asleep, though in rare cases it could have been + /// awoken by (e.g.) new work having been posted. + pub(super) fn notify_worker_latch_is_set(&self, target_worker_index: usize) { + self.wake_specific_thread(target_worker_index); + } + + /// Signals that `num_jobs` new jobs were injected into the thread + /// pool from outside. This function will ensure that there are + /// threads available to process them, waking threads from sleep + /// if necessary. + /// + /// # Parameters + /// + /// - `num_jobs` -- lower bound on number of jobs available for stealing. + /// We'll try to get at least one thread per job. + #[inline] + pub(super) fn new_injected_jobs(&self, num_jobs: u32, queue_was_empty: bool) { + // This fence is needed to guarantee that threads + // as they are about to fall asleep, observe any + // new jobs that may have been injected. + std::sync::atomic::fence(Ordering::SeqCst); + + self.new_jobs(num_jobs, queue_was_empty) + } + + /// Signals that `num_jobs` new jobs were pushed onto a thread's + /// local deque. This function will try to ensure that there are + /// threads available to process them, waking threads from sleep + /// if necessary. However, this is not guaranteed: under certain + /// race conditions, the function may fail to wake any new + /// threads; in that case the existing thread should eventually + /// pop the job. + /// + /// # Parameters + /// + /// - `num_jobs` -- lower bound on number of jobs available for stealing. + /// We'll try to get at least one thread per job. + #[inline] + pub(super) fn new_internal_jobs(&self, num_jobs: u32, queue_was_empty: bool) { + self.new_jobs(num_jobs, queue_was_empty) + } + + /// Common helper for `new_injected_jobs` and `new_internal_jobs`. + #[inline] + fn new_jobs(&self, num_jobs: u32, queue_was_empty: bool) { + // Read the counters and -- if sleepy workers have announced themselves + // -- announce that there is now work available. The final value of `counters` + // with which we exit the loop thus corresponds to a state when + let counters = self + .counters + .increment_jobs_event_counter_if(JobsEventCounter::is_sleepy); + let num_awake_but_idle = counters.awake_but_idle_threads(); + let num_sleepers = counters.sleeping_threads(); + + if num_sleepers == 0 { + // nobody to wake + return; + } + + // Promote from u16 to u32 so we can interoperate with + // num_jobs more easily. + let num_awake_but_idle = num_awake_but_idle as u32; + let num_sleepers = num_sleepers as u32; + + // If the queue is non-empty, then we always wake up a worker + // -- clearly the existing idle jobs aren't enough. Otherwise, + // check to see if we have enough idle workers. + if !queue_was_empty { + let num_to_wake = Ord::min(num_jobs, num_sleepers); + self.wake_any_threads(num_to_wake); + } else if num_awake_but_idle < num_jobs { + let num_to_wake = Ord::min(num_jobs - num_awake_but_idle, num_sleepers); + self.wake_any_threads(num_to_wake); + } + } + + #[cold] + fn wake_any_threads(&self, mut num_to_wake: u32) { + if num_to_wake > 0 { + for i in 0..self.worker_sleep_states.len() { + if self.wake_specific_thread(i) { + num_to_wake -= 1; + if num_to_wake == 0 { + return; + } + } + } + } + } + + fn wake_specific_thread(&self, index: usize) -> bool { + let sleep_state = &self.worker_sleep_states[index]; + + let mut is_blocked = sleep_state.is_blocked.lock().unwrap(); + if *is_blocked { + *is_blocked = false; + + // Increment the number of active threads + self.data.lock().unwrap().active_threads += 1; + + sleep_state.condvar.notify_one(); + + // When the thread went to sleep, it will have incremented + // this value. When we wake it, its our job to decrement + // it. We could have the thread do it, but that would + // introduce a delay between when the thread was + // *notified* and when this counter was decremented. That + // might mislead people with new work into thinking that + // there are sleeping threads that they should try to + // wake, when in fact there is nothing left for them to + // do. + self.counters.sub_sleeping_thread(); + + true + } else { + false + } + } +} + +impl IdleState { + fn wake_fully(&mut self) { + self.rounds = 0; + self.jobs_counter = JobsEventCounter::DUMMY; + } + + fn wake_partly(&mut self) { + self.rounds = ROUNDS_UNTIL_SLEEPY; + self.jobs_counter = JobsEventCounter::DUMMY; + } +} diff --git a/compiler/rustc_thread_pool/src/spawn/mod.rs b/compiler/rustc_thread_pool/src/spawn/mod.rs new file mode 100644 index 0000000000000..034df30dcfb32 --- /dev/null +++ b/compiler/rustc_thread_pool/src/spawn/mod.rs @@ -0,0 +1,164 @@ +use crate::job::*; +use crate::registry::Registry; +use crate::tlv::Tlv; +use crate::unwind; +use std::mem; +use std::sync::Arc; + +/// Puts the task into the Rayon threadpool's job queue in the "static" +/// or "global" scope. Just like a standard thread, this task is not +/// tied to the current stack frame, and hence it cannot hold any +/// references other than those with `'static` lifetime. If you want +/// to spawn a task that references stack data, use [the `scope()` +/// function][scope] to create a scope. +/// +/// [scope]: fn.scope.html +/// +/// Since tasks spawned with this function cannot hold references into +/// the enclosing stack frame, you almost certainly want to use a +/// `move` closure as their argument (otherwise, the closure will +/// typically hold references to any variables from the enclosing +/// function that you happen to use). +/// +/// This API assumes that the closure is executed purely for its +/// side-effects (i.e., it might send messages, modify data protected +/// by a mutex, or some such thing). +/// +/// There is no guaranteed order of execution for spawns, given that +/// other threads may steal tasks at any time. However, they are +/// generally prioritized in a LIFO order on the thread from which +/// they were spawned. Other threads always steal from the other end of +/// the deque, like FIFO order. The idea is that "recent" tasks are +/// most likely to be fresh in the local CPU's cache, while other +/// threads can steal older "stale" tasks. For an alternate approach, +/// consider [`spawn_fifo()`] instead. +/// +/// [`spawn_fifo()`]: fn.spawn_fifo.html +/// +/// # Panic handling +/// +/// If this closure should panic, the resulting panic will be +/// propagated to the panic handler registered in the `ThreadPoolBuilder`, +/// if any. See [`ThreadPoolBuilder::panic_handler()`][ph] for more +/// details. +/// +/// [ph]: struct.ThreadPoolBuilder.html#method.panic_handler +/// +/// # Examples +/// +/// This code creates a Rayon task that increments a global counter. +/// +/// ```rust +/// # use rayon_core as rayon; +/// use std::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT}; +/// +/// static GLOBAL_COUNTER: AtomicUsize = ATOMIC_USIZE_INIT; +/// +/// rayon::spawn(move || { +/// GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst); +/// }); +/// ``` +pub fn spawn(func: F) +where + F: FnOnce() + Send + 'static, +{ + // We assert that current registry has not terminated. + unsafe { spawn_in(func, &Registry::current()) } +} + +/// Spawns an asynchronous job in `registry.` +/// +/// Unsafe because `registry` must not yet have terminated. +pub(super) unsafe fn spawn_in(func: F, registry: &Arc) +where + F: FnOnce() + Send + 'static, +{ + // We assert that this does not hold any references (we know + // this because of the `'static` bound in the interface); + // moreover, we assert that the code below is not supposed to + // be able to panic, and hence the data won't leak but will be + // enqueued into some deque for later execution. + let abort_guard = unwind::AbortIfPanic; // just in case we are wrong, and code CAN panic + let job_ref = spawn_job(func, registry); + registry.inject_or_push(job_ref); + mem::forget(abort_guard); +} + +unsafe fn spawn_job(func: F, registry: &Arc) -> JobRef +where + F: FnOnce() + Send + 'static, +{ + // Ensure that registry cannot terminate until this job has + // executed. This ref is decremented at the (*) below. + registry.increment_terminate_count(); + + HeapJob::new(Tlv::null(), { + let registry = Arc::clone(registry); + move || { + registry.catch_unwind(func); + registry.terminate(); // (*) permit registry to terminate now + } + }) + .into_static_job_ref() +} + +/// Fires off a task into the Rayon threadpool in the "static" or +/// "global" scope. Just like a standard thread, this task is not +/// tied to the current stack frame, and hence it cannot hold any +/// references other than those with `'static` lifetime. If you want +/// to spawn a task that references stack data, use [the `scope_fifo()` +/// function](fn.scope_fifo.html) to create a scope. +/// +/// The behavior is essentially the same as [the `spawn` +/// function](fn.spawn.html), except that calls from the same thread +/// will be prioritized in FIFO order. This is similar to the now- +/// deprecated [`breadth_first`] option, except the effect is isolated +/// to relative `spawn_fifo` calls, not all threadpool tasks. +/// +/// For more details on this design, see Rayon [RFC #1]. +/// +/// [`breadth_first`]: struct.ThreadPoolBuilder.html#method.breadth_first +/// [RFC #1]: https://github.com/rayon-rs/rfcs/blob/master/accepted/rfc0001-scope-scheduling.md +/// +/// # Panic handling +/// +/// If this closure should panic, the resulting panic will be +/// propagated to the panic handler registered in the `ThreadPoolBuilder`, +/// if any. See [`ThreadPoolBuilder::panic_handler()`][ph] for more +/// details. +/// +/// [ph]: struct.ThreadPoolBuilder.html#method.panic_handler +pub fn spawn_fifo(func: F) +where + F: FnOnce() + Send + 'static, +{ + // We assert that current registry has not terminated. + unsafe { spawn_fifo_in(func, &Registry::current()) } +} + +/// Spawns an asynchronous FIFO job in `registry.` +/// +/// Unsafe because `registry` must not yet have terminated. +pub(super) unsafe fn spawn_fifo_in(func: F, registry: &Arc) +where + F: FnOnce() + Send + 'static, +{ + // We assert that this does not hold any references (we know + // this because of the `'static` bound in the interface); + // moreover, we assert that the code below is not supposed to + // be able to panic, and hence the data won't leak but will be + // enqueued into some deque for later execution. + let abort_guard = unwind::AbortIfPanic; // just in case we are wrong, and code CAN panic + let job_ref = spawn_job(func, registry); + + // If we're in the pool, use our thread's private fifo for this thread to execute + // in a locally-FIFO order. Otherwise, just use the pool's global injector. + match registry.current_thread() { + Some(worker) => worker.push_fifo(job_ref), + None => registry.inject(job_ref), + } + mem::forget(abort_guard); +} + +#[cfg(test)] +mod test; diff --git a/compiler/rustc_thread_pool/src/spawn/test.rs b/compiler/rustc_thread_pool/src/spawn/test.rs new file mode 100644 index 0000000000000..b7a0535aabf6d --- /dev/null +++ b/compiler/rustc_thread_pool/src/spawn/test.rs @@ -0,0 +1,255 @@ +use crate::scope; +use std::any::Any; +use std::sync::mpsc::channel; +use std::sync::Mutex; + +use super::{spawn, spawn_fifo}; +use crate::ThreadPoolBuilder; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_then_join_in_worker() { + let (tx, rx) = channel(); + scope(move |_| { + spawn(move || tx.send(22).unwrap()); + }); + assert_eq!(22, rx.recv().unwrap()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_then_join_outside_worker() { + let (tx, rx) = channel(); + spawn(move || tx.send(22).unwrap()); + assert_eq!(22, rx.recv().unwrap()); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_fwd() { + let (tx, rx) = channel(); + + let tx = Mutex::new(tx); + let panic_handler = move |err: Box| { + let tx = tx.lock().unwrap(); + if let Some(&msg) = err.downcast_ref::<&str>() { + if msg == "Hello, world!" { + tx.send(1).unwrap(); + } else { + tx.send(2).unwrap(); + } + } else { + tx.send(3).unwrap(); + } + }; + + let builder = ThreadPoolBuilder::new().panic_handler(panic_handler); + + builder + .build() + .unwrap() + .spawn(move || panic!("Hello, world!")); + + assert_eq!(1, rx.recv().unwrap()); +} + +/// Test what happens when the thread-pool is dropped but there are +/// still active asynchronous tasks. We expect the thread-pool to stay +/// alive and executing until those threads are complete. +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn termination_while_things_are_executing() { + let (tx0, rx0) = channel(); + let (tx1, rx1) = channel(); + + // Create a thread-pool and spawn some code in it, but then drop + // our reference to it. + { + let thread_pool = ThreadPoolBuilder::new().build().unwrap(); + thread_pool.spawn(move || { + let data = rx0.recv().unwrap(); + + // At this point, we know the "main" reference to the + // `ThreadPool` has been dropped, but there are still + // active threads. Launch one more. + spawn(move || { + tx1.send(data).unwrap(); + }); + }); + } + + tx0.send(22).unwrap(); + let v = rx1.recv().unwrap(); + assert_eq!(v, 22); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn custom_panic_handler_and_spawn() { + let (tx, rx) = channel(); + + // Create a parallel closure that will send panics on the + // channel; since the closure is potentially executed in parallel + // with itself, we have to wrap `tx` in a mutex. + let tx = Mutex::new(tx); + let panic_handler = move |e: Box| { + tx.lock().unwrap().send(e).unwrap(); + }; + + // Execute an async that will panic. + let builder = ThreadPoolBuilder::new().panic_handler(panic_handler); + builder.build().unwrap().spawn(move || { + panic!("Hello, world!"); + }); + + // Check that we got back the panic we expected. + let error = rx.recv().unwrap(); + if let Some(&msg) = error.downcast_ref::<&str>() { + assert_eq!(msg, "Hello, world!"); + } else { + panic!("did not receive a string from panic handler"); + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn custom_panic_handler_and_nested_spawn() { + let (tx, rx) = channel(); + + // Create a parallel closure that will send panics on the + // channel; since the closure is potentially executed in parallel + // with itself, we have to wrap `tx` in a mutex. + let tx = Mutex::new(tx); + let panic_handler = move |e| { + tx.lock().unwrap().send(e).unwrap(); + }; + + // Execute an async that will (eventually) panic. + const PANICS: usize = 3; + let builder = ThreadPoolBuilder::new().panic_handler(panic_handler); + builder.build().unwrap().spawn(move || { + // launch 3 nested spawn-asyncs; these should be in the same + // thread-pool and hence inherit the same panic handler + for _ in 0..PANICS { + spawn(move || { + panic!("Hello, world!"); + }); + } + }); + + // Check that we get back the panics we expected. + for _ in 0..PANICS { + let error = rx.recv().unwrap(); + if let Some(&msg) = error.downcast_ref::<&str>() { + assert_eq!(msg, "Hello, world!"); + } else { + panic!("did not receive a string from panic handler"); + } + } +} + +macro_rules! test_order { + ($outer_spawn:ident, $inner_spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + let (tx, rx) = channel(); + pool.install(move || { + for i in 0..10 { + let tx = tx.clone(); + $outer_spawn(move || { + for j in 0..10 { + let tx = tx.clone(); + $inner_spawn(move || { + tx.send(i * 10 + j).unwrap(); + }); + } + }); + } + }); + rx.iter().collect::>() + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn lifo_order() { + // In the absence of stealing, `spawn()` jobs on a thread will run in LIFO order. + let vec = test_order!(spawn, spawn); + let expected: Vec = (0..100).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn fifo_order() { + // In the absence of stealing, `spawn_fifo()` jobs on a thread will run in FIFO order. + let vec = test_order!(spawn_fifo, spawn_fifo); + let expected: Vec = (0..100).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn lifo_fifo_order() { + // LIFO on the outside, FIFO on the inside + let vec = test_order!(spawn, spawn_fifo); + let expected: Vec = (0..10) + .rev() + .flat_map(|i| (0..10).map(move |j| i * 10 + j)) + .collect(); + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn fifo_lifo_order() { + // FIFO on the outside, LIFO on the inside + let vec = test_order!(spawn_fifo, spawn); + let expected: Vec = (0..10) + .flat_map(|i| (0..10).rev().map(move |j| i * 10 + j)) + .collect(); + assert_eq!(vec, expected); +} + +macro_rules! spawn_send { + ($spawn:ident, $tx:ident, $i:expr) => {{ + let tx = $tx.clone(); + $spawn(move || tx.send($i).unwrap()); + }}; +} + +/// Test mixed spawns pushing a series of numbers, interleaved such +/// such that negative values are using the second kind of spawn. +macro_rules! test_mixed_order { + ($pos_spawn:ident, $neg_spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + let (tx, rx) = channel(); + pool.install(move || { + spawn_send!($pos_spawn, tx, 0); + spawn_send!($neg_spawn, tx, -1); + spawn_send!($pos_spawn, tx, 1); + spawn_send!($neg_spawn, tx, -2); + spawn_send!($pos_spawn, tx, 2); + spawn_send!($neg_spawn, tx, -3); + spawn_send!($pos_spawn, tx, 3); + }); + rx.iter().collect::>() + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_lifo_fifo_order() { + let vec = test_mixed_order!(spawn, spawn_fifo); + let expected = vec![3, -1, 2, -2, 1, -3, 0]; + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_fifo_lifo_order() { + let vec = test_mixed_order!(spawn_fifo, spawn); + let expected = vec![0, -3, 1, -2, 2, -1, 3]; + assert_eq!(vec, expected); +} diff --git a/compiler/rustc_thread_pool/src/test.rs b/compiler/rustc_thread_pool/src/test.rs new file mode 100644 index 0000000000000..25b8487f73bc9 --- /dev/null +++ b/compiler/rustc_thread_pool/src/test.rs @@ -0,0 +1,200 @@ +#![cfg(test)] + +use crate::{ThreadPoolBuildError, ThreadPoolBuilder}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Barrier}; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn worker_thread_index() { + let pool = ThreadPoolBuilder::new().num_threads(22).build().unwrap(); + assert_eq!(pool.current_num_threads(), 22); + assert_eq!(pool.current_thread_index(), None); + let index = pool.install(|| pool.current_thread_index().unwrap()); + assert!(index < 22); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn start_callback_called() { + let n_threads = 16; + let n_called = Arc::new(AtomicUsize::new(0)); + // Wait for all the threads in the pool plus the one running tests. + let barrier = Arc::new(Barrier::new(n_threads + 1)); + + let b = Arc::clone(&barrier); + let nc = Arc::clone(&n_called); + let start_handler = move |_| { + nc.fetch_add(1, Ordering::SeqCst); + b.wait(); + }; + + let conf = ThreadPoolBuilder::new() + .num_threads(n_threads) + .start_handler(start_handler); + let _ = conf.build().unwrap(); + + // Wait for all the threads to have been scheduled to run. + barrier.wait(); + + // The handler must have been called on every started thread. + assert_eq!(n_called.load(Ordering::SeqCst), n_threads); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn exit_callback_called() { + let n_threads = 16; + let n_called = Arc::new(AtomicUsize::new(0)); + // Wait for all the threads in the pool plus the one running tests. + let barrier = Arc::new(Barrier::new(n_threads + 1)); + + let b = Arc::clone(&barrier); + let nc = Arc::clone(&n_called); + let exit_handler = move |_| { + nc.fetch_add(1, Ordering::SeqCst); + b.wait(); + }; + + let conf = ThreadPoolBuilder::new() + .num_threads(n_threads) + .exit_handler(exit_handler); + { + let _ = conf.build().unwrap(); + // Drop the pool so it stops the running threads. + } + + // Wait for all the threads to have been scheduled to run. + barrier.wait(); + + // The handler must have been called on every exiting thread. + assert_eq!(n_called.load(Ordering::SeqCst), n_threads); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn handler_panics_handled_correctly() { + let n_threads = 16; + let n_called = Arc::new(AtomicUsize::new(0)); + // Wait for all the threads in the pool plus the one running tests. + let start_barrier = Arc::new(Barrier::new(n_threads + 1)); + let exit_barrier = Arc::new(Barrier::new(n_threads + 1)); + + let start_handler = move |_| { + panic!("ensure panic handler is called when starting"); + }; + let exit_handler = move |_| { + panic!("ensure panic handler is called when exiting"); + }; + + let sb = Arc::clone(&start_barrier); + let eb = Arc::clone(&exit_barrier); + let nc = Arc::clone(&n_called); + let panic_handler = move |_| { + let val = nc.fetch_add(1, Ordering::SeqCst); + if val < n_threads { + sb.wait(); + } else { + eb.wait(); + } + }; + + let conf = ThreadPoolBuilder::new() + .num_threads(n_threads) + .start_handler(start_handler) + .exit_handler(exit_handler) + .panic_handler(panic_handler); + { + let _ = conf.build().unwrap(); + + // Wait for all the threads to start, panic in the start handler, + // and been taken care of by the panic handler. + start_barrier.wait(); + + // Drop the pool so it stops the running threads. + } + + // Wait for all the threads to exit, panic in the exit handler, + // and been taken care of by the panic handler. + exit_barrier.wait(); + + // The panic handler must have been called twice on every thread. + assert_eq!(n_called.load(Ordering::SeqCst), 2 * n_threads); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn check_config_build() { + let pool = ThreadPoolBuilder::new().num_threads(22).build().unwrap(); + assert_eq!(pool.current_num_threads(), 22); +} + +/// Helper used by check_error_send_sync to ensure ThreadPoolBuildError is Send + Sync +fn _send_sync() {} + +#[test] +fn check_error_send_sync() { + _send_sync::(); +} + +#[allow(deprecated)] +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn configuration() { + let start_handler = move |_| {}; + let exit_handler = move |_| {}; + let panic_handler = move |_| {}; + let thread_name = move |i| format!("thread_name_{}", i); + + // Ensure we can call all public methods on Configuration + crate::Configuration::new() + .thread_name(thread_name) + .num_threads(5) + .panic_handler(panic_handler) + .stack_size(4e6 as usize) + .breadth_first() + .start_handler(start_handler) + .exit_handler(exit_handler) + .build() + .unwrap(); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn default_pool() { + ThreadPoolBuilder::default().build().unwrap(); +} + +/// Test that custom spawned threads get their `WorkerThread` cleared once +/// the pool is done with them, allowing them to be used with rayon again +/// later. e.g. WebAssembly want to have their own pool of available threads. +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn cleared_current_thread() -> Result<(), ThreadPoolBuildError> { + let n_threads = 5; + let mut handles = vec![]; + let pool = ThreadPoolBuilder::new() + .num_threads(n_threads) + .spawn_handler(|thread| { + let handle = std::thread::spawn(move || { + thread.run(); + + // Afterward, the current thread shouldn't be set anymore. + assert_eq!(crate::current_thread_index(), None); + }); + handles.push(handle); + Ok(()) + }) + .build()?; + assert_eq!(handles.len(), n_threads); + + pool.install(|| assert!(crate::current_thread_index().is_some())); + drop(pool); + + // Wait for all threads to make their assertions and exit + for handle in handles { + handle.join().unwrap(); + } + + Ok(()) +} diff --git a/compiler/rustc_thread_pool/src/thread_pool/mod.rs b/compiler/rustc_thread_pool/src/thread_pool/mod.rs new file mode 100644 index 0000000000000..65af6d7106e64 --- /dev/null +++ b/compiler/rustc_thread_pool/src/thread_pool/mod.rs @@ -0,0 +1,514 @@ +//! Contains support for user-managed thread pools, represented by the +//! the [`ThreadPool`] type (see that struct for details). +//! +//! [`ThreadPool`]: struct.ThreadPool.html + +use crate::broadcast::{self, BroadcastContext}; +use crate::join; +use crate::registry::{Registry, ThreadSpawn, WorkerThread}; +use crate::scope::{do_in_place_scope, do_in_place_scope_fifo}; +use crate::spawn; +use crate::{scope, Scope}; +use crate::{scope_fifo, ScopeFifo}; +use crate::{ThreadPoolBuildError, ThreadPoolBuilder}; +use std::error::Error; +use std::fmt; +use std::sync::Arc; + +mod test; + +/// Represents a user created [thread-pool]. +/// +/// Use a [`ThreadPoolBuilder`] to specify the number and/or names of threads +/// in the pool. After calling [`ThreadPoolBuilder::build()`], you can then +/// execute functions explicitly within this [`ThreadPool`] using +/// [`ThreadPool::install()`]. By contrast, top level rayon functions +/// (like `join()`) will execute implicitly within the current thread-pool. +/// +/// +/// ## Creating a ThreadPool +/// +/// ```rust +/// # use rayon_core as rayon; +/// let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); +/// ``` +/// +/// [`install()`][`ThreadPool::install()`] executes a closure in one of the `ThreadPool`'s +/// threads. In addition, any other rayon operations called inside of `install()` will also +/// execute in the context of the `ThreadPool`. +/// +/// When the `ThreadPool` is dropped, that's a signal for the threads it manages to terminate, +/// they will complete executing any remaining work that you have spawned, and automatically +/// terminate. +/// +/// +/// [thread-pool]: https://en.wikipedia.org/wiki/Thread_pool +/// [`ThreadPool`]: struct.ThreadPool.html +/// [`ThreadPool::new()`]: struct.ThreadPool.html#method.new +/// [`ThreadPoolBuilder`]: struct.ThreadPoolBuilder.html +/// [`ThreadPoolBuilder::build()`]: struct.ThreadPoolBuilder.html#method.build +/// [`ThreadPool::install()`]: struct.ThreadPool.html#method.install +pub struct ThreadPool { + registry: Arc, +} + +impl ThreadPool { + #[deprecated(note = "Use `ThreadPoolBuilder::build`")] + #[allow(deprecated)] + /// Deprecated in favor of `ThreadPoolBuilder::build`. + pub fn new(configuration: crate::Configuration) -> Result> { + Self::build(configuration.into_builder()).map_err(Box::from) + } + + pub(super) fn build( + builder: ThreadPoolBuilder, + ) -> Result + where + S: ThreadSpawn, + { + let registry = Registry::new(builder)?; + Ok(ThreadPool { registry }) + } + + /// Executes `op` within the threadpool. Any attempts to use + /// `join`, `scope`, or parallel iterators will then operate + /// within that threadpool. + /// + /// # Warning: thread-local data + /// + /// Because `op` is executing within the Rayon thread-pool, + /// thread-local data from the current thread will not be + /// accessible. + /// + /// # Warning: execution order + /// + /// If the current thread is part of a different thread pool, it will try to + /// keep busy while the `op` completes in its target pool, similar to + /// calling [`ThreadPool::yield_now()`] in a loop. Therefore, it may + /// potentially schedule other tasks to run on the current thread in the + /// meantime. For example + /// + /// ```rust + /// # use rayon_core as rayon; + /// fn main() { + /// rayon::ThreadPoolBuilder::new().num_threads(1).build_global().unwrap(); + /// let pool = rayon_core::ThreadPoolBuilder::default().build().unwrap(); + /// let do_it = || { + /// print!("one "); + /// pool.install(||{}); + /// print!("two "); + /// }; + /// rayon::join(|| do_it(), || do_it()); + /// } + /// ``` + /// + /// Since we configured just one thread in the global pool, one might + /// expect `do_it()` to run sequentially, producing: + /// + /// ```ascii + /// one two one two + /// ``` + /// + /// However each call to `install()` yields implicitly, allowing rayon to + /// run multiple instances of `do_it()` concurrently on the single, global + /// thread. The following output would be equally valid: + /// + /// ```ascii + /// one one two two + /// ``` + /// + /// # Panics + /// + /// If `op` should panic, that panic will be propagated. + /// + /// ## Using `install()` + /// + /// ```rust + /// # use rayon_core as rayon; + /// fn main() { + /// let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); + /// let n = pool.install(|| fib(20)); + /// println!("{}", n); + /// } + /// + /// fn fib(n: usize) -> usize { + /// if n == 0 || n == 1 { + /// return n; + /// } + /// let (a, b) = rayon::join(|| fib(n - 1), || fib(n - 2)); // runs inside of `pool` + /// return a + b; + /// } + /// ``` + pub fn install(&self, op: OP) -> R + where + OP: FnOnce() -> R + Send, + R: Send, + { + self.registry.in_worker(|_, _| op()) + } + + /// Executes `op` within every thread in the threadpool. Any attempts to use + /// `join`, `scope`, or parallel iterators will then operate within that + /// threadpool. + /// + /// Broadcasts are executed on each thread after they have exhausted their + /// local work queue, before they attempt work-stealing from other threads. + /// The goal of that strategy is to run everywhere in a timely manner + /// *without* being too disruptive to current work. There may be alternative + /// broadcast styles added in the future for more or less aggressive + /// injection, if the need arises. + /// + /// # Warning: thread-local data + /// + /// Because `op` is executing within the Rayon thread-pool, + /// thread-local data from the current thread will not be + /// accessible. + /// + /// # Panics + /// + /// If `op` should panic on one or more threads, exactly one panic + /// will be propagated, only after all threads have completed + /// (or panicked) their own `op`. + /// + /// # Examples + /// + /// ``` + /// # use rayon_core as rayon; + /// use std::sync::atomic::{AtomicUsize, Ordering}; + /// + /// fn main() { + /// let pool = rayon::ThreadPoolBuilder::new().num_threads(5).build().unwrap(); + /// + /// // The argument gives context, including the index of each thread. + /// let v: Vec = pool.broadcast(|ctx| ctx.index() * ctx.index()); + /// assert_eq!(v, &[0, 1, 4, 9, 16]); + /// + /// // The closure can reference the local stack + /// let count = AtomicUsize::new(0); + /// pool.broadcast(|_| count.fetch_add(1, Ordering::Relaxed)); + /// assert_eq!(count.into_inner(), 5); + /// } + /// ``` + pub fn broadcast(&self, op: OP) -> Vec + where + OP: Fn(BroadcastContext<'_>) -> R + Sync, + R: Send, + { + // We assert that `self.registry` has not terminated. + unsafe { broadcast::broadcast_in(op, &self.registry) } + } + + /// Returns the (current) number of threads in the thread pool. + /// + /// # Future compatibility note + /// + /// Note that unless this thread-pool was created with a + /// [`ThreadPoolBuilder`] that specifies the number of threads, + /// then this number may vary over time in future versions (see [the + /// `num_threads()` method for details][snt]). + /// + /// [snt]: struct.ThreadPoolBuilder.html#method.num_threads + /// [`ThreadPoolBuilder`]: struct.ThreadPoolBuilder.html + #[inline] + pub fn current_num_threads(&self) -> usize { + self.registry.num_threads() + } + + /// If called from a Rayon worker thread in this thread-pool, + /// returns the index of that thread; if not called from a Rayon + /// thread, or called from a Rayon thread that belongs to a + /// different thread-pool, returns `None`. + /// + /// The index for a given thread will not change over the thread's + /// lifetime. However, multiple threads may share the same index if + /// they are in distinct thread-pools. + /// + /// # Future compatibility note + /// + /// Currently, every thread-pool (including the global + /// thread-pool) has a fixed number of threads, but this may + /// change in future Rayon versions (see [the `num_threads()` method + /// for details][snt]). In that case, the index for a + /// thread would not change during its lifetime, but thread + /// indices may wind up being reused if threads are terminated and + /// restarted. + /// + /// [snt]: struct.ThreadPoolBuilder.html#method.num_threads + #[inline] + pub fn current_thread_index(&self) -> Option { + let curr = self.registry.current_thread()?; + Some(curr.index()) + } + + /// Returns true if the current worker thread currently has "local + /// tasks" pending. This can be useful as part of a heuristic for + /// deciding whether to spawn a new task or execute code on the + /// current thread, particularly in breadth-first + /// schedulers. However, keep in mind that this is an inherently + /// racy check, as other worker threads may be actively "stealing" + /// tasks from our local deque. + /// + /// **Background:** Rayon's uses a [work-stealing] scheduler. The + /// key idea is that each thread has its own [deque] of + /// tasks. Whenever a new task is spawned -- whether through + /// `join()`, `Scope::spawn()`, or some other means -- that new + /// task is pushed onto the thread's *local* deque. Worker threads + /// have a preference for executing their own tasks; if however + /// they run out of tasks, they will go try to "steal" tasks from + /// other threads. This function therefore has an inherent race + /// with other active worker threads, which may be removing items + /// from the local deque. + /// + /// [work-stealing]: https://en.wikipedia.org/wiki/Work_stealing + /// [deque]: https://en.wikipedia.org/wiki/Double-ended_queue + #[inline] + pub fn current_thread_has_pending_tasks(&self) -> Option { + let curr = self.registry.current_thread()?; + Some(!curr.local_deque_is_empty()) + } + + /// Execute `oper_a` and `oper_b` in the thread-pool and return + /// the results. Equivalent to `self.install(|| join(oper_a, + /// oper_b))`. + pub fn join(&self, oper_a: A, oper_b: B) -> (RA, RB) + where + A: FnOnce() -> RA + Send, + B: FnOnce() -> RB + Send, + RA: Send, + RB: Send, + { + self.install(|| join(oper_a, oper_b)) + } + + /// Creates a scope that executes within this thread-pool. + /// Equivalent to `self.install(|| scope(...))`. + /// + /// See also: [the `scope()` function][scope]. + /// + /// [scope]: fn.scope.html + pub fn scope<'scope, OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&Scope<'scope>) -> R + Send, + R: Send, + { + self.install(|| scope(op)) + } + + /// Creates a scope that executes within this thread-pool. + /// Spawns from the same thread are prioritized in relative FIFO order. + /// Equivalent to `self.install(|| scope_fifo(...))`. + /// + /// See also: [the `scope_fifo()` function][scope_fifo]. + /// + /// [scope_fifo]: fn.scope_fifo.html + pub fn scope_fifo<'scope, OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&ScopeFifo<'scope>) -> R + Send, + R: Send, + { + self.install(|| scope_fifo(op)) + } + + /// Creates a scope that spawns work into this thread-pool. + /// + /// See also: [the `in_place_scope()` function][in_place_scope]. + /// + /// [in_place_scope]: fn.in_place_scope.html + pub fn in_place_scope<'scope, OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&Scope<'scope>) -> R, + { + do_in_place_scope(Some(&self.registry), op) + } + + /// Creates a scope that spawns work into this thread-pool in FIFO order. + /// + /// See also: [the `in_place_scope_fifo()` function][in_place_scope_fifo]. + /// + /// [in_place_scope_fifo]: fn.in_place_scope_fifo.html + pub fn in_place_scope_fifo<'scope, OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&ScopeFifo<'scope>) -> R, + { + do_in_place_scope_fifo(Some(&self.registry), op) + } + + /// Spawns an asynchronous task in this thread-pool. This task will + /// run in the implicit, global scope, which means that it may outlast + /// the current stack frame -- therefore, it cannot capture any references + /// onto the stack (you will likely need a `move` closure). + /// + /// See also: [the `spawn()` function defined on scopes][spawn]. + /// + /// [spawn]: struct.Scope.html#method.spawn + pub fn spawn(&self, op: OP) + where + OP: FnOnce() + Send + 'static, + { + // We assert that `self.registry` has not terminated. + unsafe { spawn::spawn_in(op, &self.registry) } + } + + /// Spawns an asynchronous task in this thread-pool. This task will + /// run in the implicit, global scope, which means that it may outlast + /// the current stack frame -- therefore, it cannot capture any references + /// onto the stack (you will likely need a `move` closure). + /// + /// See also: [the `spawn_fifo()` function defined on scopes][spawn_fifo]. + /// + /// [spawn_fifo]: struct.ScopeFifo.html#method.spawn_fifo + pub fn spawn_fifo(&self, op: OP) + where + OP: FnOnce() + Send + 'static, + { + // We assert that `self.registry` has not terminated. + unsafe { spawn::spawn_fifo_in(op, &self.registry) } + } + + /// Spawns an asynchronous task on every thread in this thread-pool. This task + /// will run in the implicit, global scope, which means that it may outlast the + /// current stack frame -- therefore, it cannot capture any references onto the + /// stack (you will likely need a `move` closure). + pub fn spawn_broadcast(&self, op: OP) + where + OP: Fn(BroadcastContext<'_>) + Send + Sync + 'static, + { + // We assert that `self.registry` has not terminated. + unsafe { broadcast::spawn_broadcast_in(op, &self.registry) } + } + + /// Cooperatively yields execution to Rayon. + /// + /// This is similar to the general [`yield_now()`], but only if the current + /// thread is part of *this* thread pool. + /// + /// Returns `Some(Yield::Executed)` if anything was executed, `Some(Yield::Idle)` if + /// nothing was available, or `None` if the current thread is not part this pool. + pub fn yield_now(&self) -> Option { + let curr = self.registry.current_thread()?; + Some(curr.yield_now()) + } + + /// Cooperatively yields execution to local Rayon work. + /// + /// This is similar to the general [`yield_local()`], but only if the current + /// thread is part of *this* thread pool. + /// + /// Returns `Some(Yield::Executed)` if anything was executed, `Some(Yield::Idle)` if + /// nothing was available, or `None` if the current thread is not part this pool. + pub fn yield_local(&self) -> Option { + let curr = self.registry.current_thread()?; + Some(curr.yield_local()) + } + + pub(crate) fn wait_until_stopped(self) { + let registry = self.registry.clone(); + drop(self); + registry.wait_until_stopped(); + } +} + +impl Drop for ThreadPool { + fn drop(&mut self) { + self.registry.terminate(); + } +} + +impl fmt::Debug for ThreadPool { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("ThreadPool") + .field("num_threads", &self.current_num_threads()) + .field("id", &self.registry.id()) + .finish() + } +} + +/// If called from a Rayon worker thread, returns the index of that +/// thread within its current pool; if not called from a Rayon thread, +/// returns `None`. +/// +/// The index for a given thread will not change over the thread's +/// lifetime. However, multiple threads may share the same index if +/// they are in distinct thread-pools. +/// +/// See also: [the `ThreadPool::current_thread_index()` method]. +/// +/// [m]: struct.ThreadPool.html#method.current_thread_index +/// +/// # Future compatibility note +/// +/// Currently, every thread-pool (including the global +/// thread-pool) has a fixed number of threads, but this may +/// change in future Rayon versions (see [the `num_threads()` method +/// for details][snt]). In that case, the index for a +/// thread would not change during its lifetime, but thread +/// indices may wind up being reused if threads are terminated and +/// restarted. +/// +/// [snt]: struct.ThreadPoolBuilder.html#method.num_threads +#[inline] +pub fn current_thread_index() -> Option { + unsafe { + let curr = WorkerThread::current().as_ref()?; + Some(curr.index()) + } +} + +/// If called from a Rayon worker thread, indicates whether that +/// thread's local deque still has pending tasks. Otherwise, returns +/// `None`. For more information, see [the +/// `ThreadPool::current_thread_has_pending_tasks()` method][m]. +/// +/// [m]: struct.ThreadPool.html#method.current_thread_has_pending_tasks +#[inline] +pub fn current_thread_has_pending_tasks() -> Option { + unsafe { + let curr = WorkerThread::current().as_ref()?; + Some(!curr.local_deque_is_empty()) + } +} + +/// Cooperatively yields execution to Rayon. +/// +/// If the current thread is part of a rayon thread pool, this looks for a +/// single unit of pending work in the pool, then executes it. Completion of +/// that work might include nested work or further work stealing. +/// +/// This is similar to [`std::thread::yield_now()`], but does not literally make +/// that call. If you are implementing a polling loop, you may want to also +/// yield to the OS scheduler yourself if no Rayon work was found. +/// +/// Returns `Some(Yield::Executed)` if anything was executed, `Some(Yield::Idle)` if +/// nothing was available, or `None` if this thread is not part of any pool at all. +pub fn yield_now() -> Option { + unsafe { + let thread = WorkerThread::current().as_ref()?; + Some(thread.yield_now()) + } +} + +/// Cooperatively yields execution to local Rayon work. +/// +/// If the current thread is part of a rayon thread pool, this looks for a +/// single unit of pending work in this thread's queue, then executes it. +/// Completion of that work might include nested work or further work stealing. +/// +/// This is similar to [`yield_now()`], but does not steal from other threads. +/// +/// Returns `Some(Yield::Executed)` if anything was executed, `Some(Yield::Idle)` if +/// nothing was available, or `None` if this thread is not part of any pool at all. +pub fn yield_local() -> Option { + unsafe { + let thread = WorkerThread::current().as_ref()?; + Some(thread.yield_local()) + } +} + +/// Result of [`yield_now()`] or [`yield_local()`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Yield { + /// Work was found and executed. + Executed, + /// No available work was found. + Idle, +} diff --git a/compiler/rustc_thread_pool/src/thread_pool/test.rs b/compiler/rustc_thread_pool/src/thread_pool/test.rs new file mode 100644 index 0000000000000..88b36282d481d --- /dev/null +++ b/compiler/rustc_thread_pool/src/thread_pool/test.rs @@ -0,0 +1,418 @@ +#![cfg(test)] + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::channel; +use std::sync::{Arc, Mutex}; + +use crate::{join, Scope, ScopeFifo, ThreadPool, ThreadPoolBuilder}; + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate() { + let thread_pool = ThreadPoolBuilder::new().build().unwrap(); + thread_pool.install(|| { + panic!("Hello, world!"); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn workers_stop() { + let registry; + + { + // once we exit this block, thread-pool will be dropped + let thread_pool = ThreadPoolBuilder::new().num_threads(22).build().unwrap(); + registry = thread_pool.install(|| { + // do some work on these threads + join_a_lot(22); + + Arc::clone(&thread_pool.registry) + }); + assert_eq!(registry.num_threads(), 22); + } + + // once thread-pool is dropped, registry should terminate, which + // should lead to worker threads stopping + registry.wait_until_stopped(); +} + +fn join_a_lot(n: usize) { + if n > 0 { + join(|| join_a_lot(n - 1), || join_a_lot(n - 1)); + } +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn sleeper_stop() { + use std::{thread, time}; + + let registry; + + { + // once we exit this block, thread-pool will be dropped + let thread_pool = ThreadPoolBuilder::new().num_threads(22).build().unwrap(); + registry = Arc::clone(&thread_pool.registry); + + // Give time for at least some of the thread pool to fall asleep. + thread::sleep(time::Duration::from_secs(1)); + } + + // once thread-pool is dropped, registry should terminate, which + // should lead to worker threads stopping + registry.wait_until_stopped(); +} + +/// Creates a start/exit handler that increments an atomic counter. +fn count_handler() -> (Arc, impl Fn(usize)) { + let count = Arc::new(AtomicUsize::new(0)); + (Arc::clone(&count), move |_| { + count.fetch_add(1, Ordering::SeqCst); + }) +} + +/// Wait until a counter is no longer shared, then return its value. +fn wait_for_counter(mut counter: Arc) -> usize { + use std::{thread, time}; + + for _ in 0..60 { + counter = match Arc::try_unwrap(counter) { + Ok(counter) => return counter.into_inner(), + Err(counter) => { + thread::sleep(time::Duration::from_secs(1)); + counter + } + }; + } + + // That's too long! + panic!("Counter is still shared!"); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn failed_thread_stack() { + // Note: we first tried to force failure with a `usize::MAX` stack, but + // macOS and Windows weren't fazed, or at least didn't fail the way we want. + // They work with `isize::MAX`, but 32-bit platforms may feasibly allocate a + // 2GB stack, so it might not fail until the second thread. + let stack_size = ::std::isize::MAX as usize; + + let (start_count, start_handler) = count_handler(); + let (exit_count, exit_handler) = count_handler(); + let builder = ThreadPoolBuilder::new() + .num_threads(10) + .stack_size(stack_size) + .start_handler(start_handler) + .exit_handler(exit_handler); + + let pool = builder.build(); + assert!(pool.is_err(), "thread stack should have failed!"); + + // With such a huge stack, 64-bit will probably fail on the first thread; + // 32-bit might manage the first 2GB, but certainly fail the second. + let start_count = wait_for_counter(start_count); + assert!(start_count <= 1); + assert_eq!(start_count, wait_for_counter(exit_count)); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_thread_name() { + let (start_count, start_handler) = count_handler(); + let (exit_count, exit_handler) = count_handler(); + let builder = ThreadPoolBuilder::new() + .num_threads(10) + .start_handler(start_handler) + .exit_handler(exit_handler) + .thread_name(|i| { + if i >= 5 { + panic!(); + } + format!("panic_thread_name#{}", i) + }); + + let pool = crate::unwind::halt_unwinding(|| builder.build()); + assert!(pool.is_err(), "thread-name panic should propagate!"); + + // Assuming they're created in order, threads 0 through 4 should have + // been started already, and then terminated by the panic. + assert_eq!(5, wait_for_counter(start_count)); + assert_eq!(5, wait_for_counter(exit_count)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn self_install() { + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + // If the inner `install` blocks, then nothing will actually run it! + assert!(pool.install(|| pool.install(|| true))); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mutual_install() { + let pool1 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + let ok = pool1.install(|| { + // This creates a dependency from `pool1` -> `pool2` + pool2.install(|| { + // This creates a dependency from `pool2` -> `pool1` + pool1.install(|| { + // If they blocked on inter-pool installs, there would be no + // threads left to run this! + true + }) + }) + }); + assert!(ok); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mutual_install_sleepy() { + use std::{thread, time}; + + let pool1 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + let ok = pool1.install(|| { + // This creates a dependency from `pool1` -> `pool2` + pool2.install(|| { + // Give `pool1` time to fall asleep. + thread::sleep(time::Duration::from_secs(1)); + + // This creates a dependency from `pool2` -> `pool1` + pool1.install(|| { + // Give `pool2` time to fall asleep. + thread::sleep(time::Duration::from_secs(1)); + + // If they blocked on inter-pool installs, there would be no + // threads left to run this! + true + }) + }) + }); + assert!(ok); +} + +#[test] +#[allow(deprecated)] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn check_thread_pool_new() { + let pool = ThreadPool::new(crate::Configuration::new().num_threads(22)).unwrap(); + assert_eq!(pool.current_num_threads(), 22); +} + +macro_rules! test_scope_order { + ($scope:ident => $spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let vec = Mutex::new(vec![]); + pool.$scope(|scope| { + let vec = &vec; + for i in 0..10 { + scope.$spawn(move |_| { + vec.lock().unwrap().push(i); + }); + } + }); + vec.into_inner().unwrap() + }) + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_lifo_order() { + let vec = test_scope_order!(scope => spawn); + let expected: Vec = (0..10).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_fifo_order() { + let vec = test_scope_order!(scope_fifo => spawn_fifo); + let expected: Vec = (0..10).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +macro_rules! test_spawn_order { + ($spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = &builder.build().unwrap(); + let (tx, rx) = channel(); + pool.install(move || { + for i in 0..10 { + let tx = tx.clone(); + pool.$spawn(move || { + tx.send(i).unwrap(); + }); + } + }); + rx.iter().collect::>() + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_lifo_order() { + let vec = test_spawn_order!(spawn); + let expected: Vec = (0..10).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_fifo_order() { + let vec = test_spawn_order!(spawn_fifo); + let expected: Vec = (0..10).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_scopes() { + // Create matching scopes for every thread pool. + fn nest<'scope, OP>(pools: &[ThreadPool], scopes: Vec<&Scope<'scope>>, op: OP) + where + OP: FnOnce(&[&Scope<'scope>]) + Send, + { + if let Some((pool, tail)) = pools.split_first() { + pool.scope(move |s| { + // This move reduces the reference lifetimes by variance to match s, + // but the actual scopes are still tied to the invariant 'scope. + let mut scopes = scopes; + scopes.push(s); + nest(tail, scopes, op) + }) + } else { + (op)(&scopes) + } + } + + let pools: Vec<_> = (0..10) + .map(|_| ThreadPoolBuilder::new().num_threads(1).build().unwrap()) + .collect(); + + let counter = AtomicUsize::new(0); + nest(&pools, vec![], |scopes| { + for &s in scopes { + s.spawn(|_| { + // Our 'scope lets us borrow the counter in every pool. + counter.fetch_add(1, Ordering::Relaxed); + }); + } + }); + assert_eq!(counter.into_inner(), pools.len()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_fifo_scopes() { + // Create matching fifo scopes for every thread pool. + fn nest<'scope, OP>(pools: &[ThreadPool], scopes: Vec<&ScopeFifo<'scope>>, op: OP) + where + OP: FnOnce(&[&ScopeFifo<'scope>]) + Send, + { + if let Some((pool, tail)) = pools.split_first() { + pool.scope_fifo(move |s| { + // This move reduces the reference lifetimes by variance to match s, + // but the actual scopes are still tied to the invariant 'scope. + let mut scopes = scopes; + scopes.push(s); + nest(tail, scopes, op) + }) + } else { + (op)(&scopes) + } + } + + let pools: Vec<_> = (0..10) + .map(|_| ThreadPoolBuilder::new().num_threads(1).build().unwrap()) + .collect(); + + let counter = AtomicUsize::new(0); + nest(&pools, vec![], |scopes| { + for &s in scopes { + s.spawn_fifo(|_| { + // Our 'scope lets us borrow the counter in every pool. + counter.fetch_add(1, Ordering::Relaxed); + }); + } + }); + assert_eq!(counter.into_inner(), pools.len()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn in_place_scope_no_deadlock() { + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let (tx, rx) = channel(); + let rx_ref = ℞ + pool.in_place_scope(move |s| { + // With regular scopes this closure would never run because this scope op + // itself would block the only worker thread. + s.spawn(move |_| { + tx.send(()).unwrap(); + }); + rx_ref.recv().unwrap(); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn in_place_scope_fifo_no_deadlock() { + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let (tx, rx) = channel(); + let rx_ref = ℞ + pool.in_place_scope_fifo(move |s| { + // With regular scopes this closure would never run because this scope op + // itself would block the only worker thread. + s.spawn_fifo(move |_| { + tx.send(()).unwrap(); + }); + rx_ref.recv().unwrap(); + }); +} + +#[test] +fn yield_now_to_spawn() { + let (tx, rx) = channel(); + + // Queue a regular spawn. + crate::spawn(move || tx.send(22).unwrap()); + + // The single-threaded fallback mode (for wasm etc.) won't + // get a chance to run the spawn if we never yield to it. + crate::registry::in_worker(move |_, _| { + crate::yield_now(); + }); + + // The spawn **must** have started by now, but we still might have to wait + // for it to finish if a different thread stole it first. + assert_eq!(22, rx.recv().unwrap()); +} + +#[test] +fn yield_local_to_spawn() { + let (tx, rx) = channel(); + + // Queue a regular spawn. + crate::spawn(move || tx.send(22).unwrap()); + + // The single-threaded fallback mode (for wasm etc.) won't + // get a chance to run the spawn if we never yield to it. + crate::registry::in_worker(move |_, _| { + crate::yield_local(); + }); + + // The spawn **must** have started by now, but we still might have to wait + // for it to finish if a different thread stole it first. + assert_eq!(22, rx.recv().unwrap()); +} diff --git a/compiler/rustc_thread_pool/src/tlv.rs b/compiler/rustc_thread_pool/src/tlv.rs new file mode 100644 index 0000000000000..ce22f7aa0ce70 --- /dev/null +++ b/compiler/rustc_thread_pool/src/tlv.rs @@ -0,0 +1,31 @@ +//! Allows access to the Rayon's thread local value +//! which is preserved when moving jobs across threads + +use std::{cell::Cell, ptr}; + +thread_local!(pub static TLV: Cell<*const ()> = const { Cell::new(ptr::null()) }); + +#[derive(Copy, Clone)] +pub(crate) struct Tlv(pub(crate) *const ()); + +impl Tlv { + #[inline] + pub(crate) fn null() -> Self { + Self(ptr::null()) + } +} + +unsafe impl Sync for Tlv {} +unsafe impl Send for Tlv {} + +/// Sets the current thread-local value +#[inline] +pub(crate) fn set(value: Tlv) { + TLV.with(|tlv| tlv.set(value.0)); +} + +/// Returns the current thread-local value +#[inline] +pub(crate) fn get() -> Tlv { + TLV.with(|tlv| Tlv(tlv.get())) +} diff --git a/compiler/rustc_thread_pool/src/unwind.rs b/compiler/rustc_thread_pool/src/unwind.rs new file mode 100644 index 0000000000000..9671fa5782147 --- /dev/null +++ b/compiler/rustc_thread_pool/src/unwind.rs @@ -0,0 +1,31 @@ +//! Package up unwind recovery. Note that if you are in some sensitive +//! place, you can use the `AbortIfPanic` helper to protect against +//! accidental panics in the rayon code itself. + +use std::any::Any; +use std::panic::{self, AssertUnwindSafe}; +use std::thread; + +/// Executes `f` and captures any panic, translating that panic into a +/// `Err` result. The assumption is that any panic will be propagated +/// later with `resume_unwinding`, and hence `f` can be treated as +/// exception safe. +pub(super) fn halt_unwinding(func: F) -> thread::Result +where + F: FnOnce() -> R, +{ + panic::catch_unwind(AssertUnwindSafe(func)) +} + +pub(super) fn resume_unwinding(payload: Box) -> ! { + panic::resume_unwind(payload) +} + +pub(super) struct AbortIfPanic; + +impl Drop for AbortIfPanic { + fn drop(&mut self) { + eprintln!("Rayon: detected unexpected panic; aborting"); + ::std::process::abort(); + } +} diff --git a/compiler/rustc_thread_pool/src/worker_local.rs b/compiler/rustc_thread_pool/src/worker_local.rs new file mode 100644 index 0000000000000..85d51687c1902 --- /dev/null +++ b/compiler/rustc_thread_pool/src/worker_local.rs @@ -0,0 +1,78 @@ +use crate::registry::{Registry, WorkerThread}; +use std::fmt; +use std::ops::Deref; +use std::sync::Arc; + +#[repr(align(64))] +#[derive(Debug)] +struct CacheAligned(T); + +/// Holds worker-locals values for each thread in a thread pool. +/// You can only access the worker local value through the Deref impl +/// on the thread pool it was constructed on. It will panic otherwise +pub struct WorkerLocal { + locals: Vec>, + registry: Arc, +} + +/// We prevent concurrent access to the underlying value in the +/// Deref impl, thus any values safe to send across threads can +/// be used with WorkerLocal. +unsafe impl Sync for WorkerLocal {} + +impl WorkerLocal { + /// Creates a new worker local where the `initial` closure computes the + /// value this worker local should take for each thread in the thread pool. + #[inline] + pub fn new T>(mut initial: F) -> WorkerLocal { + let registry = Registry::current(); + WorkerLocal { + locals: (0..registry.num_threads()) + .map(|i| CacheAligned(initial(i))) + .collect(), + registry, + } + } + + /// Returns the worker-local value for each thread + #[inline] + pub fn into_inner(self) -> Vec { + self.locals.into_iter().map(|c| c.0).collect() + } + + fn current(&self) -> &T { + unsafe { + let worker_thread = WorkerThread::current(); + if worker_thread.is_null() + || &*(*worker_thread).registry as *const _ != &*self.registry as *const _ + { + panic!("WorkerLocal can only be used on the thread pool it was created on") + } + &self.locals[(*worker_thread).index].0 + } + } +} + +impl WorkerLocal> { + /// Joins the elements of all the worker locals into one Vec + pub fn join(self) -> Vec { + self.into_inner().into_iter().flat_map(|v| v).collect() + } +} + +impl fmt::Debug for WorkerLocal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("WorkerLocal") + .field("registry", &self.registry.id()) + .finish() + } +} + +impl Deref for WorkerLocal { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + self.current() + } +} diff --git a/compiler/rustc_thread_pool/tests/double_init_fail.rs b/compiler/rustc_thread_pool/tests/double_init_fail.rs new file mode 100644 index 0000000000000..15915304ddccc --- /dev/null +++ b/compiler/rustc_thread_pool/tests/double_init_fail.rs @@ -0,0 +1,15 @@ +use rayon_core::ThreadPoolBuilder; +use std::error::Error; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn double_init_fail() { + let result1 = ThreadPoolBuilder::new().build_global(); + assert!(result1.is_ok()); + let err = ThreadPoolBuilder::new().build_global().unwrap_err(); + assert!(err.source().is_none()); + assert_eq!( + err.to_string(), + "The global thread pool has already been initialized.", + ); +} diff --git a/compiler/rustc_thread_pool/tests/init_zero_threads.rs b/compiler/rustc_thread_pool/tests/init_zero_threads.rs new file mode 100644 index 0000000000000..3c1ad251c7e02 --- /dev/null +++ b/compiler/rustc_thread_pool/tests/init_zero_threads.rs @@ -0,0 +1,10 @@ +use rayon_core::ThreadPoolBuilder; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn init_zero_threads() { + ThreadPoolBuilder::new() + .num_threads(0) + .build_global() + .unwrap(); +} diff --git a/compiler/rustc_thread_pool/tests/scope_join.rs b/compiler/rustc_thread_pool/tests/scope_join.rs new file mode 100644 index 0000000000000..9d88133bc5b69 --- /dev/null +++ b/compiler/rustc_thread_pool/tests/scope_join.rs @@ -0,0 +1,45 @@ +/// Test that one can emulate join with `scope`: +fn pseudo_join(f: F, g: G) +where + F: FnOnce() + Send, + G: FnOnce() + Send, +{ + rayon_core::scope(|s| { + s.spawn(|_| g()); + f(); + }); +} + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (lo, hi) = v.split_at_mut(mid); + pseudo_join(|| quick_sort(lo), || quick_sort(hi)); +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn is_sorted(v: &[T]) -> bool { + (1..v.len()).all(|i| v[i - 1] <= v[i]) +} + +#[test] +fn scope_join() { + let mut v: Vec = (0..256).rev().collect(); + quick_sort(&mut v); + assert!(is_sorted(&v)); +} diff --git a/compiler/rustc_thread_pool/tests/scoped_threadpool.rs b/compiler/rustc_thread_pool/tests/scoped_threadpool.rs new file mode 100644 index 0000000000000..932147179f5c5 --- /dev/null +++ b/compiler/rustc_thread_pool/tests/scoped_threadpool.rs @@ -0,0 +1,99 @@ +use crossbeam_utils::thread; +use rayon_core::ThreadPoolBuilder; + +#[derive(PartialEq, Eq, Debug)] +struct Local(i32); + +scoped_tls::scoped_thread_local!(static LOCAL: Local); + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn missing_scoped_tls() { + LOCAL.set(&Local(42), || { + let pool = ThreadPoolBuilder::new() + .build() + .expect("thread pool created"); + + // `LOCAL` is not set in the pool. + pool.install(|| { + assert!(!LOCAL.is_set()); + }); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_scoped_tls_threadpool() { + LOCAL.set(&Local(42), || { + LOCAL.with(|x| { + thread::scope(|scope| { + let pool = ThreadPoolBuilder::new() + .spawn_handler(move |thread| { + scope + .builder() + .spawn(move |_| { + // Borrow the same local value in the thread pool. + LOCAL.set(x, || thread.run()) + }) + .map(|_| ()) + }) + .build() + .expect("thread pool created"); + + // The pool matches our local value. + pool.install(|| { + assert!(LOCAL.is_set()); + LOCAL.with(|y| { + assert_eq!(x, y); + }); + }); + + // If we change our local value, the pool is not affected. + LOCAL.set(&Local(-1), || { + pool.install(|| { + assert!(LOCAL.is_set()); + LOCAL.with(|y| { + assert_eq!(x, y); + }); + }); + }); + }) + .expect("scope threads ok"); + // `thread::scope` will wait for the threads to exit before returning. + }); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn build_scoped_tls_threadpool() { + LOCAL.set(&Local(42), || { + LOCAL.with(|x| { + ThreadPoolBuilder::new() + .build_scoped( + move |thread| LOCAL.set(x, || thread.run()), + |pool| { + // The pool matches our local value. + pool.install(|| { + assert!(LOCAL.is_set()); + LOCAL.with(|y| { + assert_eq!(x, y); + }); + }); + + // If we change our local value, the pool is not affected. + LOCAL.set(&Local(-1), || { + pool.install(|| { + assert!(LOCAL.is_set()); + LOCAL.with(|y| { + assert_eq!(x, y); + }); + }); + }); + }, + ) + .expect("thread pool created"); + // Internally, `std::thread::scope` will wait for the threads to exit before returning. + }); + }); +} diff --git a/compiler/rustc_thread_pool/tests/simple_panic.rs b/compiler/rustc_thread_pool/tests/simple_panic.rs new file mode 100644 index 0000000000000..2564482a47e5d --- /dev/null +++ b/compiler/rustc_thread_pool/tests/simple_panic.rs @@ -0,0 +1,7 @@ +use rayon_core::join; + +#[test] +#[should_panic(expected = "should panic")] +fn simple_panic() { + join(|| {}, || panic!("should panic")); +} diff --git a/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs b/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs new file mode 100644 index 0000000000000..a6494069212ca --- /dev/null +++ b/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs @@ -0,0 +1,98 @@ +use rayon_core::ThreadPoolBuilder; + +use std::env; +use std::process::{Command, ExitStatus, Stdio}; + +#[cfg(target_os = "linux")] +use std::os::unix::process::ExitStatusExt; + +fn force_stack_overflow(depth: u32) { + let mut buffer = [0u8; 1024 * 1024]; + #[allow(clippy::incompatible_msrv)] + std::hint::black_box(&mut buffer); + if depth > 0 { + force_stack_overflow(depth - 1); + } +} + +#[cfg(unix)] +fn disable_core() { + unsafe { + libc::setrlimit( + libc::RLIMIT_CORE, + &libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }, + ); + } +} + +#[cfg(unix)] +fn overflow_code() -> Option { + None +} + +#[cfg(windows)] +fn overflow_code() -> Option { + use std::os::windows::process::ExitStatusExt; + + ExitStatus::from_raw(0xc00000fd /*STATUS_STACK_OVERFLOW*/).code() +} + +#[test] +#[cfg_attr(not(any(unix, windows)), ignore)] +fn stack_overflow_crash() { + // First check that the recursive call actually causes a stack overflow, + // and does not get optimized away. + let status = run_ignored("run_with_small_stack"); + assert!(!status.success()); + #[cfg(any(unix, windows))] + assert_eq!(status.code(), overflow_code()); + #[cfg(target_os = "linux")] + assert!(matches!( + status.signal(), + Some(libc::SIGABRT | libc::SIGSEGV) + )); + + // Now run with a larger stack and verify correct operation. + let status = run_ignored("run_with_large_stack"); + assert_eq!(status.code(), Some(0)); + #[cfg(target_os = "linux")] + assert_eq!(status.signal(), None); +} + +fn run_ignored(test: &str) -> ExitStatus { + Command::new(env::current_exe().unwrap()) + .arg("--ignored") + .arg("--exact") + .arg(test) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap() +} + +#[test] +#[ignore] +fn run_with_small_stack() { + run_with_stack(8); +} + +#[test] +#[ignore] +fn run_with_large_stack() { + run_with_stack(48); +} + +fn run_with_stack(stack_size_in_mb: usize) { + let pool = ThreadPoolBuilder::new() + .stack_size(stack_size_in_mb * 1024 * 1024) + .build() + .unwrap(); + pool.install(|| { + #[cfg(unix)] + disable_core(); + force_stack_overflow(32); + }); +} From 0b9b1df0064396708a5e5ca27fd010ae3ad3a305 Mon Sep 17 00:00:00 2001 From: "Celina G. Val" Date: Wed, 11 Jun 2025 11:12:32 -0700 Subject: [PATCH 07/32] Fix format and tidy for code moved from rayon --- .../rustc_thread_pool/src/broadcast/mod.rs | 17 ++-- .../src/broadcast/{test.rs => tests.rs} | 5 +- compiler/rustc_thread_pool/src/job.rs | 26 +++-- compiler/rustc_thread_pool/src/join/mod.rs | 17 ++-- .../src/join/{test.rs => tests.rs} | 5 +- compiler/rustc_thread_pool/src/latch.rs | 54 +++------- compiler/rustc_thread_pool/src/lib.rs | 58 ++++------- compiler/rustc_thread_pool/src/private.rs | 2 +- compiler/rustc_thread_pool/src/registry.rs | 99 +++++++------------ compiler/rustc_thread_pool/src/scope/mod.rs | 34 +++---- .../src/scope/{test.rs => tests.rs} | 35 ++----- .../rustc_thread_pool/src/sleep/README.md | 2 +- .../rustc_thread_pool/src/sleep/counters.rs | 8 +- compiler/rustc_thread_pool/src/sleep/mod.rs | 26 ++--- compiler/rustc_thread_pool/src/spawn/mod.rs | 17 ++-- .../src/spawn/{test.rs => tests.rs} | 19 +--- .../src/{test.rs => tests.rs} | 11 +-- .../rustc_thread_pool/src/thread_pool/mod.rs | 15 ++- .../src/thread_pool/{test.rs => tests.rs} | 12 +-- compiler/rustc_thread_pool/src/tlv.rs | 3 +- .../rustc_thread_pool/src/worker_local.rs | 11 +-- .../tests/double_init_fail.rs | 8 +- .../tests/init_zero_threads.rs | 5 +- .../tests/scoped_threadpool.rs | 4 +- .../tests/stack_overflow_crash.rs | 25 ++--- 25 files changed, 190 insertions(+), 328 deletions(-) rename compiler/rustc_thread_pool/src/broadcast/{test.rs => tests.rs} (99%) rename compiler/rustc_thread_pool/src/join/{test.rs => tests.rs} (99%) rename compiler/rustc_thread_pool/src/scope/{test.rs => tests.rs} (96%) rename compiler/rustc_thread_pool/src/spawn/{test.rs => tests.rs} (95%) rename compiler/rustc_thread_pool/src/{test.rs => tests.rs} (96%) rename compiler/rustc_thread_pool/src/thread_pool/{test.rs => tests.rs} (97%) diff --git a/compiler/rustc_thread_pool/src/broadcast/mod.rs b/compiler/rustc_thread_pool/src/broadcast/mod.rs index 442891f2d2876..c2b0d47f82995 100644 --- a/compiler/rustc_thread_pool/src/broadcast/mod.rs +++ b/compiler/rustc_thread_pool/src/broadcast/mod.rs @@ -1,10 +1,11 @@ -use crate::job::{ArcJob, StackJob}; -use crate::latch::{CountLatch, LatchRef}; -use crate::registry::{Registry, WorkerThread}; use std::fmt; use std::marker::PhantomData; use std::sync::Arc; +use crate::job::{ArcJob, StackJob}; +use crate::latch::{CountLatch, LatchRef}; +use crate::registry::{Registry, WorkerThread}; + mod test; /// Executes `op` within every thread in the current threadpool. If this is @@ -53,10 +54,7 @@ impl<'a> BroadcastContext<'a> { pub(super) fn with(f: impl FnOnce(BroadcastContext<'_>) -> R) -> R { let worker_thread = WorkerThread::current(); assert!(!worker_thread.is_null()); - f(BroadcastContext { - worker: unsafe { &*worker_thread }, - _marker: PhantomData, - }) + f(BroadcastContext { worker: unsafe { &*worker_thread }, _marker: PhantomData }) } /// Our index amongst the broadcast threads (ranges from `0..self.num_threads()`). @@ -108,9 +106,8 @@ where let current_thread = WorkerThread::current().as_ref(); let tlv = crate::tlv::get(); let latch = CountLatch::with_count(n_threads, current_thread); - let jobs: Vec<_> = (0..n_threads) - .map(|_| StackJob::new(tlv, &f, LatchRef::new(&latch))) - .collect(); + let jobs: Vec<_> = + (0..n_threads).map(|_| StackJob::new(tlv, &f, LatchRef::new(&latch))).collect(); let job_refs = jobs.iter().map(|job| job.as_job_ref()); registry.inject_broadcast(job_refs); diff --git a/compiler/rustc_thread_pool/src/broadcast/test.rs b/compiler/rustc_thread_pool/src/broadcast/tests.rs similarity index 99% rename from compiler/rustc_thread_pool/src/broadcast/test.rs rename to compiler/rustc_thread_pool/src/broadcast/tests.rs index 00ab4ad7fe41f..fac8b8ad46660 100644 --- a/compiler/rustc_thread_pool/src/broadcast/test.rs +++ b/compiler/rustc_thread_pool/src/broadcast/tests.rs @@ -1,11 +1,12 @@ #![cfg(test)] -use crate::ThreadPoolBuilder; +use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::channel; -use std::sync::Arc; use std::{thread, time}; +use crate::ThreadPoolBuilder; + #[test] fn broadcast_global() { let v = crate::broadcast(|ctx| ctx.index()); diff --git a/compiler/rustc_thread_pool/src/job.rs b/compiler/rustc_thread_pool/src/job.rs index 394c7576b2cb2..3241914ba81c7 100644 --- a/compiler/rustc_thread_pool/src/job.rs +++ b/compiler/rustc_thread_pool/src/job.rs @@ -1,13 +1,14 @@ -use crate::latch::Latch; -use crate::tlv; -use crate::tlv::Tlv; -use crate::unwind; -use crossbeam_deque::{Injector, Steal}; use std::any::Any; use std::cell::UnsafeCell; use std::mem; use std::sync::Arc; +use crossbeam_deque::{Injector, Steal}; + +use crate::latch::Latch; +use crate::tlv::Tlv; +use crate::{tlv, unwind}; + pub(super) enum JobResult { None, Ok(T), @@ -29,7 +30,7 @@ pub(super) trait Job { /// Effectively a Job trait object. Each JobRef **must** be executed /// exactly once, or else data may leak. /// -/// Internally, we store the job's data in a `*const ()` pointer. The +/// Internally, we store the job's data in a `*const ()` pointer. The /// true type is something like `*const StackJob<...>`, but we hide /// it. We also carry the "execute fn" from the `Job` trait. pub(super) struct JobRef { @@ -48,10 +49,7 @@ impl JobRef { T: Job, { // erase types: - JobRef { - pointer: data as *const (), - execute_fn: ::execute, - } + JobRef { pointer: data as *const (), execute_fn: ::execute } } /// Returns an opaque handle that can be saved and compared, @@ -69,7 +67,7 @@ impl JobRef { /// A job that will be owned by a stack slot. This means that when it /// executes it need not free any heap data, the cleanup occurs when -/// the stack frame is later popped. The function parameter indicates +/// the stack frame is later popped. The function parameter indicates /// `true` if the job was stolen -- executed on a different thread. pub(super) struct StackJob where @@ -248,13 +246,11 @@ pub(super) struct JobFifo { impl JobFifo { pub(super) fn new() -> Self { - JobFifo { - inner: Injector::new(), - } + JobFifo { inner: Injector::new() } } pub(super) unsafe fn push(&self, job_ref: JobRef) -> JobRef { - // A little indirection ensures that spawns are always prioritized in FIFO order. The + // A little indirection ensures that spawns are always prioritized in FIFO order. The // jobs in a thread's deque may be popped from the back (LIFO) or stolen from the front // (FIFO), but either way they will end up popping from the front of this queue. self.inner.push(job_ref); diff --git a/compiler/rustc_thread_pool/src/join/mod.rs b/compiler/rustc_thread_pool/src/join/mod.rs index 032eec9c4c84d..798a8347d798c 100644 --- a/compiler/rustc_thread_pool/src/join/mod.rs +++ b/compiler/rustc_thread_pool/src/join/mod.rs @@ -1,11 +1,10 @@ +use std::any::Any; + use crate::job::StackJob; use crate::latch::SpinLatch; use crate::registry::{self, WorkerThread}; use crate::tlv::{self, Tlv}; -use crate::unwind; -use std::any::Any; - -use crate::FnContext; +use crate::{FnContext, unwind}; #[cfg(test)] mod test; @@ -22,7 +21,7 @@ mod test; /// it. /// /// When `join` is called from outside the thread pool, the calling -/// thread will block while the closures execute in the pool. When +/// thread will block while the closures execute in the pool. When /// `join` is called within the pool, the calling thread still actively /// participates in the thread pool. It will begin by executing closure /// A (on the current thread). While it is doing that, it will advertise @@ -80,13 +79,13 @@ mod test; /// CPU-bound tasks that do not perform I/O or other blocking /// operations. If you do perform I/O, and that I/O should block /// (e.g., waiting for a network request), the overall performance may -/// be poor. Moreover, if you cause one closure to be blocked waiting +/// be poor. Moreover, if you cause one closure to be blocked waiting /// on another (for example, using a channel), that could lead to a /// deadlock. /// /// # Panics /// -/// No matter what happens, both closures will always be executed. If +/// No matter what happens, both closures will always be executed. If /// a single closure panics, whether it be the first or second /// closure, that panic will be propagated and hence `join()` will /// panic with the same panic value. If both closures panic, `join()` @@ -109,7 +108,7 @@ where /// Identical to `join`, except that the closures have a parameter /// that provides context for the way the closure has been called, /// especially indicating whether they're executing on a different -/// thread than where `join_context` was called. This will occur if +/// thread than where `join_context` was called. This will occur if /// the second job is stolen by a different thread, or if /// `join_context` was called from outside the thread pool to begin /// with. @@ -148,7 +147,7 @@ where }; // Now that task A has finished, try to pop job B from the - // local stack. It may already have been popped by job A; it + // local stack. It may already have been popped by job A; it // may also have been stolen. There may also be some tasks // pushed on top of it in the stack, and we will have to pop // those off to get to it. diff --git a/compiler/rustc_thread_pool/src/join/test.rs b/compiler/rustc_thread_pool/src/join/tests.rs similarity index 99% rename from compiler/rustc_thread_pool/src/join/test.rs rename to compiler/rustc_thread_pool/src/join/tests.rs index 03f4ab4478d40..9df99072c3a10 100644 --- a/compiler/rustc_thread_pool/src/join/test.rs +++ b/compiler/rustc_thread_pool/src/join/tests.rs @@ -1,11 +1,12 @@ //! Tests for the join code. -use super::*; -use crate::ThreadPoolBuilder; use rand::distr::StandardUniform; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; +use super::*; +use crate::ThreadPoolBuilder; + fn quick_sort(v: &mut [T]) { if v.len() <= 1 { return; diff --git a/compiler/rustc_thread_pool/src/latch.rs b/compiler/rustc_thread_pool/src/latch.rs index 8903942a8ce36..f2f806e01845f 100644 --- a/compiler/rustc_thread_pool/src/latch.rs +++ b/compiler/rustc_thread_pool/src/latch.rs @@ -28,7 +28,7 @@ use crate::registry::{Registry, WorkerThread}; /// - Once `probe()` returns true, all memory effects from the `set()` /// are visible (in other words, the set should synchronize-with /// the probe). -/// - Once `set()` occurs, the next `probe()` *will* observe it. This +/// - Once `set()` occurs, the next `probe()` *will* observe it. This /// typically requires a seq-cst ordering. See [the "tickle-then-get-sleepy" scenario in the sleep /// README](/src/sleep/README.md#tickle-then-get-sleepy) for details. pub(super) trait Latch { @@ -78,9 +78,7 @@ pub(super) struct CoreLatch { impl CoreLatch { #[inline] fn new() -> Self { - Self { - state: AtomicUsize::new(0), - } + Self { state: AtomicUsize::new(0) } } /// Invoked by owning thread as it prepares to sleep. Returns true @@ -88,9 +86,7 @@ impl CoreLatch { /// latch was set in the meantime. #[inline] pub(super) fn get_sleepy(&self) -> bool { - self.state - .compare_exchange(UNSET, SLEEPY, Ordering::SeqCst, Ordering::Relaxed) - .is_ok() + self.state.compare_exchange(UNSET, SLEEPY, Ordering::SeqCst, Ordering::Relaxed).is_ok() } /// Invoked by owning thread as it falls asleep sleep. Returns @@ -98,9 +94,7 @@ impl CoreLatch { /// was set in the meantime. #[inline] pub(super) fn fall_asleep(&self) -> bool { - self.state - .compare_exchange(SLEEPY, SLEEPING, Ordering::SeqCst, Ordering::Relaxed) - .is_ok() + self.state.compare_exchange(SLEEPY, SLEEPING, Ordering::SeqCst, Ordering::Relaxed).is_ok() } /// Invoked by owning thread as it falls asleep sleep. Returns @@ -110,8 +104,7 @@ impl CoreLatch { pub(super) fn wake_up(&self) { if !self.probe() { let _ = - self.state - .compare_exchange(SLEEPING, UNSET, Ordering::SeqCst, Ordering::Relaxed); + self.state.compare_exchange(SLEEPING, UNSET, Ordering::SeqCst, Ordering::Relaxed); } } @@ -166,15 +159,12 @@ impl<'r> SpinLatch<'r> { } } - /// Creates a new spin latch for cross-threadpool blocking. Notably, we + /// Creates a new spin latch for cross-threadpool blocking. Notably, we /// need to make sure the registry is kept alive after setting, so we can /// safely call the notification. #[inline] pub(super) fn cross(thread: &'r WorkerThread) -> SpinLatch<'r> { - SpinLatch { - cross: true, - ..SpinLatch::new(thread) - } + SpinLatch { cross: true, ..SpinLatch::new(thread) } } #[inline] @@ -235,10 +225,7 @@ pub(super) struct LockLatch { impl LockLatch { #[inline] pub(super) fn new() -> LockLatch { - LockLatch { - m: Mutex::new(false), - v: Condvar::new(), - } + LockLatch { m: Mutex::new(false), v: Condvar::new() } } /// Block until latch is set, then resets this lock latch so it can be reused again. @@ -288,9 +275,7 @@ pub(super) struct OnceLatch { impl OnceLatch { #[inline] pub(super) fn new() -> OnceLatch { - Self { - core_latch: CoreLatch::new(), - } + Self { core_latch: CoreLatch::new() } } /// Set the latch, then tickle the specific worker thread, @@ -372,9 +357,7 @@ impl CountLatch { registry: Arc::clone(owner.registry()), worker_index: owner.index(), }, - None => CountLatchKind::Blocking { - latch: LockLatch::new(), - }, + None => CountLatchKind::Blocking { latch: LockLatch::new() }, }, } } @@ -387,11 +370,7 @@ impl CountLatch { pub(super) fn wait(&self, owner: Option<&WorkerThread>) { match &self.kind { - CountLatchKind::Stealing { - latch, - registry, - worker_index, - } => unsafe { + CountLatchKind::Stealing { latch, registry, worker_index } => unsafe { let owner = owner.expect("owner thread"); debug_assert_eq!(registry.id(), owner.registry().id()); debug_assert_eq!(*worker_index, owner.index()); @@ -409,11 +388,7 @@ impl Latch for CountLatch { // NOTE: Once we call `set` on the internal `latch`, // the target may proceed and invalidate `this`! match (*this).kind { - CountLatchKind::Stealing { - ref latch, - ref registry, - worker_index, - } => { + CountLatchKind::Stealing { ref latch, ref registry, worker_index } => { let registry = Arc::clone(registry); if CoreLatch::set(latch) { registry.notify_worker_latch_is_set(worker_index); @@ -433,10 +408,7 @@ pub(super) struct LatchRef<'a, L> { impl LatchRef<'_, L> { pub(super) fn new(inner: &L) -> LatchRef<'_, L> { - LatchRef { - inner, - marker: PhantomData, - } + LatchRef { inner, marker: PhantomData } } } diff --git a/compiler/rustc_thread_pool/src/lib.rs b/compiler/rustc_thread_pool/src/lib.rs index 72064547e526e..179d63ed668a1 100644 --- a/compiler/rustc_thread_pool/src/lib.rs +++ b/compiler/rustc_thread_pool/src/lib.rs @@ -57,20 +57,17 @@ //! //! While we strive to keep `rayon-core` semver-compatible, it's still //! possible to arrive at this situation if different crates have overly -//! restrictive tilde or inequality requirements for `rayon-core`. The +//! restrictive tilde or inequality requirements for `rayon-core`. The //! conflicting requirements will need to be resolved before the build will //! succeed. #![warn(rust_2018_idioms)] use std::any::Any; -use std::env; use std::error::Error; -use std::fmt; -use std::io; use std::marker::PhantomData; use std::str::FromStr; -use std::thread; +use std::{env, fmt, io, thread}; #[macro_use] mod private; @@ -92,20 +89,18 @@ mod test; pub mod tlv; -pub use self::broadcast::{broadcast, spawn_broadcast, BroadcastContext}; -pub use self::join::{join, join_context}; -pub use self::registry::ThreadBuilder; -pub use self::registry::{mark_blocked, mark_unblocked, Registry}; -pub use self::scope::{in_place_scope, scope, Scope}; -pub use self::scope::{in_place_scope_fifo, scope_fifo, ScopeFifo}; -pub use self::spawn::{spawn, spawn_fifo}; -pub use self::thread_pool::current_thread_has_pending_tasks; -pub use self::thread_pool::current_thread_index; -pub use self::thread_pool::ThreadPool; -pub use self::thread_pool::{yield_local, yield_now, Yield}; pub use worker_local::WorkerLocal; +pub use self::broadcast::{BroadcastContext, broadcast, spawn_broadcast}; +pub use self::join::{join, join_context}; use self::registry::{CustomSpawn, DefaultSpawn, ThreadSpawn}; +pub use self::registry::{Registry, ThreadBuilder, mark_blocked, mark_unblocked}; +pub use self::scope::{Scope, ScopeFifo, in_place_scope, in_place_scope_fifo, scope, scope_fifo}; +pub use self::spawn::{spawn, spawn_fifo}; +pub use self::thread_pool::{ + ThreadPool, Yield, current_thread_has_pending_tasks, current_thread_index, yield_local, + yield_now, +}; /// Returns the maximum number of threads that Rayon supports in a single thread-pool. /// @@ -282,7 +277,7 @@ where } /// Initializes the global thread pool. This initialization is - /// **optional**. If you do not call this function, the thread pool + /// **optional**. If you do not call this function, the thread pool /// will be automatically initialized with the default /// configuration. Calling `build_global` is not recommended, except /// in two scenarios: @@ -290,7 +285,7 @@ where /// - You wish to change the default configuration. /// - You are running a benchmark, in which case initializing may /// yield slightly more consistent results, since the worker threads - /// will already be ready to go even in the first iteration. But + /// will already be ready to go even in the first iteration. But /// this cost is minimal. /// /// Initialization of the global thread pool happens exactly @@ -490,26 +485,16 @@ impl ThreadPoolBuilder { if self.num_threads > 0 { self.num_threads } else { - let default = || { - thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - }; - - match env::var("RAYON_NUM_THREADS") - .ok() - .and_then(|s| usize::from_str(&s).ok()) - { + let default = || thread::available_parallelism().map(|n| n.get()).unwrap_or(1); + + match env::var("RAYON_NUM_THREADS").ok().and_then(|s| usize::from_str(&s).ok()) { Some(x @ 1..) => return x, Some(0) => return default(), _ => {} } // Support for deprecated `RAYON_RS_NUM_CPUS`. - match env::var("RAYON_RS_NUM_CPUS") - .ok() - .and_then(|s| usize::from_str(&s).ok()) - { + match env::var("RAYON_RS_NUM_CPUS").ok().and_then(|s| usize::from_str(&s).ok()) { Some(x @ 1..) => x, _ => default(), } @@ -723,9 +708,7 @@ impl ThreadPoolBuilder { impl Configuration { /// Creates and return a valid rayon thread pool configuration, but does not initialize it. pub fn new() -> Configuration { - Configuration { - builder: ThreadPoolBuilder::new(), - } + Configuration { builder: ThreadPoolBuilder::new() } } /// Deprecated in favor of `ThreadPoolBuilder::build`. @@ -905,10 +888,7 @@ pub struct FnContext { impl FnContext { #[inline] fn new(migrated: bool) -> Self { - FnContext { - migrated, - _marker: PhantomData, - } + FnContext { migrated, _marker: PhantomData } } } diff --git a/compiler/rustc_thread_pool/src/private.rs b/compiler/rustc_thread_pool/src/private.rs index c85e77b9cbb73..5d4f4a8c2caf5 100644 --- a/compiler/rustc_thread_pool/src/private.rs +++ b/compiler/rustc_thread_pool/src/private.rs @@ -1,5 +1,5 @@ //! The public parts of this private module are used to create traits -//! that cannot be implemented outside of our own crate. This way we +//! that cannot be implemented outside of our own crate. This way we //! can feel free to extend those traits without worrying about it //! being a breaking change for other implementations. diff --git a/compiler/rustc_thread_pool/src/registry.rs b/compiler/rustc_thread_pool/src/registry.rs index 781b6827b829e..2848556aab6a1 100644 --- a/compiler/rustc_thread_pool/src/registry.rs +++ b/compiler/rustc_thread_pool/src/registry.rs @@ -1,23 +1,20 @@ +use std::cell::Cell; +use std::collections::hash_map::DefaultHasher; +use std::hash::Hasher; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, Once}; +use std::{fmt, io, mem, ptr, thread}; + +use crossbeam_deque::{Injector, Steal, Stealer, Worker}; + use crate::job::{JobFifo, JobRef, StackJob}; use crate::latch::{AsCoreLatch, CoreLatch, Latch, LatchRef, LockLatch, OnceLatch, SpinLatch}; use crate::sleep::Sleep; use crate::tlv::Tlv; -use crate::unwind; use crate::{ AcquireThreadHandler, DeadlockHandler, ErrorKind, ExitHandler, PanicHandler, - ReleaseThreadHandler, StartHandler, ThreadPoolBuildError, ThreadPoolBuilder, Yield, + ReleaseThreadHandler, StartHandler, ThreadPoolBuildError, ThreadPoolBuilder, Yield, unwind, }; -use crossbeam_deque::{Injector, Steal, Stealer, Worker}; -use std::cell::Cell; -use std::collections::hash_map::DefaultHasher; -use std::fmt; -use std::hash::Hasher; -use std::io; -use std::mem; -use std::ptr; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex, Once}; -use std::thread; /// Thread builder used for customization via /// [`ThreadPoolBuilder::spawn_handler`](struct.ThreadPoolBuilder.html#method.spawn_handler). @@ -193,9 +190,7 @@ fn set_global_registry(registry: F) -> Result<&'static Arc, ThreadP where F: FnOnce() -> Result, ThreadPoolBuildError>, { - let mut result = Err(ThreadPoolBuildError::new( - ErrorKind::GlobalPoolAlreadyInitialized, - )); + let mut result = Err(ThreadPoolBuildError::new(ErrorKind::GlobalPoolAlreadyInitialized)); THE_REGISTRY_SET.call_once(|| { result = registry().map(|registry: Arc| { @@ -222,25 +217,23 @@ fn default_global_registry() -> Result, ThreadPoolBuildError> { // is stubbed out, and we won't have to change anything if they do add real threading. let unsupported = matches!(&result, Err(e) if e.is_unsupported()); if unsupported && WorkerThread::current().is_null() { - let builder = ThreadPoolBuilder::new() - .num_threads(1) - .spawn_handler(|thread| { - // Rather than starting a new thread, we're just taking over the current thread - // *without* running the main loop, so we can still return from here. - // The WorkerThread is leaked, but we never shutdown the global pool anyway. - let worker_thread = Box::leak(Box::new(WorkerThread::from(thread))); - let registry = &*worker_thread.registry; - let index = worker_thread.index; - - unsafe { - WorkerThread::set_current(worker_thread); - - // let registry know we are ready to do work - Latch::set(®istry.thread_infos[index].primed); - } + let builder = ThreadPoolBuilder::new().num_threads(1).spawn_handler(|thread| { + // Rather than starting a new thread, we're just taking over the current thread + // *without* running the main loop, so we can still return from here. + // The WorkerThread is leaked, but we never shutdown the global pool anyway. + let worker_thread = Box::leak(Box::new(WorkerThread::from(thread))); + let registry = &*worker_thread.registry; + let index = worker_thread.index; + + unsafe { + WorkerThread::set_current(worker_thread); + + // let registry know we are ready to do work + Latch::set(®istry.thread_infos[index].primed); + } - Ok(()) - }); + Ok(()) + }); let fallback_result = Registry::new(builder); if fallback_result.is_ok() { @@ -273,11 +266,7 @@ impl Registry { let (workers, stealers): (Vec<_>, Vec<_>) = (0..n_threads) .map(|_| { - let worker = if breadth_first { - Worker::new_fifo() - } else { - Worker::new_lifo() - }; + let worker = if breadth_first { Worker::new_fifo() } else { Worker::new_lifo() }; let stealer = worker.stealer(); (worker, stealer) @@ -341,7 +330,7 @@ impl Registry { } } - /// Returns the number of threads in the current registry. This + /// Returns the number of threads in the current registry. This /// is better than `Registry::current().num_threads()` because it /// avoids incrementing the `Arc`. pub(super) fn current_num_threads() -> usize { @@ -359,11 +348,7 @@ impl Registry { pub(super) fn current_thread(&self) -> Option<&WorkerThread> { unsafe { let worker = WorkerThread::current().as_ref()?; - if worker.registry().id() == self.id() { - Some(worker) - } else { - None - } + if worker.registry().id() == self.id() { Some(worker) } else { None } } } @@ -371,9 +356,7 @@ impl Registry { pub(super) fn id(&self) -> RegistryId { // We can rely on `self` not to change since we only ever create // registries that are boxed up in an `Arc` (see `new()` above). - RegistryId { - addr: self as *const Self as usize, - } + RegistryId { addr: self as *const Self as usize } } pub(super) fn num_threads(&self) -> usize { @@ -391,7 +374,7 @@ impl Registry { } } - /// Waits for the worker threads to get up and running. This is + /// Waits for the worker threads to get up and running. This is /// meant to be used for benchmarking purposes, primarily, so that /// you can get more consistent numbers by having everything /// "ready to go". @@ -512,7 +495,7 @@ impl Registry { /// If already in a worker-thread of this registry, just execute `op`. /// Otherwise, inject `op` in this thread-pool. Either way, block until `op` /// completes and return its return value. If `op` panics, that panic will - /// be propagated as well. The second argument indicates `true` if injection + /// be propagated as well. The second argument indicates `true` if injection /// was performed, `false` if executed directly. pub(super) fn in_worker(&self, op: OP) -> R where @@ -844,9 +827,7 @@ impl WorkerThread { // The job might have injected local work, so go back to the outer loop. continue 'outer; } else { - self.registry - .sleep - .no_work_found(&mut idle_state, latch, &self) + self.registry.sleep.no_work_found(&mut idle_state, latch, &self) } } @@ -880,9 +861,7 @@ impl WorkerThread { // deques, and finally to injected jobs from the // outside. The idea is to finish what we started before // we take on something new. - self.take_local_job() - .or_else(|| self.steal()) - .or_else(|| self.registry.pop_injected_job()) + self.take_local_job().or_else(|| self.steal()).or_else(|| self.registry.pop_injected_job()) } pub(super) fn yield_now(&self) -> Yield { @@ -984,10 +963,10 @@ unsafe fn main_loop(thread: ThreadBuilder) { registry.release_thread(); } -/// If already in a worker-thread, just execute `op`. Otherwise, +/// If already in a worker-thread, just execute `op`. Otherwise, /// execute `op` in the default thread-pool. Either way, block until /// `op` completes and return its return value. If `op` panics, that -/// panic will be propagated as well. The second argument indicates +/// panic will be propagated as well. The second argument indicates /// `true` if injection was performed, `false` if executed directly. pub(super) fn in_worker(op: OP) -> R where @@ -1026,9 +1005,7 @@ impl XorShift64Star { seed = hasher.finish(); } - XorShift64Star { - state: Cell::new(seed), - } + XorShift64Star { state: Cell::new(seed) } } fn next(&self) -> u64 { diff --git a/compiler/rustc_thread_pool/src/scope/mod.rs b/compiler/rustc_thread_pool/src/scope/mod.rs index 364b322baadf4..95a4e0b7a18fc 100644 --- a/compiler/rustc_thread_pool/src/scope/mod.rs +++ b/compiler/rustc_thread_pool/src/scope/mod.rs @@ -5,19 +5,19 @@ //! [`in_place_scope()`]: fn.in_place_scope.html //! [`join()`]: ../join/join.fn.html +use std::any::Any; +use std::marker::PhantomData; +use std::mem::ManuallyDrop; +use std::sync::Arc; +use std::sync::atomic::{AtomicPtr, Ordering}; +use std::{fmt, ptr}; + use crate::broadcast::BroadcastContext; use crate::job::{ArcJob, HeapJob, JobFifo, JobRef}; use crate::latch::{CountLatch, Latch}; -use crate::registry::{global_registry, in_worker, Registry, WorkerThread}; +use crate::registry::{Registry, WorkerThread, global_registry, in_worker}; use crate::tlv::{self, Tlv}; use crate::unwind; -use std::any::Any; -use std::fmt; -use std::marker::PhantomData; -use std::mem::ManuallyDrop; -use std::ptr; -use std::sync::atomic::{AtomicPtr, Ordering}; -use std::sync::Arc; #[cfg(test)] mod test; @@ -53,7 +53,7 @@ struct ScopeBase<'scope> { job_completed_latch: CountLatch, /// You can think of a scope as containing a list of closures to execute, - /// all of which outlive `'scope`. They're not actually required to be + /// all of which outlive `'scope`. They're not actually required to be /// `Sync`, but it's still safe to let the `Scope` implement `Sync` because /// the closures are only *moved* across threads to be executed. #[allow(clippy::type_complexity)] @@ -179,9 +179,9 @@ struct ScopeBase<'scope> { /// they were spawned. So in this example, absent any stealing, we can /// expect `s.2` to execute before `s.1`, and `t.2` before `t.1`. Other /// threads always steal from the other end of the deque, like FIFO -/// order. The idea is that "recent" tasks are most likely to be fresh +/// order. The idea is that "recent" tasks are most likely to be fresh /// in the local CPU's cache, while other threads can steal older -/// "stale" tasks. For an alternate approach, consider +/// "stale" tasks. For an alternate approach, consider /// [`scope_fifo()`] instead. /// /// [`scope_fifo()`]: fn.scope_fifo.html @@ -353,7 +353,7 @@ where /// /// Under `scope_fifo()`, the spawns are prioritized in a FIFO order on /// the thread from which they were spawned, as opposed to `scope()`'s -/// LIFO. So in this example, we can expect `s.1` to execute before +/// LIFO. So in this example, we can expect `s.1` to execute before /// `s.2`, and `t.1` before `t.2`. Other threads also steal tasks in /// FIFO order, as usual. Overall, this has roughly the same order as /// the now-deprecated [`breadth_first`] option, except the effect is @@ -469,7 +469,7 @@ impl<'scope> Scope<'scope> { } /// Spawns a job into the fork-join scope `self`. This job will - /// execute sometime before the fork-join scope completes. The + /// execute sometime before the fork-join scope completes. The /// job is specified as a closure, and this closure receives its /// own reference to the scope `self` as argument. This can be /// used to inject new jobs into `self`. @@ -539,7 +539,7 @@ impl<'scope> Scope<'scope> { } /// Spawns a job into every thread of the fork-join scope `self`. This job will - /// execute on each thread sometime before the fork-join scope completes. The + /// execute on each thread sometime before the fork-join scope completes. The /// job is specified as a closure, and this closure receives its own reference /// to the scope `self` as argument, as well as a `BroadcastContext`. pub fn spawn_broadcast(&self, body: BODY) @@ -567,7 +567,7 @@ impl<'scope> ScopeFifo<'scope> { } /// Spawns a job into the fork-join scope `self`. This job will - /// execute sometime before the fork-join scope completes. The + /// execute sometime before the fork-join scope completes. The /// job is specified as a closure, and this closure receives its /// own reference to the scope `self` as argument. This can be /// used to inject new jobs into `self`. @@ -575,7 +575,7 @@ impl<'scope> ScopeFifo<'scope> { /// # See also /// /// This method is akin to [`Scope::spawn()`], but with a FIFO - /// priority. The [`scope_fifo` function] has more details about + /// priority. The [`scope_fifo` function] has more details about /// this distinction. /// /// [`Scope::spawn()`]: struct.Scope.html#method.spawn @@ -605,7 +605,7 @@ impl<'scope> ScopeFifo<'scope> { } /// Spawns a job into every thread of the fork-join scope `self`. This job will - /// execute on each thread sometime before the fork-join scope completes. The + /// execute on each thread sometime before the fork-join scope completes. The /// job is specified as a closure, and this closure receives its own reference /// to the scope `self` as argument, as well as a `BroadcastContext`. pub fn spawn_broadcast(&self, body: BODY) diff --git a/compiler/rustc_thread_pool/src/scope/test.rs b/compiler/rustc_thread_pool/src/scope/tests.rs similarity index 96% rename from compiler/rustc_thread_pool/src/scope/test.rs rename to compiler/rustc_thread_pool/src/scope/tests.rs index 4505ba7c4fba5..2df3bc67e298b 100644 --- a/compiler/rustc_thread_pool/src/scope/test.rs +++ b/compiler/rustc_thread_pool/src/scope/tests.rs @@ -1,13 +1,13 @@ -use crate::unwind; -use crate::ThreadPoolBuilder; -use crate::{scope, scope_fifo, Scope, ScopeFifo}; -use rand::{Rng, SeedableRng}; -use rand_xorshift::XorShiftRng; use std::iter::once; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Barrier, Mutex}; use std::vec; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; + +use crate::{Scope, ScopeFifo, ThreadPoolBuilder, scope, scope_fifo, unwind}; + #[test] fn scope_empty() { scope(|_| {}); @@ -93,10 +93,7 @@ impl Tree { where OP: Fn(&mut T) + Sync, { - let Tree { - ref mut value, - ref mut children, - } = *self; + let Tree { ref mut value, ref mut children } = *self; scope.spawn(move |scope| { for child in children { scope.spawn(move |scope| child.update_in_scope(op, scope)); @@ -124,10 +121,7 @@ fn random_tree1(depth: usize, rng: &mut XorShiftRng) -> Tree { .collect() }; - Tree { - value: rng.random_range(0..1_000_000), - children, - } + Tree { value: rng.random_range(0..1_000_000), children } } #[test] @@ -161,11 +155,7 @@ fn linear_stack_growth() { let diff_when_500 = *max_diff.get_mut().unwrap() as f64; let ratio = diff_when_5 / diff_when_500; - assert!( - ratio > 0.9 && ratio < 1.1, - "stack usage ratio out of bounds: {}", - ratio - ); + assert!(ratio > 0.9 && ratio < 1.1, "stack usage ratio out of bounds: {}", ratio); }); } @@ -366,10 +356,7 @@ fn nested_fifo_order() { fn nested_lifo_fifo_order() { // LIFO on the outside, FIFO on the inside let vec = test_nested_order!(scope => spawn, scope_fifo => spawn_fifo); - let expected: Vec = (0..10) - .rev() - .flat_map(|i| (0..10).map(move |j| i * 10 + j)) - .collect(); + let expected: Vec = (0..10).rev().flat_map(|i| (0..10).map(move |j| i * 10 + j)).collect(); assert_eq!(vec, expected); } @@ -378,9 +365,7 @@ fn nested_lifo_fifo_order() { fn nested_fifo_lifo_order() { // FIFO on the outside, LIFO on the inside let vec = test_nested_order!(scope_fifo => spawn_fifo, scope => spawn); - let expected: Vec = (0..10) - .flat_map(|i| (0..10).rev().map(move |j| i * 10 + j)) - .collect(); + let expected: Vec = (0..10).flat_map(|i| (0..10).rev().map(move |j| i * 10 + j)).collect(); assert_eq!(vec, expected); } diff --git a/compiler/rustc_thread_pool/src/sleep/README.md b/compiler/rustc_thread_pool/src/sleep/README.md index e79efd15ca9b0..1e11da55f4a4e 100644 --- a/compiler/rustc_thread_pool/src/sleep/README.md +++ b/compiler/rustc_thread_pool/src/sleep/README.md @@ -182,7 +182,7 @@ This is possible because the C++ memory model typically offers guarantees of the form "if you see the access A, then you must see those other accesses" -- but it doesn't guarantee that you will see the access A (i.e., if you think of processors with independent caches, you may be operating on very out of date -cache state). +cache state). ## Using seq-cst fences to prevent deadlock diff --git a/compiler/rustc_thread_pool/src/sleep/counters.rs b/compiler/rustc_thread_pool/src/sleep/counters.rs index 05941becd1c62..f2682028b96a6 100644 --- a/compiler/rustc_thread_pool/src/sleep/counters.rs +++ b/compiler/rustc_thread_pool/src/sleep/counters.rs @@ -89,9 +89,7 @@ const ONE_JEC: usize = 1 << JEC_SHIFT; impl AtomicCounters { #[inline] pub(super) fn new() -> AtomicCounters { - AtomicCounters { - value: AtomicUsize::new(0), - } + AtomicCounters { value: AtomicUsize::new(0) } } /// Load and return the current value of the various counters. @@ -230,9 +228,7 @@ impl Counters { fn increment_jobs_counter(self) -> Counters { // We can freely add to JEC because it occupies the most significant bits. // Thus it doesn't overflow into the other counters, just wraps itself. - Counters { - word: self.word.wrapping_add(ONE_JEC), - } + Counters { word: self.word.wrapping_add(ONE_JEC) } } #[inline] diff --git a/compiler/rustc_thread_pool/src/sleep/mod.rs b/compiler/rustc_thread_pool/src/sleep/mod.rs index 7d88ece210769..bee7c82c450c0 100644 --- a/compiler/rustc_thread_pool/src/sleep/mod.rs +++ b/compiler/rustc_thread_pool/src/sleep/mod.rs @@ -1,14 +1,16 @@ //! Code that decides when workers should go to sleep. See README.md //! for an overview. -use crate::latch::CoreLatch; -use crate::registry::WorkerThread; -use crate::DeadlockHandler; -use crossbeam_utils::CachePadded; use std::sync::atomic::Ordering; use std::sync::{Condvar, Mutex}; use std::thread; +use crossbeam_utils::CachePadded; + +use crate::DeadlockHandler; +use crate::latch::CoreLatch; +use crate::registry::WorkerThread; + mod counters; pub(crate) use self::counters::THREADS_MAX; use self::counters::{AtomicCounters, JobsEventCounter}; @@ -125,11 +127,7 @@ impl Sleep { pub(super) fn start_looking(&self, worker_index: usize) -> IdleState { self.counters.add_inactive_thread(); - IdleState { - worker_index, - rounds: 0, - jobs_counter: JobsEventCounter::DUMMY, - } + IdleState { worker_index, rounds: 0, jobs_counter: JobsEventCounter::DUMMY } } #[inline] @@ -165,9 +163,7 @@ impl Sleep { #[cold] fn announce_sleepy(&self) -> JobsEventCounter { - self.counters - .increment_jobs_event_counter_if(JobsEventCounter::is_active) - .jobs_counter() + self.counters.increment_jobs_event_counter_if(JobsEventCounter::is_active).jobs_counter() } #[cold] @@ -258,7 +254,7 @@ impl Sleep { } /// Notify the given thread that it should wake up (if it is - /// sleeping). When this method is invoked, we typically know the + /// sleeping). When this method is invoked, we typically know the /// thread is asleep, though in rare cases it could have been /// awoken by (e.g.) new work having been posted. pub(super) fn notify_worker_latch_is_set(&self, target_worker_index: usize) { @@ -307,9 +303,7 @@ impl Sleep { // Read the counters and -- if sleepy workers have announced themselves // -- announce that there is now work available. The final value of `counters` // with which we exit the loop thus corresponds to a state when - let counters = self - .counters - .increment_jobs_event_counter_if(JobsEventCounter::is_sleepy); + let counters = self.counters.increment_jobs_event_counter_if(JobsEventCounter::is_sleepy); let num_awake_but_idle = counters.awake_but_idle_threads(); let num_sleepers = counters.sleeping_threads(); diff --git a/compiler/rustc_thread_pool/src/spawn/mod.rs b/compiler/rustc_thread_pool/src/spawn/mod.rs index 034df30dcfb32..f1679a9823432 100644 --- a/compiler/rustc_thread_pool/src/spawn/mod.rs +++ b/compiler/rustc_thread_pool/src/spawn/mod.rs @@ -1,9 +1,10 @@ +use std::mem; +use std::sync::Arc; + use crate::job::*; use crate::registry::Registry; use crate::tlv::Tlv; use crate::unwind; -use std::mem; -use std::sync::Arc; /// Puts the task into the Rayon threadpool's job queue in the "static" /// or "global" scope. Just like a standard thread, this task is not @@ -28,9 +29,9 @@ use std::sync::Arc; /// other threads may steal tasks at any time. However, they are /// generally prioritized in a LIFO order on the thread from which /// they were spawned. Other threads always steal from the other end of -/// the deque, like FIFO order. The idea is that "recent" tasks are +/// the deque, like FIFO order. The idea is that "recent" tasks are /// most likely to be fresh in the local CPU's cache, while other -/// threads can steal older "stale" tasks. For an alternate approach, +/// threads can steal older "stale" tasks. For an alternate approach, /// consider [`spawn_fifo()`] instead. /// /// [`spawn_fifo()`]: fn.spawn_fifo.html @@ -39,7 +40,7 @@ use std::sync::Arc; /// /// If this closure should panic, the resulting panic will be /// propagated to the panic handler registered in the `ThreadPoolBuilder`, -/// if any. See [`ThreadPoolBuilder::panic_handler()`][ph] for more +/// if any. See [`ThreadPoolBuilder::panic_handler()`][ph] for more /// details. /// /// [ph]: struct.ThreadPoolBuilder.html#method.panic_handler @@ -103,7 +104,7 @@ where } /// Fires off a task into the Rayon threadpool in the "static" or -/// "global" scope. Just like a standard thread, this task is not +/// "global" scope. Just like a standard thread, this task is not /// tied to the current stack frame, and hence it cannot hold any /// references other than those with `'static` lifetime. If you want /// to spawn a task that references stack data, use [the `scope_fifo()` @@ -124,7 +125,7 @@ where /// /// If this closure should panic, the resulting panic will be /// propagated to the panic handler registered in the `ThreadPoolBuilder`, -/// if any. See [`ThreadPoolBuilder::panic_handler()`][ph] for more +/// if any. See [`ThreadPoolBuilder::panic_handler()`][ph] for more /// details. /// /// [ph]: struct.ThreadPoolBuilder.html#method.panic_handler @@ -152,7 +153,7 @@ where let job_ref = spawn_job(func, registry); // If we're in the pool, use our thread's private fifo for this thread to execute - // in a locally-FIFO order. Otherwise, just use the pool's global injector. + // in a locally-FIFO order. Otherwise, just use the pool's global injector. match registry.current_thread() { Some(worker) => worker.push_fifo(job_ref), None => registry.inject(job_ref), diff --git a/compiler/rustc_thread_pool/src/spawn/test.rs b/compiler/rustc_thread_pool/src/spawn/tests.rs similarity index 95% rename from compiler/rustc_thread_pool/src/spawn/test.rs rename to compiler/rustc_thread_pool/src/spawn/tests.rs index b7a0535aabf6d..8a70d2faf9c6c 100644 --- a/compiler/rustc_thread_pool/src/spawn/test.rs +++ b/compiler/rustc_thread_pool/src/spawn/tests.rs @@ -1,10 +1,9 @@ -use crate::scope; use std::any::Any; -use std::sync::mpsc::channel; use std::sync::Mutex; +use std::sync::mpsc::channel; use super::{spawn, spawn_fifo}; -use crate::ThreadPoolBuilder; +use crate::{ThreadPoolBuilder, scope}; #[test] #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] @@ -45,10 +44,7 @@ fn panic_fwd() { let builder = ThreadPoolBuilder::new().panic_handler(panic_handler); - builder - .build() - .unwrap() - .spawn(move || panic!("Hello, world!")); + builder.build().unwrap().spawn(move || panic!("Hello, world!")); assert_eq!(1, rx.recv().unwrap()); } @@ -193,10 +189,7 @@ fn fifo_order() { fn lifo_fifo_order() { // LIFO on the outside, FIFO on the inside let vec = test_order!(spawn, spawn_fifo); - let expected: Vec = (0..10) - .rev() - .flat_map(|i| (0..10).map(move |j| i * 10 + j)) - .collect(); + let expected: Vec = (0..10).rev().flat_map(|i| (0..10).map(move |j| i * 10 + j)).collect(); assert_eq!(vec, expected); } @@ -205,9 +198,7 @@ fn lifo_fifo_order() { fn fifo_lifo_order() { // FIFO on the outside, LIFO on the inside let vec = test_order!(spawn_fifo, spawn); - let expected: Vec = (0..10) - .flat_map(|i| (0..10).rev().map(move |j| i * 10 + j)) - .collect(); + let expected: Vec = (0..10).flat_map(|i| (0..10).rev().map(move |j| i * 10 + j)).collect(); assert_eq!(vec, expected); } diff --git a/compiler/rustc_thread_pool/src/test.rs b/compiler/rustc_thread_pool/src/tests.rs similarity index 96% rename from compiler/rustc_thread_pool/src/test.rs rename to compiler/rustc_thread_pool/src/tests.rs index 25b8487f73bc9..3082f11a167ac 100644 --- a/compiler/rustc_thread_pool/src/test.rs +++ b/compiler/rustc_thread_pool/src/tests.rs @@ -1,9 +1,10 @@ #![cfg(test)] -use crate::{ThreadPoolBuildError, ThreadPoolBuilder}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Barrier}; +use crate::{ThreadPoolBuildError, ThreadPoolBuilder}; + #[test] #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] fn worker_thread_index() { @@ -29,9 +30,7 @@ fn start_callback_called() { b.wait(); }; - let conf = ThreadPoolBuilder::new() - .num_threads(n_threads) - .start_handler(start_handler); + let conf = ThreadPoolBuilder::new().num_threads(n_threads).start_handler(start_handler); let _ = conf.build().unwrap(); // Wait for all the threads to have been scheduled to run. @@ -56,9 +55,7 @@ fn exit_callback_called() { b.wait(); }; - let conf = ThreadPoolBuilder::new() - .num_threads(n_threads) - .exit_handler(exit_handler); + let conf = ThreadPoolBuilder::new().num_threads(n_threads).exit_handler(exit_handler); { let _ = conf.build().unwrap(); // Drop the pool so it stops the running threads. diff --git a/compiler/rustc_thread_pool/src/thread_pool/mod.rs b/compiler/rustc_thread_pool/src/thread_pool/mod.rs index 65af6d7106e64..ce8783cf0d69c 100644 --- a/compiler/rustc_thread_pool/src/thread_pool/mod.rs +++ b/compiler/rustc_thread_pool/src/thread_pool/mod.rs @@ -3,18 +3,17 @@ //! //! [`ThreadPool`]: struct.ThreadPool.html -use crate::broadcast::{self, BroadcastContext}; -use crate::join; -use crate::registry::{Registry, ThreadSpawn, WorkerThread}; -use crate::scope::{do_in_place_scope, do_in_place_scope_fifo}; -use crate::spawn; -use crate::{scope, Scope}; -use crate::{scope_fifo, ScopeFifo}; -use crate::{ThreadPoolBuildError, ThreadPoolBuilder}; use std::error::Error; use std::fmt; use std::sync::Arc; +use crate::broadcast::{self, BroadcastContext}; +use crate::registry::{Registry, ThreadSpawn, WorkerThread}; +use crate::scope::{do_in_place_scope, do_in_place_scope_fifo}; +use crate::{ + Scope, ScopeFifo, ThreadPoolBuildError, ThreadPoolBuilder, join, scope, scope_fifo, spawn, +}; + mod test; /// Represents a user created [thread-pool]. diff --git a/compiler/rustc_thread_pool/src/thread_pool/test.rs b/compiler/rustc_thread_pool/src/thread_pool/tests.rs similarity index 97% rename from compiler/rustc_thread_pool/src/thread_pool/test.rs rename to compiler/rustc_thread_pool/src/thread_pool/tests.rs index 88b36282d481d..42c99565088af 100644 --- a/compiler/rustc_thread_pool/src/thread_pool/test.rs +++ b/compiler/rustc_thread_pool/src/thread_pool/tests.rs @@ -4,7 +4,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::channel; use std::sync::{Arc, Mutex}; -use crate::{join, Scope, ScopeFifo, ThreadPool, ThreadPoolBuilder}; +use crate::{Scope, ScopeFifo, ThreadPool, ThreadPoolBuilder, join}; #[test] #[should_panic(expected = "Hello, world!")] @@ -296,9 +296,8 @@ fn nested_scopes() { } } - let pools: Vec<_> = (0..10) - .map(|_| ThreadPoolBuilder::new().num_threads(1).build().unwrap()) - .collect(); + let pools: Vec<_> = + (0..10).map(|_| ThreadPoolBuilder::new().num_threads(1).build().unwrap()).collect(); let counter = AtomicUsize::new(0); nest(&pools, vec![], |scopes| { @@ -333,9 +332,8 @@ fn nested_fifo_scopes() { } } - let pools: Vec<_> = (0..10) - .map(|_| ThreadPoolBuilder::new().num_threads(1).build().unwrap()) - .collect(); + let pools: Vec<_> = + (0..10).map(|_| ThreadPoolBuilder::new().num_threads(1).build().unwrap()).collect(); let counter = AtomicUsize::new(0); nest(&pools, vec![], |scopes| { diff --git a/compiler/rustc_thread_pool/src/tlv.rs b/compiler/rustc_thread_pool/src/tlv.rs index ce22f7aa0ce70..b5f63479e2fe9 100644 --- a/compiler/rustc_thread_pool/src/tlv.rs +++ b/compiler/rustc_thread_pool/src/tlv.rs @@ -1,7 +1,8 @@ //! Allows access to the Rayon's thread local value //! which is preserved when moving jobs across threads -use std::{cell::Cell, ptr}; +use std::cell::Cell; +use std::ptr; thread_local!(pub static TLV: Cell<*const ()> = const { Cell::new(ptr::null()) }); diff --git a/compiler/rustc_thread_pool/src/worker_local.rs b/compiler/rustc_thread_pool/src/worker_local.rs index 85d51687c1902..d108c91f9ee53 100644 --- a/compiler/rustc_thread_pool/src/worker_local.rs +++ b/compiler/rustc_thread_pool/src/worker_local.rs @@ -1,8 +1,9 @@ -use crate::registry::{Registry, WorkerThread}; use std::fmt; use std::ops::Deref; use std::sync::Arc; +use crate::registry::{Registry, WorkerThread}; + #[repr(align(64))] #[derive(Debug)] struct CacheAligned(T); @@ -27,9 +28,7 @@ impl WorkerLocal { pub fn new T>(mut initial: F) -> WorkerLocal { let registry = Registry::current(); WorkerLocal { - locals: (0..registry.num_threads()) - .map(|i| CacheAligned(initial(i))) - .collect(), + locals: (0..registry.num_threads()).map(|i| CacheAligned(initial(i))).collect(), registry, } } @@ -62,9 +61,7 @@ impl WorkerLocal> { impl fmt::Debug for WorkerLocal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("WorkerLocal") - .field("registry", &self.registry.id()) - .finish() + f.debug_struct("WorkerLocal").field("registry", &self.registry.id()).finish() } } diff --git a/compiler/rustc_thread_pool/tests/double_init_fail.rs b/compiler/rustc_thread_pool/tests/double_init_fail.rs index 15915304ddccc..71ed425bb32c7 100644 --- a/compiler/rustc_thread_pool/tests/double_init_fail.rs +++ b/compiler/rustc_thread_pool/tests/double_init_fail.rs @@ -1,6 +1,7 @@ -use rayon_core::ThreadPoolBuilder; use std::error::Error; +use rayon_core::ThreadPoolBuilder; + #[test] #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] fn double_init_fail() { @@ -8,8 +9,5 @@ fn double_init_fail() { assert!(result1.is_ok()); let err = ThreadPoolBuilder::new().build_global().unwrap_err(); assert!(err.source().is_none()); - assert_eq!( - err.to_string(), - "The global thread pool has already been initialized.", - ); + assert_eq!(err.to_string(), "The global thread pool has already been initialized.",); } diff --git a/compiler/rustc_thread_pool/tests/init_zero_threads.rs b/compiler/rustc_thread_pool/tests/init_zero_threads.rs index 3c1ad251c7e02..c1770e57f3c89 100644 --- a/compiler/rustc_thread_pool/tests/init_zero_threads.rs +++ b/compiler/rustc_thread_pool/tests/init_zero_threads.rs @@ -3,8 +3,5 @@ use rayon_core::ThreadPoolBuilder; #[test] #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] fn init_zero_threads() { - ThreadPoolBuilder::new() - .num_threads(0) - .build_global() - .unwrap(); + ThreadPoolBuilder::new().num_threads(0).build_global().unwrap(); } diff --git a/compiler/rustc_thread_pool/tests/scoped_threadpool.rs b/compiler/rustc_thread_pool/tests/scoped_threadpool.rs index 932147179f5c5..8cc2c859c0c6a 100644 --- a/compiler/rustc_thread_pool/tests/scoped_threadpool.rs +++ b/compiler/rustc_thread_pool/tests/scoped_threadpool.rs @@ -10,9 +10,7 @@ scoped_tls::scoped_thread_local!(static LOCAL: Local); #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] fn missing_scoped_tls() { LOCAL.set(&Local(42), || { - let pool = ThreadPoolBuilder::new() - .build() - .expect("thread pool created"); + let pool = ThreadPoolBuilder::new().build().expect("thread pool created"); // `LOCAL` is not set in the pool. pool.install(|| { diff --git a/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs b/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs index a6494069212ca..c7a880de8bb41 100644 --- a/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs +++ b/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs @@ -1,10 +1,9 @@ -use rayon_core::ThreadPoolBuilder; - use std::env; -use std::process::{Command, ExitStatus, Stdio}; - #[cfg(target_os = "linux")] use std::os::unix::process::ExitStatusExt; +use std::process::{Command, ExitStatus, Stdio}; + +use rayon_core::ThreadPoolBuilder; fn force_stack_overflow(depth: u32) { let mut buffer = [0u8; 1024 * 1024]; @@ -18,13 +17,7 @@ fn force_stack_overflow(depth: u32) { #[cfg(unix)] fn disable_core() { unsafe { - libc::setrlimit( - libc::RLIMIT_CORE, - &libc::rlimit { - rlim_cur: 0, - rlim_max: 0, - }, - ); + libc::setrlimit(libc::RLIMIT_CORE, &libc::rlimit { rlim_cur: 0, rlim_max: 0 }); } } @@ -50,10 +43,7 @@ fn stack_overflow_crash() { #[cfg(any(unix, windows))] assert_eq!(status.code(), overflow_code()); #[cfg(target_os = "linux")] - assert!(matches!( - status.signal(), - Some(libc::SIGABRT | libc::SIGSEGV) - )); + assert!(matches!(status.signal(), Some(libc::SIGABRT | libc::SIGSEGV))); // Now run with a larger stack and verify correct operation. let status = run_ignored("run_with_large_stack"); @@ -86,10 +76,7 @@ fn run_with_large_stack() { } fn run_with_stack(stack_size_in_mb: usize) { - let pool = ThreadPoolBuilder::new() - .stack_size(stack_size_in_mb * 1024 * 1024) - .build() - .unwrap(); + let pool = ThreadPoolBuilder::new().stack_size(stack_size_in_mb * 1024 * 1024).build().unwrap(); pool.install(|| { #[cfg(unix)] disable_core(); From 4aa62ea9e9015621969a0f505abf7a6894e99e9e Mon Sep 17 00:00:00 2001 From: "Celina G. Val" Date: Wed, 11 Jun 2025 11:45:06 -0700 Subject: [PATCH 08/32] Use `rustc_thread_pool` instead of `rustc-rayon-core` --- Cargo.lock | 39 ++++++++++++------- compiler/rustc_data_structures/Cargo.toml | 2 +- compiler/rustc_data_structures/src/sync.rs | 2 +- .../src/sync/parallel.rs | 12 +++--- compiler/rustc_interface/Cargo.toml | 2 +- compiler/rustc_interface/src/util.rs | 8 ++-- compiler/rustc_middle/Cargo.toml | 2 +- compiler/rustc_middle/src/ty/context/tls.rs | 2 +- compiler/rustc_query_system/Cargo.toml | 2 +- compiler/rustc_query_system/src/query/job.rs | 10 ++--- compiler/rustc_thread_pool/Cargo.toml | 14 ++----- .../src/compile_fail/quicksort_race1.rs | 2 +- .../src/compile_fail/quicksort_race2.rs | 2 +- .../src/compile_fail/quicksort_race3.rs | 2 +- .../src/compile_fail/rc_return.rs | 4 +- .../src/compile_fail/rc_upvar.rs | 2 +- .../src/compile_fail/scope_join_bad.rs | 4 +- compiler/rustc_thread_pool/src/join/mod.rs | 2 +- compiler/rustc_thread_pool/src/lib.rs | 12 +++--- compiler/rustc_thread_pool/src/scope/mod.rs | 16 ++++---- compiler/rustc_thread_pool/src/spawn/mod.rs | 2 +- .../rustc_thread_pool/src/thread_pool/mod.rs | 10 ++--- .../tests/double_init_fail.rs | 2 +- .../tests/init_zero_threads.rs | 2 +- .../rustc_thread_pool/tests/scope_join.rs | 2 +- .../tests/scoped_threadpool.rs | 2 +- .../rustc_thread_pool/tests/simple_panic.rs | 2 +- .../tests/stack_overflow_crash.rs | 2 +- src/tools/tidy/src/deps.rs | 2 +- 29 files changed, 86 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93abab8469a7b..ba3a10bad95e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2977,6 +2977,15 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.3", +] + [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -3176,16 +3185,6 @@ dependencies = [ "tikv-jemalloc-sys", ] -[[package]] -name = "rustc-rayon-core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f42932dcd3bcbe484b38a3ccf79b7906fac41c02d408b5b1bac26da3416efdb" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "rustc-semver" version = "1.1.0" @@ -3554,7 +3553,6 @@ dependencies = [ "parking_lot", "portable-atomic", "rustc-hash 2.1.1", - "rustc-rayon-core", "rustc-stable-hash", "rustc_arena", "rustc_graphviz", @@ -3562,6 +3560,7 @@ dependencies = [ "rustc_index", "rustc_macros", "rustc_serialize", + "rustc_thread_pool", "smallvec", "stacker", "tempfile", @@ -3908,7 +3907,6 @@ dependencies = [ name = "rustc_interface" version = "0.0.0" dependencies = [ - "rustc-rayon-core", "rustc_abi", "rustc_ast", "rustc_ast_lowering", @@ -3947,6 +3945,7 @@ dependencies = [ "rustc_span", "rustc_symbol_mangling", "rustc_target", + "rustc_thread_pool", "rustc_trait_selection", "rustc_traits", "rustc_ty_utils", @@ -4074,7 +4073,6 @@ dependencies = [ "either", "gsgdt", "polonius-engine", - "rustc-rayon-core", "rustc_abi", "rustc_apfloat", "rustc_arena", @@ -4098,6 +4096,7 @@ dependencies = [ "rustc_session", "rustc_span", "rustc_target", + "rustc_thread_pool", "rustc_type_ir", "smallvec", "thin-vec", @@ -4344,7 +4343,6 @@ version = "0.0.0" dependencies = [ "hashbrown", "parking_lot", - "rustc-rayon-core", "rustc_abi", "rustc_ast", "rustc_attr_data_structures", @@ -4359,6 +4357,7 @@ dependencies = [ "rustc_serialize", "rustc_session", "rustc_span", + "rustc_thread_pool", "smallvec", "tracing", ] @@ -4520,6 +4519,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "rustc_thread_pool" +version = "0.0.0" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", + "libc", + "rand 0.9.1", + "rand_xorshift", + "scoped-tls", +] + [[package]] name = "rustc_tools_util" version = "0.4.2" diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index f6a0201161851..17204883fb033 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -14,7 +14,6 @@ indexmap = "2.4.0" jobserver_crate = { version = "0.1.28", package = "jobserver" } measureme = "12.0.1" rustc-hash = "2.0.0" -rustc-rayon-core = { version = "0.5.0" } rustc-stable-hash = { version = "0.1.0", features = ["nightly"] } rustc_arena = { path = "../rustc_arena" } rustc_graphviz = { path = "../rustc_graphviz" } @@ -22,6 +21,7 @@ rustc_hashes = { path = "../rustc_hashes" } rustc_index = { path = "../rustc_index", package = "rustc_index" } rustc_macros = { path = "../rustc_macros" } rustc_serialize = { path = "../rustc_serialize" } +rustc_thread_pool = { path = "../rustc_thread_pool" } smallvec = { version = "1.8.1", features = ["const_generics", "union", "may_dangle"] } stacker = "0.1.17" tempfile = "3.2" diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index b28c333d860c6..68527dde1f3c8 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -23,7 +23,7 @@ //! | `RwLock` | `RefCell` | `parking_lot::RwLock` | //! | `MTLock` [^1] | `T` | `Lock` | //! | | | | -//! | `ParallelIterator` | `Iterator` | `rayon::iter::ParallelIterator` | +//! | `ParallelIterator` | `Iterator` | `rustc_thread_pool::iter::ParallelIterator` | //! //! [^1]: `MTLock` is similar to `Lock`, but the serial version avoids the cost //! of a `RefCell`. This is appropriate when interior mutability is not diff --git a/compiler/rustc_data_structures/src/sync/parallel.rs b/compiler/rustc_data_structures/src/sync/parallel.rs index ab65c7f3a6b5f..ff4b60a1031b9 100644 --- a/compiler/rustc_data_structures/src/sync/parallel.rs +++ b/compiler/rustc_data_structures/src/sync/parallel.rs @@ -96,7 +96,7 @@ macro_rules! parallel { pub fn spawn(func: impl FnOnce() + DynSend + 'static) { if mode::is_dyn_thread_safe() { let func = FromDyn::from(func); - rayon_core::spawn(|| { + rustc_thread_pool::spawn(|| { (func.into_inner())(); }); } else { @@ -107,11 +107,11 @@ pub fn spawn(func: impl FnOnce() + DynSend + 'static) { // This function only works when `mode::is_dyn_thread_safe()`. pub fn scope<'scope, OP, R>(op: OP) -> R where - OP: FnOnce(&rayon_core::Scope<'scope>) -> R + DynSend, + OP: FnOnce(&rustc_thread_pool::Scope<'scope>) -> R + DynSend, R: DynSend, { let op = FromDyn::from(op); - rayon_core::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner() + rustc_thread_pool::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner() } #[inline] @@ -124,7 +124,7 @@ where let oper_a = FromDyn::from(oper_a); let oper_b = FromDyn::from(oper_b); let (a, b) = parallel_guard(|guard| { - rayon_core::join( + rustc_thread_pool::join( move || guard.run(move || FromDyn::from(oper_a.into_inner()())), move || guard.run(move || FromDyn::from(oper_b.into_inner()())), ) @@ -158,7 +158,7 @@ fn par_slice( let (left, right) = items.split_at_mut(items.len() / 2); let mut left = state.for_each.derive(left); let mut right = state.for_each.derive(right); - rayon_core::join(move || par_rec(*left, state), move || par_rec(*right, state)); + rustc_thread_pool::join(move || par_rec(*left, state), move || par_rec(*right, state)); } } @@ -241,7 +241,7 @@ pub fn par_map, R: DynSend, C: FromIterato pub fn broadcast(op: impl Fn(usize) -> R + DynSync) -> Vec { if mode::is_dyn_thread_safe() { let op = FromDyn::from(op); - let results = rayon_core::broadcast(|context| op.derive(op(context.index()))); + let results = rustc_thread_pool::broadcast(|context| op.derive(op(context.index()))); results.into_iter().map(|r| r.into_inner()).collect() } else { vec![op(0)] diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index ff28dbeaee698..a72a795878786 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -5,7 +5,6 @@ edition = "2024" [dependencies] # tidy-alphabetical-start -rustc-rayon-core = { version = "0.5.0" } rustc_ast = { path = "../rustc_ast" } rustc_ast_lowering = { path = "../rustc_ast_lowering" } rustc_ast_passes = { path = "../rustc_ast_passes" } @@ -43,6 +42,7 @@ rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } rustc_target = { path = "../rustc_target" } +rustc_thread_pool = { path = "../rustc_thread_pool" } rustc_trait_selection = { path = "../rustc_trait_selection" } rustc_traits = { path = "../rustc_traits" } rustc_ty_utils = { path = "../rustc_ty_utils" } diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 8bdc24d47d98a..8a7d61172655f 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -208,7 +208,7 @@ pub(crate) fn run_in_thread_pool_with_globals< let proxy_ = Arc::clone(&proxy); let proxy__ = Arc::clone(&proxy); - let builder = rayon_core::ThreadPoolBuilder::new() + let builder = rustc_thread_pool::ThreadPoolBuilder::new() .thread_name(|_| "rustc".to_string()) .acquire_thread_handler(move || proxy_.acquire_thread()) .release_thread_handler(move || proxy__.release_thread()) @@ -218,7 +218,7 @@ pub(crate) fn run_in_thread_pool_with_globals< // locals to it. The new thread runs the deadlock handler. let current_gcx2 = current_gcx2.clone(); - let registry = rayon_core::Registry::current(); + let registry = rustc_thread_pool::Registry::current(); let session_globals = rustc_span::with_session_globals(|session_globals| { session_globals as *const SessionGlobals as usize }); @@ -265,7 +265,7 @@ pub(crate) fn run_in_thread_pool_with_globals< builder .build_scoped( // Initialize each new worker thread when created. - move |thread: rayon_core::ThreadBuilder| { + move |thread: rustc_thread_pool::ThreadBuilder| { // Register the thread for use with the `WorkerLocal` type. registry.register(); @@ -274,7 +274,7 @@ pub(crate) fn run_in_thread_pool_with_globals< }) }, // Run `f` on the first thread in the thread pool. - move |pool: &rayon_core::ThreadPool| { + move |pool: &rustc_thread_pool::ThreadPool| { pool.install(|| f(current_gcx.into_inner(), proxy)) }, ) diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml index 43c1af642dd56..edd0af6e4f535 100644 --- a/compiler/rustc_middle/Cargo.toml +++ b/compiler/rustc_middle/Cargo.toml @@ -9,7 +9,6 @@ bitflags = "2.4.1" either = "1.5.0" gsgdt = "0.1.2" polonius-engine = "0.13.0" -rustc-rayon-core = { version = "0.5.0" } rustc_abi = { path = "../rustc_abi" } rustc_apfloat = "0.2.0" rustc_arena = { path = "../rustc_arena" } @@ -33,6 +32,7 @@ rustc_serialize = { path = "../rustc_serialize" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } rustc_target = { path = "../rustc_target" } +rustc_thread_pool = { path = "../rustc_thread_pool" } rustc_type_ir = { path = "../rustc_type_ir" } smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } thin-vec = "0.2.12" diff --git a/compiler/rustc_middle/src/ty/context/tls.rs b/compiler/rustc_middle/src/ty/context/tls.rs index 5fc80bc793673..fa9995898ac20 100644 --- a/compiler/rustc_middle/src/ty/context/tls.rs +++ b/compiler/rustc_middle/src/ty/context/tls.rs @@ -36,7 +36,7 @@ impl<'a, 'tcx> ImplicitCtxt<'a, 'tcx> { } // Import the thread-local variable from Rayon, which is preserved for Rayon jobs. -use rayon_core::tlv::TLV; +use rustc_thread_pool::tlv::TLV; #[inline] fn erase(context: &ImplicitCtxt<'_, '_>) -> *const () { diff --git a/compiler/rustc_query_system/Cargo.toml b/compiler/rustc_query_system/Cargo.toml index 7db06953aeb64..3d2d879a76445 100644 --- a/compiler/rustc_query_system/Cargo.toml +++ b/compiler/rustc_query_system/Cargo.toml @@ -6,7 +6,6 @@ edition = "2024" [dependencies] # tidy-alphabetical-start parking_lot = "0.12" -rustc-rayon-core = { version = "0.5.0" } rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } @@ -21,6 +20,7 @@ rustc_macros = { path = "../rustc_macros" } rustc_serialize = { path = "../rustc_serialize" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } +rustc_thread_pool = { path = "../rustc_thread_pool" } smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } tracing = "0.1" # tidy-alphabetical-end diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index 1e79bd461d2a4..7e61f5026da4a 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -236,7 +236,7 @@ impl QueryLatch { // If this detects a deadlock and the deadlock handler wants to resume this thread // we have to be in the `wait` call. This is ensured by the deadlock handler // getting the self.info lock. - rayon_core::mark_blocked(); + rustc_thread_pool::mark_blocked(); let proxy = qcx.jobserver_proxy(); proxy.release_thread(); waiter.condvar.wait(&mut info); @@ -251,9 +251,9 @@ impl QueryLatch { let mut info = self.info.lock(); debug_assert!(!info.complete); info.complete = true; - let registry = rayon_core::Registry::current(); + let registry = rustc_thread_pool::Registry::current(); for waiter in info.waiters.drain(..) { - rayon_core::mark_unblocked(®istry); + rustc_thread_pool::mark_unblocked(®istry); waiter.condvar.notify_one(); } } @@ -507,7 +507,7 @@ fn remove_cycle( /// all active queries for cycles before finally resuming all the waiters at once. pub fn break_query_cycles( query_map: QueryMap, - registry: &rayon_core::Registry, + registry: &rustc_thread_pool::Registry, ) { let mut wakelist = Vec::new(); // It is OK per the comments: @@ -543,7 +543,7 @@ pub fn break_query_cycles( // we wake the threads up as otherwise Rayon could detect a deadlock if a thread we // resumed fell asleep and this thread had yet to mark the remaining threads as unblocked. for _ in 0..wakelist.len() { - rayon_core::mark_unblocked(registry); + rustc_thread_pool::mark_unblocked(registry); } for waiter in wakelist.into_iter() { diff --git a/compiler/rustc_thread_pool/Cargo.toml b/compiler/rustc_thread_pool/Cargo.toml index f7a3d1306b43f..d0bd065c45787 100644 --- a/compiler/rustc_thread_pool/Cargo.toml +++ b/compiler/rustc_thread_pool/Cargo.toml @@ -1,25 +1,19 @@ [package] -name = "rustc-rayon-core" -version = "0.5.1" +name = "rustc_thread_pool" +version = "0.0.0" authors = ["Niko Matsakis ", "Josh Stone "] description = "Core APIs for Rayon - fork for rustc" license = "MIT OR Apache-2.0" -repository = "https://github.com/rust-lang/rustc-rayon" -documentation = "https://docs.rs/rustc-rayon-core/" rust-version = "1.63" edition = "2021" readme = "README.md" keywords = ["parallel", "thread", "concurrency", "join", "performance"] categories = ["concurrency"] -[lib] -name = "rayon_core" - -# Some dependencies may not be their latest version, in order to support older rustc. [dependencies] -crossbeam-deque = "0.8.1" -crossbeam-utils = "0.8.0" +crossbeam-deque = "0.8" +crossbeam-utils = "0.8" [dev-dependencies] rand = "0.9" diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs index 5615033895a00..1f7a7b0b429b4 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs @@ -7,7 +7,7 @@ fn quick_sort(v: &mut [T]) { let mid = partition(v); let (lo, _hi) = v.split_at_mut(mid); - rayon_core::join(|| quick_sort(lo), || quick_sort(lo)); //~ ERROR + rustc_thred_pool::join(|| quick_sort(lo), || quick_sort(lo)); //~ ERROR } fn partition(v: &mut [T]) -> usize { diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs index 020589c29a849..71b695dd8551d 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs @@ -7,7 +7,7 @@ fn quick_sort(v: &mut [T]) { let mid = partition(v); let (lo, _hi) = v.split_at_mut(mid); - rayon_core::join(|| quick_sort(lo), || quick_sort(v)); //~ ERROR + rustc_thred_pool::join(|| quick_sort(lo), || quick_sort(v)); //~ ERROR } fn partition(v: &mut [T]) -> usize { diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs index 16fbf3b824d35..8484cebaae80d 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs @@ -7,7 +7,7 @@ fn quick_sort(v: &mut [T]) { let mid = partition(v); let (_lo, hi) = v.split_at_mut(mid); - rayon_core::join(|| quick_sort(hi), || quick_sort(hi)); //~ ERROR + rustc_thred_pool::join(|| quick_sort(hi), || quick_sort(hi)); //~ ERROR } fn partition(v: &mut [T]) -> usize { diff --git a/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs b/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs index 93e3a603849a1..509c8d62ad113 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs @@ -2,7 +2,7 @@ use std::rc::Rc; -rayon_core::join(|| Rc::new(22), || ()); //~ ERROR +rustc_thred_pool::join(|| Rc::new(22), || ()); //~ ERROR ``` */ mod left {} @@ -11,7 +11,7 @@ mod left {} use std::rc::Rc; -rayon_core::join(|| (), || Rc::new(23)); //~ ERROR +rustc_thred_pool::join(|| (), || Rc::new(23)); //~ ERROR ``` */ mod right {} diff --git a/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs b/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs index d8aebcfcbf24b..a27b3c8c39fce 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs @@ -3,7 +3,7 @@ use std::rc::Rc; let r = Rc::new(22); -rayon_core::join(|| r.clone(), || r.clone()); +rustc_thred_pool::join(|| r.clone(), || r.clone()); //~^ ERROR ``` */ diff --git a/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs b/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs index 75e4c5ca6c0d5..6e700a483b103 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs @@ -3,7 +3,7 @@ fn bad_scope(f: F) where F: FnOnce(&i32) + Send, { - rayon_core::scope(|s| { + rustc_thred_pool::scope(|s| { let x = 22; s.spawn(|_| f(&x)); //~ ERROR `x` does not live long enough }); @@ -13,7 +13,7 @@ fn good_scope(f: F) where F: FnOnce(&i32) + Send, { let x = 22; - rayon_core::scope(|s| { + rustc_thred_pool::scope(|s| { s.spawn(|_| f(&x)); }); } diff --git a/compiler/rustc_thread_pool/src/join/mod.rs b/compiler/rustc_thread_pool/src/join/mod.rs index 798a8347d798c..e48d17f16a3ac 100644 --- a/compiler/rustc_thread_pool/src/join/mod.rs +++ b/compiler/rustc_thread_pool/src/join/mod.rs @@ -41,7 +41,7 @@ mod test; /// [the `par_sort` method]: ../rayon/slice/trait.ParallelSliceMut.html#method.par_sort /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// let mut v = vec![5, 1, 8, 22, 0, 44]; /// quick_sort(&mut v); /// assert_eq!(v, vec![0, 1, 5, 8, 22, 44]); diff --git a/compiler/rustc_thread_pool/src/lib.rs b/compiler/rustc_thread_pool/src/lib.rs index 179d63ed668a1..f1d466b49487a 100644 --- a/compiler/rustc_thread_pool/src/lib.rs +++ b/compiler/rustc_thread_pool/src/lib.rs @@ -152,14 +152,14 @@ enum ErrorKind { /// The following creates a thread pool with 22 threads. /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// let pool = rayon::ThreadPoolBuilder::new().num_threads(22).build().unwrap(); /// ``` /// /// To instead configure the global thread pool, use [`build_global()`]: /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// rayon::ThreadPoolBuilder::new().num_threads(22).build_global().unwrap(); /// ``` /// @@ -315,7 +315,7 @@ impl ThreadPoolBuilder { /// A scoped pool may be useful in combination with scoped thread-local variables. /// /// ``` - /// # use rayon_core as rayon; + /// # use rustc_thred_pool as rayon; /// /// scoped_tls::scoped_thread_local!(static POOL_DATA: Vec); /// @@ -382,7 +382,7 @@ impl ThreadPoolBuilder { /// A minimal spawn handler just needs to call `run()` from an independent thread. /// /// ``` - /// # use rayon_core as rayon; + /// # use rustc_thred_pool as rayon; /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { /// let pool = rayon::ThreadPoolBuilder::new() /// .spawn_handler(|thread| { @@ -400,7 +400,7 @@ impl ThreadPoolBuilder { /// any errors from the thread builder. /// /// ``` - /// # use rayon_core as rayon; + /// # use rustc_thred_pool as rayon; /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { /// let pool = rayon::ThreadPoolBuilder::new() /// .spawn_handler(|thread| { @@ -429,7 +429,7 @@ impl ThreadPoolBuilder { /// [`std::thread::scope`]: https://doc.rust-lang.org/std/thread/fn.scope.html /// /// ``` - /// # use rayon_core as rayon; + /// # use rustc_thred_pool as rayon; /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { /// std::thread::scope(|scope| { /// let pool = rayon::ThreadPoolBuilder::new() diff --git a/compiler/rustc_thread_pool/src/scope/mod.rs b/compiler/rustc_thread_pool/src/scope/mod.rs index 95a4e0b7a18fc..82b3d053dcb88 100644 --- a/compiler/rustc_thread_pool/src/scope/mod.rs +++ b/compiler/rustc_thread_pool/src/scope/mod.rs @@ -84,7 +84,7 @@ struct ScopeBase<'scope> { /// it would be less efficient than the real implementation: /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// pub fn join(oper_a: A, oper_b: B) -> (RA, RB) /// where A: FnOnce() -> RA + Send, /// B: FnOnce() -> RB + Send, @@ -125,7 +125,7 @@ struct ScopeBase<'scope> { /// To see how and when tasks are joined, consider this example: /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// // point start /// rayon::scope(|s| { /// s.spawn(|s| { // task s.1 @@ -193,7 +193,7 @@ struct ScopeBase<'scope> { /// spawned task. /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// let ok: Vec = vec![1, 2, 3]; /// rayon::scope(|s| { /// let bad: Vec = vec![4, 5, 6]; @@ -217,7 +217,7 @@ struct ScopeBase<'scope> { /// in this case including both `ok` *and* `bad`: /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// let ok: Vec = vec![1, 2, 3]; /// rayon::scope(|s| { /// let bad: Vec = vec![4, 5, 6]; @@ -238,7 +238,7 @@ struct ScopeBase<'scope> { /// is a borrow of `ok` and capture *that*: /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// let ok: Vec = vec![1, 2, 3]; /// rayon::scope(|s| { /// let bad: Vec = vec![4, 5, 6]; @@ -260,7 +260,7 @@ struct ScopeBase<'scope> { /// of individual variables: /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// let ok: Vec = vec![1, 2, 3]; /// rayon::scope(|s| { /// let bad: Vec = vec![4, 5, 6]; @@ -312,7 +312,7 @@ where /// [`scope()`]: fn.scope.html /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// // point start /// rayon::scope_fifo(|s| { /// s.spawn_fifo(|s| { // task s.1 @@ -487,7 +487,7 @@ impl<'scope> Scope<'scope> { /// # Examples /// /// ```rust - /// # use rayon_core as rayon; + /// # use rustc_thred_pool as rayon; /// let mut value_a = None; /// let mut value_b = None; /// let mut value_c = None; diff --git a/compiler/rustc_thread_pool/src/spawn/mod.rs b/compiler/rustc_thread_pool/src/spawn/mod.rs index f1679a9823432..92b89ed59480c 100644 --- a/compiler/rustc_thread_pool/src/spawn/mod.rs +++ b/compiler/rustc_thread_pool/src/spawn/mod.rs @@ -50,7 +50,7 @@ use crate::unwind; /// This code creates a Rayon task that increments a global counter. /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// use std::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT}; /// /// static GLOBAL_COUNTER: AtomicUsize = ATOMIC_USIZE_INIT; diff --git a/compiler/rustc_thread_pool/src/thread_pool/mod.rs b/compiler/rustc_thread_pool/src/thread_pool/mod.rs index ce8783cf0d69c..c8644ecf9a9de 100644 --- a/compiler/rustc_thread_pool/src/thread_pool/mod.rs +++ b/compiler/rustc_thread_pool/src/thread_pool/mod.rs @@ -28,7 +28,7 @@ mod test; /// ## Creating a ThreadPool /// /// ```rust -/// # use rayon_core as rayon; +/// # use rustc_thred_pool as rayon; /// let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); /// ``` /// @@ -88,10 +88,10 @@ impl ThreadPool { /// meantime. For example /// /// ```rust - /// # use rayon_core as rayon; + /// # use rustc_thred_pool as rayon; /// fn main() { /// rayon::ThreadPoolBuilder::new().num_threads(1).build_global().unwrap(); - /// let pool = rayon_core::ThreadPoolBuilder::default().build().unwrap(); + /// let pool = rustc_thred_pool::ThreadPoolBuilder::default().build().unwrap(); /// let do_it = || { /// print!("one "); /// pool.install(||{}); @@ -123,7 +123,7 @@ impl ThreadPool { /// ## Using `install()` /// /// ```rust - /// # use rayon_core as rayon; + /// # use rustc_thred_pool as rayon; /// fn main() { /// let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); /// let n = pool.install(|| fib(20)); @@ -172,7 +172,7 @@ impl ThreadPool { /// # Examples /// /// ``` - /// # use rayon_core as rayon; + /// # use rustc_thred_pool as rayon; /// use std::sync::atomic::{AtomicUsize, Ordering}; /// /// fn main() { diff --git a/compiler/rustc_thread_pool/tests/double_init_fail.rs b/compiler/rustc_thread_pool/tests/double_init_fail.rs index 71ed425bb32c7..ef190099293e5 100644 --- a/compiler/rustc_thread_pool/tests/double_init_fail.rs +++ b/compiler/rustc_thread_pool/tests/double_init_fail.rs @@ -1,6 +1,6 @@ use std::error::Error; -use rayon_core::ThreadPoolBuilder; +use rustc_thred_pool::ThreadPoolBuilder; #[test] #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] diff --git a/compiler/rustc_thread_pool/tests/init_zero_threads.rs b/compiler/rustc_thread_pool/tests/init_zero_threads.rs index c1770e57f3c89..1f7e299e3e9c1 100644 --- a/compiler/rustc_thread_pool/tests/init_zero_threads.rs +++ b/compiler/rustc_thread_pool/tests/init_zero_threads.rs @@ -1,4 +1,4 @@ -use rayon_core::ThreadPoolBuilder; +use rustc_thred_pool::ThreadPoolBuilder; #[test] #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] diff --git a/compiler/rustc_thread_pool/tests/scope_join.rs b/compiler/rustc_thread_pool/tests/scope_join.rs index 9d88133bc5b69..0bd33d086cf1c 100644 --- a/compiler/rustc_thread_pool/tests/scope_join.rs +++ b/compiler/rustc_thread_pool/tests/scope_join.rs @@ -4,7 +4,7 @@ where F: FnOnce() + Send, G: FnOnce() + Send, { - rayon_core::scope(|s| { + rustc_thred_pool::scope(|s| { s.spawn(|_| g()); f(); }); diff --git a/compiler/rustc_thread_pool/tests/scoped_threadpool.rs b/compiler/rustc_thread_pool/tests/scoped_threadpool.rs index 8cc2c859c0c6a..e4b0f6c41e1f1 100644 --- a/compiler/rustc_thread_pool/tests/scoped_threadpool.rs +++ b/compiler/rustc_thread_pool/tests/scoped_threadpool.rs @@ -1,5 +1,5 @@ use crossbeam_utils::thread; -use rayon_core::ThreadPoolBuilder; +use rustc_thred_pool::ThreadPoolBuilder; #[derive(PartialEq, Eq, Debug)] struct Local(i32); diff --git a/compiler/rustc_thread_pool/tests/simple_panic.rs b/compiler/rustc_thread_pool/tests/simple_panic.rs index 2564482a47e5d..16896e36fa04d 100644 --- a/compiler/rustc_thread_pool/tests/simple_panic.rs +++ b/compiler/rustc_thread_pool/tests/simple_panic.rs @@ -1,4 +1,4 @@ -use rayon_core::join; +use rustc_thred_pool::join; #[test] #[should_panic(expected = "should panic")] diff --git a/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs b/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs index c7a880de8bb41..49c9ca1d75e9a 100644 --- a/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs +++ b/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs @@ -3,7 +3,7 @@ use std::env; use std::os::unix::process::ExitStatusExt; use std::process::{Command, ExitStatus, Stdio}; -use rayon_core::ThreadPoolBuilder; +use rustc_thred_pool::ThreadPoolBuilder; fn force_stack_overflow(depth: u32) { let mut buffer = [0u8; 1024 * 1024]; diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index fdca7a7a40e20..7cdfe79a40b7d 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -356,6 +356,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "rand", "rand_chacha", "rand_core", + "rand_xorshift", // dependency for doc-tests in rustc_thread_pool "rand_xoshiro", "redox_syscall", "regex", @@ -364,7 +365,6 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "rustc-demangle", "rustc-hash", "rustc-literal-escaper", - "rustc-rayon-core", "rustc-stable-hash", "rustc_apfloat", "rustix", From f52c6eee02fb9a9cfe203ce95c4968c2835c034b Mon Sep 17 00:00:00 2001 From: "Celina G. Val" Date: Wed, 11 Jun 2025 12:32:09 -0700 Subject: [PATCH 09/32] Another round of tidy / warning fixes --- Cargo.toml | 2 +- compiler/rustc_data_structures/src/sync.rs | 2 -- .../rustc_thread_pool/src/broadcast/mod.rs | 8 ++--- .../src/compile_fail/quicksort_race1.rs | 2 +- .../src/compile_fail/quicksort_race2.rs | 2 +- .../src/compile_fail/quicksort_race3.rs | 2 +- .../src/compile_fail/rc_return.rs | 4 +-- .../src/compile_fail/rc_upvar.rs | 2 +- .../src/compile_fail/scope_join_bad.rs | 4 +-- compiler/rustc_thread_pool/src/job.rs | 30 +++++++++------- compiler/rustc_thread_pool/src/join/mod.rs | 6 ++-- compiler/rustc_thread_pool/src/latch.rs | 32 ++++++++--------- compiler/rustc_thread_pool/src/lib.rs | 15 ++++---- compiler/rustc_thread_pool/src/registry.rs | 34 +++++++++---------- compiler/rustc_thread_pool/src/scope/mod.rs | 26 +++++++------- compiler/rustc_thread_pool/src/sleep/mod.rs | 6 ++-- compiler/rustc_thread_pool/src/spawn/mod.rs | 10 +++--- .../rustc_thread_pool/src/thread_pool/mod.rs | 14 ++++---- .../tests/double_init_fail.rs | 4 ++- .../tests/init_zero_threads.rs | 4 ++- .../rustc_thread_pool/tests/scope_join.rs | 4 ++- .../tests/scoped_threadpool.rs | 4 ++- .../rustc_thread_pool/tests/simple_panic.rs | 4 ++- .../tests/stack_overflow_crash.rs | 4 ++- 24 files changed, 120 insertions(+), 105 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c4d2a06f4cb17..e08f14d210167 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ exclude = [ "obj", ] -[profile.release.package.rustc-rayon-core] +[profile.release.package.rustc_thread_pool] # The rustc fork of Rayon has deadlock detection code which intermittently # causes overflows in the CI (see https://github.com/rust-lang/rust/issues/90227) # so we turn overflow checks off for now. diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index 68527dde1f3c8..3881f3c2aa841 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -22,8 +22,6 @@ //! | | | `parking_lot::Mutex` | //! | `RwLock` | `RefCell` | `parking_lot::RwLock` | //! | `MTLock` [^1] | `T` | `Lock` | -//! | | | | -//! | `ParallelIterator` | `Iterator` | `rustc_thread_pool::iter::ParallelIterator` | //! //! [^1]: `MTLock` is similar to `Lock`, but the serial version avoids the cost //! of a `RefCell`. This is appropriate when interior mutability is not diff --git a/compiler/rustc_thread_pool/src/broadcast/mod.rs b/compiler/rustc_thread_pool/src/broadcast/mod.rs index c2b0d47f82995..9545c4b15d8fc 100644 --- a/compiler/rustc_thread_pool/src/broadcast/mod.rs +++ b/compiler/rustc_thread_pool/src/broadcast/mod.rs @@ -6,7 +6,7 @@ use crate::job::{ArcJob, StackJob}; use crate::latch::{CountLatch, LatchRef}; use crate::registry::{Registry, WorkerThread}; -mod test; +mod tests; /// Executes `op` within every thread in the current threadpool. If this is /// called from a non-Rayon thread, it will execute in the global threadpool. @@ -103,18 +103,18 @@ where }; let n_threads = registry.num_threads(); - let current_thread = WorkerThread::current().as_ref(); + let current_thread = unsafe { WorkerThread::current().as_ref() }; let tlv = crate::tlv::get(); let latch = CountLatch::with_count(n_threads, current_thread); let jobs: Vec<_> = (0..n_threads).map(|_| StackJob::new(tlv, &f, LatchRef::new(&latch))).collect(); - let job_refs = jobs.iter().map(|job| job.as_job_ref()); + let job_refs = jobs.iter().map(|job| unsafe { job.as_job_ref() }); registry.inject_broadcast(job_refs); // Wait for all jobs to complete, then collect the results, maybe propagating a panic. latch.wait(current_thread); - jobs.into_iter().map(|job| job.into_result()).collect() + jobs.into_iter().map(|job| unsafe { job.into_result() }).collect() } /// Execute `op` on every thread in the pool. It will be executed on each diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs index 1f7a7b0b429b4..f6dbc76969938 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race1.rs @@ -7,7 +7,7 @@ fn quick_sort(v: &mut [T]) { let mid = partition(v); let (lo, _hi) = v.split_at_mut(mid); - rustc_thred_pool::join(|| quick_sort(lo), || quick_sort(lo)); //~ ERROR + rustc_thread_pool::join(|| quick_sort(lo), || quick_sort(lo)); //~ ERROR } fn partition(v: &mut [T]) -> usize { diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs index 71b695dd8551d..ccd737a700d8f 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race2.rs @@ -7,7 +7,7 @@ fn quick_sort(v: &mut [T]) { let mid = partition(v); let (lo, _hi) = v.split_at_mut(mid); - rustc_thred_pool::join(|| quick_sort(lo), || quick_sort(v)); //~ ERROR + rustc_thread_pool::join(|| quick_sort(lo), || quick_sort(v)); //~ ERROR } fn partition(v: &mut [T]) -> usize { diff --git a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs index 8484cebaae80d..6acdf08443371 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/quicksort_race3.rs @@ -7,7 +7,7 @@ fn quick_sort(v: &mut [T]) { let mid = partition(v); let (_lo, hi) = v.split_at_mut(mid); - rustc_thred_pool::join(|| quick_sort(hi), || quick_sort(hi)); //~ ERROR + rustc_thread_pool::join(|| quick_sort(hi), || quick_sort(hi)); //~ ERROR } fn partition(v: &mut [T]) -> usize { diff --git a/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs b/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs index 509c8d62ad113..165c685aba101 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/rc_return.rs @@ -2,7 +2,7 @@ use std::rc::Rc; -rustc_thred_pool::join(|| Rc::new(22), || ()); //~ ERROR +rustc_thread_pool::join(|| Rc::new(22), || ()); //~ ERROR ``` */ mod left {} @@ -11,7 +11,7 @@ mod left {} use std::rc::Rc; -rustc_thred_pool::join(|| (), || Rc::new(23)); //~ ERROR +rustc_thread_pool::join(|| (), || Rc::new(23)); //~ ERROR ``` */ mod right {} diff --git a/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs b/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs index a27b3c8c39fce..6dc9ead48a09e 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/rc_upvar.rs @@ -3,7 +3,7 @@ use std::rc::Rc; let r = Rc::new(22); -rustc_thred_pool::join(|| r.clone(), || r.clone()); +rustc_thread_pool::join(|| r.clone(), || r.clone()); //~^ ERROR ``` */ diff --git a/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs b/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs index 6e700a483b103..e65abfc3c1e88 100644 --- a/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs +++ b/compiler/rustc_thread_pool/src/compile_fail/scope_join_bad.rs @@ -3,7 +3,7 @@ fn bad_scope(f: F) where F: FnOnce(&i32) + Send, { - rustc_thred_pool::scope(|s| { + rustc_thread_pool::scope(|s| { let x = 22; s.spawn(|_| f(&x)); //~ ERROR `x` does not live long enough }); @@ -13,7 +13,7 @@ fn good_scope(f: F) where F: FnOnce(&i32) + Send, { let x = 22; - rustc_thred_pool::scope(|s| { + rustc_thread_pool::scope(|s| { s.spawn(|_| f(&x)); }); } diff --git a/compiler/rustc_thread_pool/src/job.rs b/compiler/rustc_thread_pool/src/job.rs index 3241914ba81c7..e6e84ac2320bf 100644 --- a/compiler/rustc_thread_pool/src/job.rs +++ b/compiler/rustc_thread_pool/src/job.rs @@ -61,7 +61,7 @@ impl JobRef { #[inline] pub(super) unsafe fn execute(self) { - (self.execute_fn)(self.pointer) + unsafe { (self.execute_fn)(self.pointer) } } } @@ -97,7 +97,7 @@ where } pub(super) unsafe fn as_job_ref(&self) -> JobRef { - JobRef::new(self) + unsafe { JobRef::new(self) } } pub(super) unsafe fn run_inline(self, stolen: bool) -> R { @@ -116,12 +116,16 @@ where R: Send, { unsafe fn execute(this: *const ()) { - let this = &*(this as *const Self); + let this = unsafe { &*(this as *const Self) }; tlv::set(this.tlv); let abort = unwind::AbortIfPanic; - let func = (*this.func.get()).take().unwrap(); - (*this.result.get()) = JobResult::call(func); - Latch::set(&this.latch); + let func = unsafe { (*this.func.get()).take().unwrap() }; + unsafe { + (*this.result.get()) = JobResult::call(func); + } + unsafe { + Latch::set(&this.latch); + } mem::forget(abort); } } @@ -152,7 +156,7 @@ where /// lifetimes, so it is up to you to ensure that this JobRef /// doesn't outlive any data that it closes over. pub(super) unsafe fn into_job_ref(self: Box) -> JobRef { - JobRef::new(Box::into_raw(self)) + unsafe { JobRef::new(Box::into_raw(self)) } } /// Creates a static `JobRef` from this job. @@ -169,7 +173,7 @@ where BODY: FnOnce() + Send, { unsafe fn execute(this: *const ()) { - let this = Box::from_raw(this as *mut Self); + let this = unsafe { Box::from_raw(this as *mut Self) }; tlv::set(this.tlv); (this.job)(); } @@ -196,7 +200,7 @@ where /// lifetimes, so it is up to you to ensure that this JobRef /// doesn't outlive any data that it closes over. pub(super) unsafe fn as_job_ref(this: &Arc) -> JobRef { - JobRef::new(Arc::into_raw(Arc::clone(this))) + unsafe { JobRef::new(Arc::into_raw(Arc::clone(this))) } } /// Creates a static `JobRef` from this job. @@ -213,7 +217,7 @@ where BODY: Fn() + Send + Sync, { unsafe fn execute(this: *const ()) { - let this = Arc::from_raw(this as *mut Self); + let this = unsafe { Arc::from_raw(this as *mut Self) }; (this.job)(); } } @@ -254,17 +258,17 @@ impl JobFifo { // jobs in a thread's deque may be popped from the back (LIFO) or stolen from the front // (FIFO), but either way they will end up popping from the front of this queue. self.inner.push(job_ref); - JobRef::new(self) + unsafe { JobRef::new(self) } } } impl Job for JobFifo { unsafe fn execute(this: *const ()) { // We "execute" a queue by executing its first job, FIFO. - let this = &*(this as *const Self); + let this = unsafe { &*(this as *const Self) }; loop { match this.inner.steal() { - Steal::Success(job_ref) => break job_ref.execute(), + Steal::Success(job_ref) => break unsafe { job_ref.execute() }, Steal::Empty => panic!("FIFO is empty"), Steal::Retry => {} } diff --git a/compiler/rustc_thread_pool/src/join/mod.rs b/compiler/rustc_thread_pool/src/join/mod.rs index e48d17f16a3ac..f285362c19b1b 100644 --- a/compiler/rustc_thread_pool/src/join/mod.rs +++ b/compiler/rustc_thread_pool/src/join/mod.rs @@ -7,7 +7,7 @@ use crate::tlv::{self, Tlv}; use crate::{FnContext, unwind}; #[cfg(test)] -mod test; +mod tests; /// Takes two closures and *potentially* runs them in parallel. It /// returns a pair of the results from those closures. @@ -41,7 +41,7 @@ mod test; /// [the `par_sort` method]: ../rayon/slice/trait.ParallelSliceMut.html#method.par_sort /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// let mut v = vec![5, 1, 8, 22, 0, 44]; /// quick_sort(&mut v); /// assert_eq!(v, vec![0, 1, 5, 8, 22, 44]); @@ -192,7 +192,7 @@ unsafe fn join_recover_from_panic( err: Box, tlv: Tlv, ) -> ! { - worker_thread.wait_until(job_b_latch); + unsafe { worker_thread.wait_until(job_b_latch) }; // Restore the TLV since we might have run some jobs overwriting it when waiting for job b. tlv::set(tlv); diff --git a/compiler/rustc_thread_pool/src/latch.rs b/compiler/rustc_thread_pool/src/latch.rs index f2f806e01845f..49ba62d3bea3d 100644 --- a/compiler/rustc_thread_pool/src/latch.rs +++ b/compiler/rustc_thread_pool/src/latch.rs @@ -116,7 +116,7 @@ impl CoreLatch { /// latch code. #[inline] unsafe fn set(this: *const Self) -> bool { - let old_state = (*this).state.swap(SET, Ordering::AcqRel); + let old_state = unsafe { (*this).state.swap(SET, Ordering::AcqRel) }; old_state == SLEEPING } @@ -185,13 +185,13 @@ impl<'r> Latch for SpinLatch<'r> { unsafe fn set(this: *const Self) { let cross_registry; - let registry: &Registry = if (*this).cross { + let registry: &Registry = if unsafe { (*this).cross } { // Ensure the registry stays alive while we notify it. // Otherwise, it would be possible that we set the spin // latch and the other thread sees it and exits, causing // the registry to be deallocated, all before we get a // chance to invoke `registry.notify_worker_latch_is_set`. - cross_registry = Arc::clone((*this).registry); + cross_registry = Arc::clone(unsafe { (*this).registry }); &cross_registry } else { // If this is not a "cross-registry" spin-latch, then the @@ -199,12 +199,12 @@ impl<'r> Latch for SpinLatch<'r> { // that the registry stays alive. However, that doesn't // include this *particular* `Arc` handle if the waiting // thread then exits, so we must completely dereference it. - (*this).registry + unsafe { (*this).registry } }; - let target_worker_index = (*this).target_worker_index; + let target_worker_index = unsafe { (*this).target_worker_index }; // NOTE: Once we `set`, the target may proceed and invalidate `this`! - if CoreLatch::set(&(*this).core_latch) { + if unsafe { CoreLatch::set(&(*this).core_latch) } { // Subtle: at this point, we can no longer read from // `self`, because the thread owning this spin latch may // have awoken and deallocated the latch. Therefore, we @@ -249,9 +249,9 @@ impl LockLatch { impl Latch for LockLatch { #[inline] unsafe fn set(this: *const Self) { - let mut guard = (*this).m.lock().unwrap(); + let mut guard = unsafe { (*this).m.lock().unwrap() }; *guard = true; - (*this).v.notify_all(); + unsafe { (*this).v.notify_all() }; } } @@ -286,7 +286,7 @@ impl OnceLatch { registry: &Registry, target_worker_index: usize, ) { - if CoreLatch::set(&(*this).core_latch) { + if unsafe { CoreLatch::set(&(*this).core_latch) } { registry.notify_worker_latch_is_set(target_worker_index); } } @@ -384,17 +384,17 @@ impl CountLatch { impl Latch for CountLatch { #[inline] unsafe fn set(this: *const Self) { - if (*this).counter.fetch_sub(1, Ordering::SeqCst) == 1 { + if unsafe { (*this).counter.fetch_sub(1, Ordering::SeqCst) == 1 } { // NOTE: Once we call `set` on the internal `latch`, // the target may proceed and invalidate `this`! - match (*this).kind { - CountLatchKind::Stealing { ref latch, ref registry, worker_index } => { + match unsafe { &(*this).kind } { + CountLatchKind::Stealing { latch, registry, worker_index } => { let registry = Arc::clone(registry); - if CoreLatch::set(latch) { - registry.notify_worker_latch_is_set(worker_index); + if unsafe { CoreLatch::set(latch) } { + registry.notify_worker_latch_is_set(*worker_index); } } - CountLatchKind::Blocking { ref latch } => LockLatch::set(latch), + CountLatchKind::Blocking { latch } => unsafe { LockLatch::set(latch) }, } } } @@ -426,6 +426,6 @@ impl Deref for LatchRef<'_, L> { impl Latch for LatchRef<'_, L> { #[inline] unsafe fn set(this: *const Self) { - L::set((*this).inner); + unsafe { L::set((*this).inner) }; } } diff --git a/compiler/rustc_thread_pool/src/lib.rs b/compiler/rustc_thread_pool/src/lib.rs index f1d466b49487a..34252d919e38e 100644 --- a/compiler/rustc_thread_pool/src/lib.rs +++ b/compiler/rustc_thread_pool/src/lib.rs @@ -61,6 +61,7 @@ //! conflicting requirements will need to be resolved before the build will //! succeed. +#![cfg_attr(test, allow(unused_crate_dependencies))] #![warn(rust_2018_idioms)] use std::any::Any; @@ -85,7 +86,7 @@ mod unwind; mod worker_local; mod compile_fail; -mod test; +mod tests; pub mod tlv; @@ -152,14 +153,14 @@ enum ErrorKind { /// The following creates a thread pool with 22 threads. /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// let pool = rayon::ThreadPoolBuilder::new().num_threads(22).build().unwrap(); /// ``` /// /// To instead configure the global thread pool, use [`build_global()`]: /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// rayon::ThreadPoolBuilder::new().num_threads(22).build_global().unwrap(); /// ``` /// @@ -315,7 +316,7 @@ impl ThreadPoolBuilder { /// A scoped pool may be useful in combination with scoped thread-local variables. /// /// ``` - /// # use rustc_thred_pool as rayon; + /// # use rustc_thread_pool as rayon; /// /// scoped_tls::scoped_thread_local!(static POOL_DATA: Vec); /// @@ -382,7 +383,7 @@ impl ThreadPoolBuilder { /// A minimal spawn handler just needs to call `run()` from an independent thread. /// /// ``` - /// # use rustc_thred_pool as rayon; + /// # use rustc_thread_pool as rayon; /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { /// let pool = rayon::ThreadPoolBuilder::new() /// .spawn_handler(|thread| { @@ -400,7 +401,7 @@ impl ThreadPoolBuilder { /// any errors from the thread builder. /// /// ``` - /// # use rustc_thred_pool as rayon; + /// # use rustc_thread_pool as rayon; /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { /// let pool = rayon::ThreadPoolBuilder::new() /// .spawn_handler(|thread| { @@ -429,7 +430,7 @@ impl ThreadPoolBuilder { /// [`std::thread::scope`]: https://doc.rust-lang.org/std/thread/fn.scope.html /// /// ``` - /// # use rustc_thred_pool as rayon; + /// # use rustc_thread_pool as rayon; /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { /// std::thread::scope(|scope| { /// let pool = rayon::ThreadPoolBuilder::new() diff --git a/compiler/rustc_thread_pool/src/registry.rs b/compiler/rustc_thread_pool/src/registry.rs index 2848556aab6a1..03a01aa29d2af 100644 --- a/compiler/rustc_thread_pool/src/registry.rs +++ b/compiler/rustc_thread_pool/src/registry.rs @@ -533,16 +533,16 @@ impl Registry { |injected| { let worker_thread = WorkerThread::current(); assert!(injected && !worker_thread.is_null()); - op(&*worker_thread, true) + op(unsafe { &*worker_thread }, true) }, LatchRef::new(l), ); - self.inject(job.as_job_ref()); + self.inject(unsafe { job.as_job_ref() }); self.release_thread(); job.latch.wait_and_reset(); // Make sure we can use the same latch again next time. self.acquire_thread(); - job.into_result() + unsafe { job.into_result() } }) } @@ -561,13 +561,13 @@ impl Registry { |injected| { let worker_thread = WorkerThread::current(); assert!(injected && !worker_thread.is_null()); - op(&*worker_thread, true) + op(unsafe { &*worker_thread }, true) }, latch, ); - self.inject(job.as_job_ref()); - current_thread.wait_until(&job.latch); - job.into_result() + self.inject(unsafe { job.as_job_ref() }); + unsafe { current_thread.wait_until(&job.latch) }; + unsafe { job.into_result() } } /// Increments the terminate counter. This increment should be @@ -759,7 +759,7 @@ impl WorkerThread { #[inline] pub(super) unsafe fn push_fifo(&self, job: JobRef) { - self.push(self.fifo.push(job)); + unsafe { self.push(self.fifo.push(job)) }; } #[inline] @@ -798,7 +798,7 @@ impl WorkerThread { pub(super) unsafe fn wait_until(&self, latch: &L) { let latch = latch.as_core_latch(); if !latch.probe() { - self.wait_until_cold(latch); + unsafe { self.wait_until_cold(latch) }; } } @@ -815,7 +815,7 @@ impl WorkerThread { // Check for local work *before* we start marking ourself idle, // especially to avoid modifying shared sleep state. if let Some(job) = self.take_local_job() { - self.execute(job); + unsafe { self.execute(job) }; continue; } @@ -823,7 +823,7 @@ impl WorkerThread { while !latch.probe() { if let Some(job) = self.find_work() { self.registry.sleep.work_found(); - self.execute(job); + unsafe { self.execute(job) }; // The job might have injected local work, so go back to the outer loop. continue 'outer; } else { @@ -846,13 +846,13 @@ impl WorkerThread { let index = self.index; registry.acquire_thread(); - self.wait_until(®istry.thread_infos[index].terminate); + unsafe { self.wait_until(®istry.thread_infos[index].terminate) }; // Should not be any work left in our queue. debug_assert!(self.take_local_job().is_none()); // Let registry know we are done - Latch::set(®istry.thread_infos[index].stopped); + unsafe { Latch::set(®istry.thread_infos[index].stopped) }; } fn find_work(&self) -> Option { @@ -886,7 +886,7 @@ impl WorkerThread { #[inline] pub(super) unsafe fn execute(&self, job: JobRef) { - job.execute(); + unsafe { job.execute() }; } /// Try to steal a single job and return it. @@ -932,12 +932,12 @@ impl WorkerThread { unsafe fn main_loop(thread: ThreadBuilder) { let worker_thread = &WorkerThread::from(thread); - WorkerThread::set_current(worker_thread); + unsafe { WorkerThread::set_current(worker_thread) }; let registry = &*worker_thread.registry; let index = worker_thread.index; // let registry know we are ready to do work - Latch::set(®istry.thread_infos[index].primed); + unsafe { Latch::set(®istry.thread_infos[index].primed) }; // Worker threads should not panic. If they do, just abort, as the // internal state of the threadpool is corrupted. Note that if @@ -949,7 +949,7 @@ unsafe fn main_loop(thread: ThreadBuilder) { registry.catch_unwind(|| handler(index)); } - worker_thread.wait_until_out_of_work(); + unsafe { worker_thread.wait_until_out_of_work() }; // Normal termination, do not abort. mem::forget(abort_guard); diff --git a/compiler/rustc_thread_pool/src/scope/mod.rs b/compiler/rustc_thread_pool/src/scope/mod.rs index 82b3d053dcb88..55e58b3509d7d 100644 --- a/compiler/rustc_thread_pool/src/scope/mod.rs +++ b/compiler/rustc_thread_pool/src/scope/mod.rs @@ -20,7 +20,7 @@ use crate::tlv::{self, Tlv}; use crate::unwind; #[cfg(test)] -mod test; +mod tests; /// Represents a fork-join scope which can be used to spawn any number of tasks. /// See [`scope()`] for more information. @@ -84,7 +84,7 @@ struct ScopeBase<'scope> { /// it would be less efficient than the real implementation: /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// pub fn join(oper_a: A, oper_b: B) -> (RA, RB) /// where A: FnOnce() -> RA + Send, /// B: FnOnce() -> RB + Send, @@ -125,7 +125,7 @@ struct ScopeBase<'scope> { /// To see how and when tasks are joined, consider this example: /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// // point start /// rayon::scope(|s| { /// s.spawn(|s| { // task s.1 @@ -193,7 +193,7 @@ struct ScopeBase<'scope> { /// spawned task. /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// let ok: Vec = vec![1, 2, 3]; /// rayon::scope(|s| { /// let bad: Vec = vec![4, 5, 6]; @@ -217,7 +217,7 @@ struct ScopeBase<'scope> { /// in this case including both `ok` *and* `bad`: /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// let ok: Vec = vec![1, 2, 3]; /// rayon::scope(|s| { /// let bad: Vec = vec![4, 5, 6]; @@ -238,7 +238,7 @@ struct ScopeBase<'scope> { /// is a borrow of `ok` and capture *that*: /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// let ok: Vec = vec![1, 2, 3]; /// rayon::scope(|s| { /// let bad: Vec = vec![4, 5, 6]; @@ -260,7 +260,7 @@ struct ScopeBase<'scope> { /// of individual variables: /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// let ok: Vec = vec![1, 2, 3]; /// rayon::scope(|s| { /// let bad: Vec = vec![4, 5, 6]; @@ -312,7 +312,7 @@ where /// [`scope()`]: fn.scope.html /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// // point start /// rayon::scope_fifo(|s| { /// s.spawn_fifo(|s| { // task s.1 @@ -487,7 +487,7 @@ impl<'scope> Scope<'scope> { /// # Examples /// /// ```rust - /// # use rustc_thred_pool as rayon; + /// # use rustc_thread_pool as rayon; /// let mut value_a = None; /// let mut value_b = None; /// let mut value_c = None; @@ -686,7 +686,7 @@ impl<'scope> ScopeBase<'scope> { where FUNC: FnOnce(), { - let _: Option<()> = Self::execute_job_closure(this, func); + let _: Option<()> = unsafe { Self::execute_job_closure(this, func) }; } /// Executes `func` as a job in scope. Adjusts the "job completed" @@ -699,11 +699,11 @@ impl<'scope> ScopeBase<'scope> { let result = match unwind::halt_unwinding(func) { Ok(r) => Some(r), Err(err) => { - (*this).job_panicked(err); + unsafe { (*this).job_panicked(err) }; None } }; - Latch::set(&(*this).job_completed_latch); + unsafe { Latch::set(&(*this).job_completed_latch) }; result } @@ -778,6 +778,6 @@ unsafe impl Sync for ScopePtr {} impl ScopePtr { // Helper to avoid disjoint captures of `scope_ptr.0` unsafe fn as_ref(&self) -> &T { - &*self.0 + unsafe { &*self.0 } } } diff --git a/compiler/rustc_thread_pool/src/sleep/mod.rs b/compiler/rustc_thread_pool/src/sleep/mod.rs index bee7c82c450c0..a9cdf68cc7ebf 100644 --- a/compiler/rustc_thread_pool/src/sleep/mod.rs +++ b/compiler/rustc_thread_pool/src/sleep/mod.rs @@ -31,7 +31,7 @@ struct SleepData { impl SleepData { /// Checks if the conditions for a deadlock holds and if so calls the deadlock handler #[inline] - pub fn deadlock_check(&self, deadlock_handler: &Option>) { + pub(super) fn deadlock_check(&self, deadlock_handler: &Option>) { if self.active_threads == 0 && self.blocked_threads > 0 { (deadlock_handler.as_ref().unwrap())(); } @@ -102,7 +102,7 @@ impl Sleep { /// Mark a Rayon worker thread as blocked. This triggers the deadlock handler /// if no other worker thread is active #[inline] - pub fn mark_blocked(&self, deadlock_handler: &Option>) { + pub(super) fn mark_blocked(&self, deadlock_handler: &Option>) { let mut data = self.data.lock().unwrap(); debug_assert!(data.active_threads > 0); debug_assert!(data.blocked_threads < data.worker_count); @@ -115,7 +115,7 @@ impl Sleep { /// Mark a previously blocked Rayon worker thread as unblocked #[inline] - pub fn mark_unblocked(&self) { + pub(super) fn mark_unblocked(&self) { let mut data = self.data.lock().unwrap(); debug_assert!(data.active_threads < data.worker_count); debug_assert!(data.blocked_threads > 0); diff --git a/compiler/rustc_thread_pool/src/spawn/mod.rs b/compiler/rustc_thread_pool/src/spawn/mod.rs index 92b89ed59480c..040a02bfa6769 100644 --- a/compiler/rustc_thread_pool/src/spawn/mod.rs +++ b/compiler/rustc_thread_pool/src/spawn/mod.rs @@ -50,7 +50,7 @@ use crate::unwind; /// This code creates a Rayon task that increments a global counter. /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// use std::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT}; /// /// static GLOBAL_COUNTER: AtomicUsize = ATOMIC_USIZE_INIT; @@ -80,7 +80,7 @@ where // be able to panic, and hence the data won't leak but will be // enqueued into some deque for later execution. let abort_guard = unwind::AbortIfPanic; // just in case we are wrong, and code CAN panic - let job_ref = spawn_job(func, registry); + let job_ref = unsafe { spawn_job(func, registry) }; registry.inject_or_push(job_ref); mem::forget(abort_guard); } @@ -150,16 +150,16 @@ where // be able to panic, and hence the data won't leak but will be // enqueued into some deque for later execution. let abort_guard = unwind::AbortIfPanic; // just in case we are wrong, and code CAN panic - let job_ref = spawn_job(func, registry); + let job_ref = unsafe { spawn_job(func, registry) }; // If we're in the pool, use our thread's private fifo for this thread to execute // in a locally-FIFO order. Otherwise, just use the pool's global injector. match registry.current_thread() { - Some(worker) => worker.push_fifo(job_ref), + Some(worker) => unsafe { worker.push_fifo(job_ref) }, None => registry.inject(job_ref), } mem::forget(abort_guard); } #[cfg(test)] -mod test; +mod tests; diff --git a/compiler/rustc_thread_pool/src/thread_pool/mod.rs b/compiler/rustc_thread_pool/src/thread_pool/mod.rs index c8644ecf9a9de..3294e2a77cbe6 100644 --- a/compiler/rustc_thread_pool/src/thread_pool/mod.rs +++ b/compiler/rustc_thread_pool/src/thread_pool/mod.rs @@ -14,7 +14,7 @@ use crate::{ Scope, ScopeFifo, ThreadPoolBuildError, ThreadPoolBuilder, join, scope, scope_fifo, spawn, }; -mod test; +mod tests; /// Represents a user created [thread-pool]. /// @@ -28,7 +28,7 @@ mod test; /// ## Creating a ThreadPool /// /// ```rust -/// # use rustc_thred_pool as rayon; +/// # use rustc_thread_pool as rayon; /// let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); /// ``` /// @@ -88,10 +88,10 @@ impl ThreadPool { /// meantime. For example /// /// ```rust - /// # use rustc_thred_pool as rayon; + /// # use rustc_thread_pool as rayon; /// fn main() { /// rayon::ThreadPoolBuilder::new().num_threads(1).build_global().unwrap(); - /// let pool = rustc_thred_pool::ThreadPoolBuilder::default().build().unwrap(); + /// let pool = rustc_thread_pool::ThreadPoolBuilder::default().build().unwrap(); /// let do_it = || { /// print!("one "); /// pool.install(||{}); @@ -123,7 +123,7 @@ impl ThreadPool { /// ## Using `install()` /// /// ```rust - /// # use rustc_thred_pool as rayon; + /// # use rustc_thread_pool as rayon; /// fn main() { /// let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); /// let n = pool.install(|| fib(20)); @@ -172,7 +172,7 @@ impl ThreadPool { /// # Examples /// /// ``` - /// # use rustc_thred_pool as rayon; + /// # use rustc_thread_pool as rayon; /// use std::sync::atomic::{AtomicUsize, Ordering}; /// /// fn main() { @@ -401,7 +401,7 @@ impl ThreadPool { } pub(crate) fn wait_until_stopped(self) { - let registry = self.registry.clone(); + let registry = Arc::clone(&self.registry); drop(self); registry.wait_until_stopped(); } diff --git a/compiler/rustc_thread_pool/tests/double_init_fail.rs b/compiler/rustc_thread_pool/tests/double_init_fail.rs index ef190099293e5..85e509518d433 100644 --- a/compiler/rustc_thread_pool/tests/double_init_fail.rs +++ b/compiler/rustc_thread_pool/tests/double_init_fail.rs @@ -1,6 +1,8 @@ +#![allow(unused_crate_dependencies)] + use std::error::Error; -use rustc_thred_pool::ThreadPoolBuilder; +use rustc_thread_pool::ThreadPoolBuilder; #[test] #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] diff --git a/compiler/rustc_thread_pool/tests/init_zero_threads.rs b/compiler/rustc_thread_pool/tests/init_zero_threads.rs index 1f7e299e3e9c1..261493fcb7b7a 100644 --- a/compiler/rustc_thread_pool/tests/init_zero_threads.rs +++ b/compiler/rustc_thread_pool/tests/init_zero_threads.rs @@ -1,4 +1,6 @@ -use rustc_thred_pool::ThreadPoolBuilder; +#![allow(unused_crate_dependencies)] + +use rustc_thread_pool::ThreadPoolBuilder; #[test] #[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] diff --git a/compiler/rustc_thread_pool/tests/scope_join.rs b/compiler/rustc_thread_pool/tests/scope_join.rs index 0bd33d086cf1c..83468da81c013 100644 --- a/compiler/rustc_thread_pool/tests/scope_join.rs +++ b/compiler/rustc_thread_pool/tests/scope_join.rs @@ -1,10 +1,12 @@ +#![allow(unused_crate_dependencies)] + /// Test that one can emulate join with `scope`: fn pseudo_join(f: F, g: G) where F: FnOnce() + Send, G: FnOnce() + Send, { - rustc_thred_pool::scope(|s| { + rustc_thread_pool::scope(|s| { s.spawn(|_| g()); f(); }); diff --git a/compiler/rustc_thread_pool/tests/scoped_threadpool.rs b/compiler/rustc_thread_pool/tests/scoped_threadpool.rs index e4b0f6c41e1f1..295da650e8805 100644 --- a/compiler/rustc_thread_pool/tests/scoped_threadpool.rs +++ b/compiler/rustc_thread_pool/tests/scoped_threadpool.rs @@ -1,5 +1,7 @@ +#![allow(unused_crate_dependencies)] + use crossbeam_utils::thread; -use rustc_thred_pool::ThreadPoolBuilder; +use rustc_thread_pool::ThreadPoolBuilder; #[derive(PartialEq, Eq, Debug)] struct Local(i32); diff --git a/compiler/rustc_thread_pool/tests/simple_panic.rs b/compiler/rustc_thread_pool/tests/simple_panic.rs index 16896e36fa04d..b35b4d632d2b9 100644 --- a/compiler/rustc_thread_pool/tests/simple_panic.rs +++ b/compiler/rustc_thread_pool/tests/simple_panic.rs @@ -1,4 +1,6 @@ -use rustc_thred_pool::join; +#![allow(unused_crate_dependencies)] + +use rustc_thread_pool::join; #[test] #[should_panic(expected = "should panic")] diff --git a/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs b/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs index 49c9ca1d75e9a..805b6d8ee3f2f 100644 --- a/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs +++ b/compiler/rustc_thread_pool/tests/stack_overflow_crash.rs @@ -1,9 +1,11 @@ +#![allow(unused_crate_dependencies)] + use std::env; #[cfg(target_os = "linux")] use std::os::unix::process::ExitStatusExt; use std::process::{Command, ExitStatus, Stdio}; -use rustc_thred_pool::ThreadPoolBuilder; +use rustc_thread_pool::ThreadPoolBuilder; fn force_stack_overflow(depth: u32) { let mut buffer = [0u8; 1024 * 1024]; From ce457e1c2628e41a02bc2dcdc44346e7c7d0a9e2 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 8 Mar 2025 20:29:17 +0100 Subject: [PATCH 10/32] Get rid of `EscapeDebugInner`. --- library/core/src/ascii.rs | 16 +-- library/core/src/char/mod.rs | 62 +++------ library/core/src/escape.rs | 222 ++++++++++++++++++++++++++------ library/core/src/slice/ascii.rs | 2 +- 4 files changed, 210 insertions(+), 92 deletions(-) diff --git a/library/core/src/ascii.rs b/library/core/src/ascii.rs index 5b3711b4071ab..d3c6c046e717f 100644 --- a/library/core/src/ascii.rs +++ b/library/core/src/ascii.rs @@ -9,9 +9,10 @@ #![stable(feature = "core_ascii", since = "1.26.0")] +use crate::escape::{AlwaysEscaped, EscapeIterInner}; +use crate::fmt; use crate::iter::FusedIterator; use crate::num::NonZero; -use crate::{escape, fmt}; mod ascii_char; #[unstable(feature = "ascii_char", issue = "110998")] @@ -24,7 +25,7 @@ pub use ascii_char::AsciiChar as Char; #[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] #[derive(Clone)] -pub struct EscapeDefault(escape::EscapeIterInner<4>); +pub struct EscapeDefault(EscapeIterInner<4, AlwaysEscaped>); /// Returns an iterator that produces an escaped version of a `u8`. /// @@ -96,17 +97,12 @@ pub fn escape_default(c: u8) -> EscapeDefault { impl EscapeDefault { #[inline] pub(crate) const fn new(c: u8) -> Self { - Self(escape::EscapeIterInner::ascii(c)) + Self(EscapeIterInner::ascii(c)) } #[inline] pub(crate) fn empty() -> Self { - Self(escape::EscapeIterInner::empty()) - } - - #[inline] - pub(crate) fn as_str(&self) -> &str { - self.0.as_str() + Self(EscapeIterInner::empty()) } } @@ -168,7 +164,7 @@ impl FusedIterator for EscapeDefault {} #[stable(feature = "ascii_escape_display", since = "1.39.0")] impl fmt::Display for EscapeDefault { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.0.as_str()) + fmt::Display::fmt(&self.0, f) } } diff --git a/library/core/src/char/mod.rs b/library/core/src/char/mod.rs index 5b9f0e2143f5d..82a3f6f916be3 100644 --- a/library/core/src/char/mod.rs +++ b/library/core/src/char/mod.rs @@ -44,7 +44,7 @@ pub use self::methods::{encode_utf8_raw, encode_utf8_raw_unchecked}; // perma-un use crate::ascii; pub(crate) use self::methods::EscapeDebugExtArgs; use crate::error::Error; -use crate::escape; +use crate::escape::{AlwaysEscaped, EscapeIterInner, MaybeEscaped}; use crate::fmt::{self, Write}; use crate::iter::{FusedIterator, TrustedLen, TrustedRandomAccess, TrustedRandomAccessNoCoerce}; use crate::num::NonZero; @@ -161,12 +161,12 @@ pub const fn from_digit(num: u32, radix: u32) -> Option { /// [`escape_unicode`]: char::escape_unicode #[derive(Clone, Debug)] #[stable(feature = "rust1", since = "1.0.0")] -pub struct EscapeUnicode(escape::EscapeIterInner<10>); +pub struct EscapeUnicode(EscapeIterInner<10, AlwaysEscaped>); impl EscapeUnicode { #[inline] const fn new(c: char) -> Self { - Self(escape::EscapeIterInner::unicode(c)) + Self(EscapeIterInner::unicode(c)) } } @@ -215,7 +215,7 @@ impl FusedIterator for EscapeUnicode {} #[stable(feature = "char_struct_display", since = "1.16.0")] impl fmt::Display for EscapeUnicode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.0.as_str()) + fmt::Display::fmt(&self.0, f) } } @@ -227,22 +227,22 @@ impl fmt::Display for EscapeUnicode { /// [`escape_default`]: char::escape_default #[derive(Clone, Debug)] #[stable(feature = "rust1", since = "1.0.0")] -pub struct EscapeDefault(escape::EscapeIterInner<10>); +pub struct EscapeDefault(EscapeIterInner<10, AlwaysEscaped>); impl EscapeDefault { #[inline] const fn printable(c: ascii::Char) -> Self { - Self(escape::EscapeIterInner::ascii(c.to_u8())) + Self(EscapeIterInner::ascii(c.to_u8())) } #[inline] const fn backslash(c: ascii::Char) -> Self { - Self(escape::EscapeIterInner::backslash(c)) + Self(EscapeIterInner::backslash(c)) } #[inline] const fn unicode(c: char) -> Self { - Self(escape::EscapeIterInner::unicode(c)) + Self(EscapeIterInner::unicode(c)) } } @@ -290,8 +290,9 @@ impl FusedIterator for EscapeDefault {} #[stable(feature = "char_struct_display", since = "1.16.0")] impl fmt::Display for EscapeDefault { + #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.0.as_str()) + fmt::Display::fmt(&self.0, f) } } @@ -303,37 +304,22 @@ impl fmt::Display for EscapeDefault { /// [`escape_debug`]: char::escape_debug #[stable(feature = "char_escape_debug", since = "1.20.0")] #[derive(Clone, Debug)] -pub struct EscapeDebug(EscapeDebugInner); - -#[derive(Clone, Debug)] -// Note: It’s possible to manually encode the EscapeDebugInner inside of -// EscapeIterInner (e.g. with alive=254..255 indicating that data[0..4] holds -// a char) which would likely result in a more optimised code. For now we use -// the option easier to implement. -enum EscapeDebugInner { - Bytes(escape::EscapeIterInner<10>), - Char(char), -} +pub struct EscapeDebug(EscapeIterInner<10, MaybeEscaped>); impl EscapeDebug { #[inline] const fn printable(chr: char) -> Self { - Self(EscapeDebugInner::Char(chr)) + Self(EscapeIterInner::printable(chr)) } #[inline] const fn backslash(c: ascii::Char) -> Self { - Self(EscapeDebugInner::Bytes(escape::EscapeIterInner::backslash(c))) + Self(EscapeIterInner::backslash(c)) } #[inline] const fn unicode(c: char) -> Self { - Self(EscapeDebugInner::Bytes(escape::EscapeIterInner::unicode(c))) - } - - #[inline] - fn clear(&mut self) { - self.0 = EscapeDebugInner::Bytes(escape::EscapeIterInner::empty()); + Self(EscapeIterInner::unicode(c)) } } @@ -343,13 +329,7 @@ impl Iterator for EscapeDebug { #[inline] fn next(&mut self) -> Option { - match self.0 { - EscapeDebugInner::Bytes(ref mut bytes) => bytes.next().map(char::from), - EscapeDebugInner::Char(chr) => { - self.clear(); - Some(chr) - } - } + self.0.next() } #[inline] @@ -367,10 +347,7 @@ impl Iterator for EscapeDebug { #[stable(feature = "char_escape_debug", since = "1.20.0")] impl ExactSizeIterator for EscapeDebug { fn len(&self) -> usize { - match &self.0 { - EscapeDebugInner::Bytes(bytes) => bytes.len(), - EscapeDebugInner::Char(_) => 1, - } + self.0.len() } } @@ -379,11 +356,9 @@ impl FusedIterator for EscapeDebug {} #[stable(feature = "char_escape_debug", since = "1.20.0")] impl fmt::Display for EscapeDebug { + #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.0 { - EscapeDebugInner::Bytes(bytes) => f.write_str(bytes.as_str()), - EscapeDebugInner::Char(chr) => f.write_char(*chr), - } + fmt::Display::fmt(&self.0, f) } } @@ -480,6 +455,7 @@ macro_rules! casemappingiter_impls { #[stable(feature = "char_struct_display", since = "1.16.0")] impl fmt::Display for $ITER_NAME { + #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.0, f) } diff --git a/library/core/src/escape.rs b/library/core/src/escape.rs index 0c3329f676eeb..f459c58270818 100644 --- a/library/core/src/escape.rs +++ b/library/core/src/escape.rs @@ -1,11 +1,16 @@ //! Helper code for character escaping. use crate::ascii; +use crate::fmt::{self, Write}; +use crate::marker::PhantomData; use crate::num::NonZero; use crate::ops::Range; const HEX_DIGITS: [ascii::Char; 16] = *b"0123456789abcdef".as_ascii().unwrap(); +/// Escapes a character with `\x` representation. +/// +/// Returns a buffer with the escaped representation and its corresponding range. #[inline] const fn backslash(a: ascii::Char) -> ([ascii::Char; N], Range) { const { assert!(N >= 2) }; @@ -18,6 +23,9 @@ const fn backslash(a: ascii::Char) -> ([ascii::Char; N], Range(byte: u8) -> ([ascii::Char; N], Range) { const { assert!(N >= 4) }; @@ -35,6 +43,7 @@ const fn hex_escape(byte: u8) -> ([ascii::Char; N], Range) { (output, 0..4) } +/// Returns a buffer with the verbatim character and its corresponding range. #[inline] const fn verbatim(a: ascii::Char) -> ([ascii::Char; N], Range) { const { assert!(N >= 1) }; @@ -48,7 +57,7 @@ const fn verbatim(a: ascii::Char) -> ([ascii::Char; N], Range(byte: u8) -> ([ascii::Char; N], Range) { const { assert!(N >= 4) }; @@ -122,9 +131,9 @@ const fn escape_ascii(byte: u8) -> ([ascii::Char; N], Range) } } -/// Escapes a character `\u{NNNN}` representation. +/// Escapes a character with `\u{NNNN}` representation. /// -/// Returns a buffer and the length of the escaped representation. +/// Returns a buffer with the escaped representation and its corresponding range. const fn escape_unicode(c: char) -> ([ascii::Char; N], Range) { const { assert!(N >= 10 && N < u8::MAX as usize) }; @@ -149,77 +158,214 @@ const fn escape_unicode(c: char) -> ([ascii::Char; N], Range (output, (start as u8)..(N as u8)) } -/// An iterator over an fixed-size array. -/// -/// This is essentially equivalent to array’s IntoIter except that indexes are -/// limited to u8 to reduce size of the structure. -#[derive(Clone, Debug)] -pub(crate) struct EscapeIterInner { - // The element type ensures this is always ASCII, and thus also valid UTF-8. - data: [ascii::Char; N], - - // Invariant: `alive.start <= alive.end <= N` +#[derive(Clone, Copy)] +union MaybeEscapedCharacter { + pub escape_seq: [ascii::Char; N], + pub literal: char, +} + +/// Marker type to indicate that the character is always escaped, +/// used to optimize the iterator implementation. +#[derive(Clone, Copy)] +#[non_exhaustive] +pub(crate) struct AlwaysEscaped; + +/// Marker type to indicate that the character may be escaped, +/// used to optimize the iterator implementation. +#[derive(Clone, Copy)] +#[non_exhaustive] +pub(crate) struct MaybeEscaped; + +/// An iterator over a possibly escaped character. +#[derive(Clone)] +pub(crate) struct EscapeIterInner { + // Invariant: + // + // If `alive.end <= Self::LITERAL_ESCAPE_START`, `data` must contain + // printable ASCII characters in the `alive` range of its `escape_seq` variant. + // + // If `alive.end > Self::LITERAL_ESCAPE_START`, `data` must contain a + // `char` in its `literal` variant, and the `alive` range must have a + // length of at most `1`. + data: MaybeEscapedCharacter, alive: Range, + escaping: PhantomData, } -impl EscapeIterInner { +impl EscapeIterInner { + const LITERAL_ESCAPE_START: u8 = 128; + + /// # Safety + /// + /// `data.escape_seq` must contain an escape sequence in the range given by `alive`. + #[inline] + const unsafe fn new(data: MaybeEscapedCharacter, alive: Range) -> Self { + // Longer escape sequences are not useful given `alive.end` is at most + // `Self::LITERAL_ESCAPE_START`. + const { assert!(N < Self::LITERAL_ESCAPE_START as usize) }; + + // Check bounds, which implicitly also checks the invariant + // `alive.end <= Self::LITERAL_ESCAPE_START`. + debug_assert!(alive.end <= (N + 1) as u8); + + Self { data, alive, escaping: PhantomData } + } + pub(crate) const fn backslash(c: ascii::Char) -> Self { - let (data, range) = backslash(c); - Self { data, alive: range } + let (escape_seq, alive) = backslash(c); + // SAFETY: `escape_seq` contains an escape sequence in the range given by `alive`. + unsafe { Self::new(MaybeEscapedCharacter { escape_seq }, alive) } } pub(crate) const fn ascii(c: u8) -> Self { - let (data, range) = escape_ascii(c); - Self { data, alive: range } + let (escape_seq, alive) = escape_ascii(c); + // SAFETY: `escape_seq` contains an escape sequence in the range given by `alive`. + unsafe { Self::new(MaybeEscapedCharacter { escape_seq }, alive) } } pub(crate) const fn unicode(c: char) -> Self { - let (data, range) = escape_unicode(c); - Self { data, alive: range } + let (escape_seq, alive) = escape_unicode(c); + // SAFETY: `escape_seq` contains an escape sequence in the range given by `alive`. + unsafe { Self::new(MaybeEscapedCharacter { escape_seq }, alive) } } #[inline] pub(crate) const fn empty() -> Self { - Self { data: [ascii::Char::Null; N], alive: 0..0 } + // SAFETY: `0..0` ensures an empty escape sequence. + unsafe { Self::new(MaybeEscapedCharacter { escape_seq: [ascii::Char::Null; N] }, 0..0) } } #[inline] - pub(crate) fn as_ascii(&self) -> &[ascii::Char] { - // SAFETY: `self.alive` is guaranteed to be a valid range for indexing `self.data`. - unsafe { - self.data.get_unchecked(usize::from(self.alive.start)..usize::from(self.alive.end)) - } + pub(crate) fn len(&self) -> usize { + usize::from(self.alive.end - self.alive.start) } #[inline] - pub(crate) fn as_str(&self) -> &str { - self.as_ascii().as_str() + pub(crate) fn advance_by(&mut self, n: usize) -> Result<(), NonZero> { + self.alive.advance_by(n) } #[inline] - pub(crate) fn len(&self) -> usize { - usize::from(self.alive.end - self.alive.start) + pub(crate) fn advance_back_by(&mut self, n: usize) -> Result<(), NonZero> { + self.alive.advance_back_by(n) + } + + /// Returns a `char` if `self.data` contains one in its `literal` variant. + #[inline] + const fn to_char(&self) -> Option { + if self.alive.end > Self::LITERAL_ESCAPE_START { + // SAFETY: We just checked that `self.data` contains a `char` in + // its `literal` variant. + return Some(unsafe { self.data.literal }); + } + + None } + /// Returns the printable ASCII characters in the `escape_seq` variant of `self.data` + /// as a string. + /// + /// # Safety + /// + /// - `self.data` must contain printable ASCII characters in its `escape_seq` variant. + /// - `self.alive` must be a valid range for `self.data.escape_seq`. + #[inline] + unsafe fn to_str_unchecked(&self) -> &str { + debug_assert!(self.alive.end <= Self::LITERAL_ESCAPE_START); + + // SAFETY: The caller guarantees `self.data` contains printable ASCII + // characters in its `escape_seq` variant, and `self.alive` is + // a valid range for `self.data.escape_seq`. + unsafe { + self.data + .escape_seq + .get_unchecked(usize::from(self.alive.start)..usize::from(self.alive.end)) + .as_str() + } + } +} + +impl EscapeIterInner { pub(crate) fn next(&mut self) -> Option { let i = self.alive.next()?; - // SAFETY: `i` is guaranteed to be a valid index for `self.data`. - unsafe { Some(self.data.get_unchecked(usize::from(i)).to_u8()) } + // SAFETY: The `AlwaysEscaped` marker guarantees that `self.data` + // contains printable ASCII characters in its `escape_seq` + // variant, and `i` is guaranteed to be a valid index for + // `self.data.escape_seq`. + unsafe { Some(self.data.escape_seq.get_unchecked(usize::from(i)).to_u8()) } } pub(crate) fn next_back(&mut self) -> Option { let i = self.alive.next_back()?; - // SAFETY: `i` is guaranteed to be a valid index for `self.data`. - unsafe { Some(self.data.get_unchecked(usize::from(i)).to_u8()) } + // SAFETY: The `AlwaysEscaped` marker guarantees that `self.data` + // contains printable ASCII characters in its `escape_seq` + // variant, and `i` is guaranteed to be a valid index for + // `self.data.escape_seq`. + unsafe { Some(self.data.escape_seq.get_unchecked(usize::from(i)).to_u8()) } } +} - pub(crate) fn advance_by(&mut self, n: usize) -> Result<(), NonZero> { - self.alive.advance_by(n) +impl EscapeIterInner { + // This is the only way to create any `EscapeIterInner` containing a `char` in + // the `literal` variant of its `self.data`, meaning the `AlwaysEscaped` marker + // guarantees that `self.data` contains printable ASCII characters in its + // `escape_seq` variant. + pub(crate) const fn printable(c: char) -> Self { + Self { + data: MaybeEscapedCharacter { literal: c }, + // Uphold the invariant `alive.end > Self::LITERAL_ESCAPE_START`, and ensure + // `len` behaves correctly for iterating through one character literal. + alive: Self::LITERAL_ESCAPE_START..(Self::LITERAL_ESCAPE_START + 1), + escaping: PhantomData, + } } - pub(crate) fn advance_back_by(&mut self, n: usize) -> Result<(), NonZero> { - self.alive.advance_back_by(n) + pub(crate) fn next(&mut self) -> Option { + let i = self.alive.next()?; + + if let Some(c) = self.to_char() { + return Some(c); + } + + // SAFETY: At this point, `self.data` must contain printable ASCII + // characters in its `escape_seq` variant, and `i` is + // guaranteed to be a valid index for `self.data.escape_seq`. + Some(char::from(unsafe { self.data.escape_seq.get_unchecked(usize::from(i)).to_u8() })) + } +} + +impl fmt::Display for EscapeIterInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: The `AlwaysEscaped` marker guarantees that `self.data` + // contains printable ASCII chars, and `self.alive` is + // guaranteed to be a valid range for `self.data`. + f.write_str(unsafe { self.to_str_unchecked() }) + } +} + +impl fmt::Display for EscapeIterInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(c) = self.to_char() { + return f.write_char(c); + } + + // SAFETY: At this point, `self.data` must contain printable ASCII + // characters in its `escape_seq` variant, and `self.alive` + // is guaranteed to be a valid range for `self.data`. + f.write_str(unsafe { self.to_str_unchecked() }) + } +} + +impl fmt::Debug for EscapeIterInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("EscapeIterInner").field(&format_args!("'{}'", self)).finish() + } +} + +impl fmt::Debug for EscapeIterInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("EscapeIterInner").field(&format_args!("'{}'", self)).finish() } } diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs index b4d9a1b1ca4fd..181ae82959c66 100644 --- a/library/core/src/slice/ascii.rs +++ b/library/core/src/slice/ascii.rs @@ -308,7 +308,7 @@ impl<'a> fmt::Display for EscapeAscii<'a> { if let Some(&b) = bytes.first() { // guaranteed to be non-empty, better to write it as a str - f.write_str(ascii::escape_default(b).as_str())?; + fmt::Display::fmt(&ascii::escape_default(b), f)?; bytes = &bytes[1..]; } } From 3cb0cba054d9d1871f3a10345d5c30cfc7ac214c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Sun, 1 Jun 2025 23:12:35 +0800 Subject: [PATCH 11/32] Handle win32 separator & prefixes for cygwin paths --- library/std/src/path.rs | 17 +- library/std/src/sys/path/cygwin.rs | 92 ++++ library/std/src/sys/path/mod.rs | 5 + library/std/src/sys/path/windows.rs | 175 +------- library/std/src/sys/path/windows/tests.rs | 2 + library/std/src/sys/path/windows_prefix.rs | 182 ++++++++ library/std/tests/path.rs | 472 +++++++++++++++++++++ 7 files changed, 772 insertions(+), 173 deletions(-) create mode 100644 library/std/src/sys/path/cygwin.rs create mode 100644 library/std/src/sys/path/windows_prefix.rs diff --git a/library/std/src/path.rs b/library/std/src/path.rs index 826d9f0f39dc6..be8d9f1b05416 100644 --- a/library/std/src/path.rs +++ b/library/std/src/path.rs @@ -1316,8 +1316,17 @@ impl PathBuf { need_sep = false } + let need_clear = if cfg!(target_os = "cygwin") { + // If path is absolute and its prefix is none, it is like `/foo`, + // and will be handled below. + path.prefix().is_some() + } else { + // On Unix: prefix is always None. + path.is_absolute() || path.prefix().is_some() + }; + // absolute `path` replaces `self` - if path.is_absolute() || path.prefix().is_some() { + if need_clear { self.inner.truncate(0); // verbatim paths need . and .. removed @@ -3616,6 +3625,11 @@ impl Error for NormalizeError {} /// paths, this is currently equivalent to calling /// [`GetFullPathNameW`][windows-path]. /// +/// On Cygwin, this is currently equivalent to calling [`cygwin_conv_path`][cygwin-path] +/// with mode `CCP_WIN_A_TO_POSIX`, and then being processed like other POSIX platforms. +/// If a Windows path is given, it will be converted to an absolute POSIX path without +/// keeping `..`. +/// /// Note that these [may change in the future][changes]. /// /// # Errors @@ -3673,6 +3687,7 @@ impl Error for NormalizeError {} /// [changes]: io#platform-specific-behavior /// [posix-semantics]: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 /// [windows-path]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfullpathnamew +/// [cygwin-path]: https://cygwin.com/cygwin-api/func-cygwin-conv-path.html #[stable(feature = "absolute_path", since = "1.79.0")] pub fn absolute>(path: P) -> io::Result { let path = path.as_ref(); diff --git a/library/std/src/sys/path/cygwin.rs b/library/std/src/sys/path/cygwin.rs new file mode 100644 index 0000000000000..e90372805bbf9 --- /dev/null +++ b/library/std/src/sys/path/cygwin.rs @@ -0,0 +1,92 @@ +use crate::ffi::OsString; +use crate::os::unix::ffi::OsStringExt; +use crate::path::{Path, PathBuf}; +use crate::sys::common::small_c_string::run_path_with_cstr; +use crate::sys::cvt; +use crate::{io, ptr}; + +#[inline] +pub fn is_sep_byte(b: u8) -> bool { + b == b'/' || b == b'\\' +} + +/// Cygwin allways prefers `/` over `\`, and it always converts all `/` to `\` +/// internally when calling Win32 APIs. Therefore, the server component of path +/// `\\?\UNC\localhost/share` is `localhost/share` on Win32, but `localhost` +/// on Cygwin. +#[inline] +pub fn is_verbatim_sep(b: u8) -> bool { + b == b'/' || b == b'\\' +} + +pub use super::windows_prefix::parse_prefix; + +pub const MAIN_SEP_STR: &str = "/"; +pub const MAIN_SEP: char = '/'; + +unsafe extern "C" { + // Doc: https://cygwin.com/cygwin-api/func-cygwin-conv-path.html + // Src: https://github.com/cygwin/cygwin/blob/718a15ba50e0d01c79800bd658c2477f9a603540/winsup/cygwin/path.cc#L3902 + // Safety: + // * `what` should be `CCP_WIN_A_TO_POSIX` here + // * `from` is null-terminated UTF-8 path + // * `to` is buffer, the buffer size is `size`. + // + // Converts a path to an absolute POSIX path, no matter the input is Win32 path or POSIX path. + fn cygwin_conv_path( + what: libc::c_uint, + from: *const libc::c_char, + to: *mut u8, + size: libc::size_t, + ) -> libc::ssize_t; +} + +const CCP_WIN_A_TO_POSIX: libc::c_uint = 2; + +/// Make a POSIX path absolute. +pub(crate) fn absolute(path: &Path) -> io::Result { + run_path_with_cstr(path, &|path| { + let conv = CCP_WIN_A_TO_POSIX; + let size = cvt(unsafe { cygwin_conv_path(conv, path.as_ptr(), ptr::null_mut(), 0) })?; + // If success, size should not be 0. + debug_assert!(size >= 1); + let size = size as usize; + let mut buffer = Vec::with_capacity(size); + cvt(unsafe { cygwin_conv_path(conv, path.as_ptr(), buffer.as_mut_ptr(), size) })?; + unsafe { + buffer.set_len(size - 1); + } + Ok(PathBuf::from(OsString::from_vec(buffer))) + }) + .map(|path| { + if path.prefix().is_some() { + return path; + } + + // From unix.rs + let mut components = path.components(); + let path_os = path.as_os_str().as_encoded_bytes(); + + let mut normalized = if path_os.starts_with(b"//") && !path_os.starts_with(b"///") { + components.next(); + PathBuf::from("//") + } else { + PathBuf::new() + }; + normalized.extend(components); + + if path_os.ends_with(b"/") { + normalized.push(""); + } + + normalized + }) +} + +pub(crate) fn is_absolute(path: &Path) -> bool { + if path.as_os_str().as_encoded_bytes().starts_with(b"\\") { + path.has_root() && path.prefix().is_some() + } else { + path.has_root() + } +} diff --git a/library/std/src/sys/path/mod.rs b/library/std/src/sys/path/mod.rs index 1fa4e80d6780c..a4ff4338cf5f4 100644 --- a/library/std/src/sys/path/mod.rs +++ b/library/std/src/sys/path/mod.rs @@ -1,6 +1,7 @@ cfg_if::cfg_if! { if #[cfg(target_os = "windows")] { mod windows; + mod windows_prefix; pub use windows::*; } else if #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] { mod sgx; @@ -11,6 +12,10 @@ cfg_if::cfg_if! { } else if #[cfg(target_os = "uefi")] { mod uefi; pub use uefi::*; + } else if #[cfg(target_os = "cygwin")] { + mod cygwin; + mod windows_prefix; + pub use cygwin::*; } else { mod unix; pub use unix::*; diff --git a/library/std/src/sys/path/windows.rs b/library/std/src/sys/path/windows.rs index e0e003f6a8192..f124e1e5a71c7 100644 --- a/library/std/src/sys/path/windows.rs +++ b/library/std/src/sys/path/windows.rs @@ -1,5 +1,5 @@ use crate::ffi::{OsStr, OsString}; -use crate::path::{Path, PathBuf, Prefix}; +use crate::path::{Path, PathBuf}; use crate::sys::api::utf16; use crate::sys::pal::{c, fill_utf16_buf, os2path, to_u16s}; use crate::{io, ptr}; @@ -7,6 +7,8 @@ use crate::{io, ptr}; #[cfg(test)] mod tests; +pub use super::windows_prefix::parse_prefix; + pub const MAIN_SEP_STR: &str = "\\"; pub const MAIN_SEP: char = '\\'; @@ -77,177 +79,6 @@ pub(crate) fn append_suffix(path: PathBuf, suffix: &OsStr) -> PathBuf { path.into() } -struct PrefixParser<'a, const LEN: usize> { - path: &'a OsStr, - prefix: [u8; LEN], -} - -impl<'a, const LEN: usize> PrefixParser<'a, LEN> { - #[inline] - fn get_prefix(path: &OsStr) -> [u8; LEN] { - let mut prefix = [0; LEN]; - // SAFETY: Only ASCII characters are modified. - for (i, &ch) in path.as_encoded_bytes().iter().take(LEN).enumerate() { - prefix[i] = if ch == b'/' { b'\\' } else { ch }; - } - prefix - } - - fn new(path: &'a OsStr) -> Self { - Self { path, prefix: Self::get_prefix(path) } - } - - fn as_slice(&self) -> PrefixParserSlice<'a, '_> { - PrefixParserSlice { - path: self.path, - prefix: &self.prefix[..LEN.min(self.path.len())], - index: 0, - } - } -} - -struct PrefixParserSlice<'a, 'b> { - path: &'a OsStr, - prefix: &'b [u8], - index: usize, -} - -impl<'a> PrefixParserSlice<'a, '_> { - fn strip_prefix(&self, prefix: &str) -> Option { - self.prefix[self.index..] - .starts_with(prefix.as_bytes()) - .then_some(Self { index: self.index + prefix.len(), ..*self }) - } - - fn prefix_bytes(&self) -> &'a [u8] { - &self.path.as_encoded_bytes()[..self.index] - } - - fn finish(self) -> &'a OsStr { - // SAFETY: The unsafety here stems from converting between &OsStr and - // &[u8] and back. This is safe to do because (1) we only look at ASCII - // contents of the encoding and (2) new &OsStr values are produced only - // from ASCII-bounded slices of existing &OsStr values. - unsafe { OsStr::from_encoded_bytes_unchecked(&self.path.as_encoded_bytes()[self.index..]) } - } -} - -pub fn parse_prefix(path: &OsStr) -> Option> { - use Prefix::{DeviceNS, Disk, UNC, Verbatim, VerbatimDisk, VerbatimUNC}; - - let parser = PrefixParser::<8>::new(path); - let parser = parser.as_slice(); - if let Some(parser) = parser.strip_prefix(r"\\") { - // \\ - - // The meaning of verbatim paths can change when they use a different - // separator. - if let Some(parser) = parser.strip_prefix(r"?\") - && !parser.prefix_bytes().iter().any(|&x| x == b'/') - { - // \\?\ - if let Some(parser) = parser.strip_prefix(r"UNC\") { - // \\?\UNC\server\share - - let path = parser.finish(); - let (server, path) = parse_next_component(path, true); - let (share, _) = parse_next_component(path, true); - - Some(VerbatimUNC(server, share)) - } else { - let path = parser.finish(); - - // in verbatim paths only recognize an exact drive prefix - if let Some(drive) = parse_drive_exact(path) { - // \\?\C: - Some(VerbatimDisk(drive)) - } else { - // \\?\prefix - let (prefix, _) = parse_next_component(path, true); - Some(Verbatim(prefix)) - } - } - } else if let Some(parser) = parser.strip_prefix(r".\") { - // \\.\COM42 - let path = parser.finish(); - let (prefix, _) = parse_next_component(path, false); - Some(DeviceNS(prefix)) - } else { - let path = parser.finish(); - let (server, path) = parse_next_component(path, false); - let (share, _) = parse_next_component(path, false); - - if !server.is_empty() && !share.is_empty() { - // \\server\share - Some(UNC(server, share)) - } else { - // no valid prefix beginning with "\\" recognized - None - } - } - } else { - // If it has a drive like `C:` then it's a disk. - // Otherwise there is no prefix. - parse_drive(path).map(Disk) - } -} - -// Parses a drive prefix, e.g. "C:" and "C:\whatever" -fn parse_drive(path: &OsStr) -> Option { - // In most DOS systems, it is not possible to have more than 26 drive letters. - // See . - fn is_valid_drive_letter(drive: &u8) -> bool { - drive.is_ascii_alphabetic() - } - - match path.as_encoded_bytes() { - [drive, b':', ..] if is_valid_drive_letter(drive) => Some(drive.to_ascii_uppercase()), - _ => None, - } -} - -// Parses a drive prefix exactly, e.g. "C:" -fn parse_drive_exact(path: &OsStr) -> Option { - // only parse two bytes: the drive letter and the drive separator - if path.as_encoded_bytes().get(2).map(|&x| is_sep_byte(x)).unwrap_or(true) { - parse_drive(path) - } else { - None - } -} - -// Parse the next path component. -// -// Returns the next component and the rest of the path excluding the component and separator. -// Does not recognize `/` as a separator character if `verbatim` is true. -fn parse_next_component(path: &OsStr, verbatim: bool) -> (&OsStr, &OsStr) { - let separator = if verbatim { is_verbatim_sep } else { is_sep_byte }; - - match path.as_encoded_bytes().iter().position(|&x| separator(x)) { - Some(separator_start) => { - let separator_end = separator_start + 1; - - let component = &path.as_encoded_bytes()[..separator_start]; - - // Panic safe - // The max `separator_end` is `bytes.len()` and `bytes[bytes.len()..]` is a valid index. - let path = &path.as_encoded_bytes()[separator_end..]; - - // SAFETY: `path` is a valid wtf8 encoded slice and each of the separators ('/', '\') - // is encoded in a single byte, therefore `bytes[separator_start]` and - // `bytes[separator_end]` must be code point boundaries and thus - // `bytes[..separator_start]` and `bytes[separator_end..]` are valid wtf8 slices. - unsafe { - ( - OsStr::from_encoded_bytes_unchecked(component), - OsStr::from_encoded_bytes_unchecked(path), - ) - } - } - None => (path, OsStr::new("")), - } -} - /// Returns a UTF-16 encoded path capable of bypassing the legacy `MAX_PATH` limits. /// /// This path may or may not have a verbatim prefix. diff --git a/library/std/src/sys/path/windows/tests.rs b/library/std/src/sys/path/windows/tests.rs index 9eb79203dcac7..830f48d7bfc94 100644 --- a/library/std/src/sys/path/windows/tests.rs +++ b/library/std/src/sys/path/windows/tests.rs @@ -1,4 +1,6 @@ +use super::super::windows_prefix::*; use super::*; +use crate::path::Prefix; #[test] fn test_parse_next_component() { diff --git a/library/std/src/sys/path/windows_prefix.rs b/library/std/src/sys/path/windows_prefix.rs new file mode 100644 index 0000000000000..b9dfe754485ab --- /dev/null +++ b/library/std/src/sys/path/windows_prefix.rs @@ -0,0 +1,182 @@ +//! Parse Windows prefixes, for both Windows and Cygwin. + +use super::{is_sep_byte, is_verbatim_sep}; +use crate::ffi::OsStr; +use crate::path::Prefix; + +struct PrefixParser<'a, const LEN: usize> { + path: &'a OsStr, + prefix: [u8; LEN], +} + +impl<'a, const LEN: usize> PrefixParser<'a, LEN> { + #[inline] + fn get_prefix(path: &OsStr) -> [u8; LEN] { + let mut prefix = [0; LEN]; + // SAFETY: Only ASCII characters are modified. + for (i, &ch) in path.as_encoded_bytes().iter().take(LEN).enumerate() { + prefix[i] = if ch == b'/' { b'\\' } else { ch }; + } + prefix + } + + fn new(path: &'a OsStr) -> Self { + Self { path, prefix: Self::get_prefix(path) } + } + + fn as_slice(&self) -> PrefixParserSlice<'a, '_> { + PrefixParserSlice { + path: self.path, + prefix: &self.prefix[..LEN.min(self.path.len())], + index: 0, + } + } +} + +struct PrefixParserSlice<'a, 'b> { + path: &'a OsStr, + prefix: &'b [u8], + index: usize, +} + +impl<'a> PrefixParserSlice<'a, '_> { + fn strip_prefix(&self, prefix: &str) -> Option { + self.prefix[self.index..] + .starts_with(prefix.as_bytes()) + .then_some(Self { index: self.index + prefix.len(), ..*self }) + } + + fn prefix_bytes(&self) -> &'a [u8] { + &self.path.as_encoded_bytes()[..self.index] + } + + fn finish(self) -> &'a OsStr { + // SAFETY: The unsafety here stems from converting between &OsStr and + // &[u8] and back. This is safe to do because (1) we only look at ASCII + // contents of the encoding and (2) new &OsStr values are produced only + // from ASCII-bounded slices of existing &OsStr values. + unsafe { OsStr::from_encoded_bytes_unchecked(&self.path.as_encoded_bytes()[self.index..]) } + } +} + +pub fn parse_prefix(path: &OsStr) -> Option> { + use Prefix::{DeviceNS, Disk, UNC, Verbatim, VerbatimDisk, VerbatimUNC}; + + let parser = PrefixParser::<8>::new(path); + let parser = parser.as_slice(); + if let Some(parser) = parser.strip_prefix(r"\\") { + // \\ + + // It's a POSIX path. + if cfg!(target_os = "cygwin") && !path.as_encoded_bytes().iter().any(|&x| x == b'\\') { + return None; + } + + // The meaning of verbatim paths can change when they use a different + // separator. + if let Some(parser) = parser.strip_prefix(r"?\") + // Cygwin allows `/` in verbatim paths. + && (cfg!(target_os = "cygwin") || !parser.prefix_bytes().iter().any(|&x| x == b'/')) + { + // \\?\ + if let Some(parser) = parser.strip_prefix(r"UNC\") { + // \\?\UNC\server\share + + let path = parser.finish(); + let (server, path) = parse_next_component(path, true); + let (share, _) = parse_next_component(path, true); + + Some(VerbatimUNC(server, share)) + } else { + let path = parser.finish(); + + // in verbatim paths only recognize an exact drive prefix + if let Some(drive) = parse_drive_exact(path) { + // \\?\C: + Some(VerbatimDisk(drive)) + } else { + // \\?\prefix + let (prefix, _) = parse_next_component(path, true); + Some(Verbatim(prefix)) + } + } + } else if let Some(parser) = parser.strip_prefix(r".\") { + // \\.\COM42 + let path = parser.finish(); + let (prefix, _) = parse_next_component(path, false); + Some(DeviceNS(prefix)) + } else { + let path = parser.finish(); + let (server, path) = parse_next_component(path, false); + let (share, _) = parse_next_component(path, false); + + if !server.is_empty() && !share.is_empty() { + // \\server\share + Some(UNC(server, share)) + } else { + // no valid prefix beginning with "\\" recognized + None + } + } + } else { + // If it has a drive like `C:` then it's a disk. + // Otherwise there is no prefix. + Some(Disk(parse_drive(path)?)) + } +} + +// Parses a drive prefix, e.g. "C:" and "C:\whatever" +fn parse_drive(path: &OsStr) -> Option { + // In most DOS systems, it is not possible to have more than 26 drive letters. + // See . + fn is_valid_drive_letter(drive: &u8) -> bool { + drive.is_ascii_alphabetic() + } + + match path.as_encoded_bytes() { + [drive, b':', ..] if is_valid_drive_letter(drive) => Some(drive.to_ascii_uppercase()), + _ => None, + } +} + +// Parses a drive prefix exactly, e.g. "C:" +fn parse_drive_exact(path: &OsStr) -> Option { + // only parse two bytes: the drive letter and the drive separator + if path.as_encoded_bytes().get(2).map(|&x| is_sep_byte(x)).unwrap_or(true) { + parse_drive(path) + } else { + None + } +} + +// Parse the next path component. +// +// Returns the next component and the rest of the path excluding the component and separator. +// Does not recognize `/` as a separator character on Windows if `verbatim` is true. +pub(crate) fn parse_next_component(path: &OsStr, verbatim: bool) -> (&OsStr, &OsStr) { + let separator = if verbatim { is_verbatim_sep } else { is_sep_byte }; + + match path.as_encoded_bytes().iter().position(|&x| separator(x)) { + Some(separator_start) => { + let separator_end = separator_start + 1; + + let component = &path.as_encoded_bytes()[..separator_start]; + + // Panic safe + // The max `separator_end` is `bytes.len()` and `bytes[bytes.len()..]` is a valid index. + let path = &path.as_encoded_bytes()[separator_end..]; + + // SAFETY: `path` is a valid wtf8 encoded slice and each of the separators ('/', '\') + // is encoded in a single byte, therefore `bytes[separator_start]` and + // `bytes[separator_end]` must be code point boundaries and thus + // `bytes[..separator_start]` and `bytes[separator_end..]` are valid wtf8 slices. + unsafe { + ( + OsStr::from_encoded_bytes_unchecked(component), + OsStr::from_encoded_bytes_unchecked(path), + ) + } + } + None => (path, OsStr::new("")), + } +} diff --git a/library/std/tests/path.rs b/library/std/tests/path.rs index be0dda1d426f3..901d2770f203e 100644 --- a/library/std/tests/path.rs +++ b/library/std/tests/path.rs @@ -1112,6 +1112,473 @@ pub fn test_decompositions_windows() { ); } +// Unix paths are tested in `test_decompositions_unix` above. +#[test] +#[cfg(target_os = "cygwin")] +pub fn test_decompositions_cygwin() { + t!("\\", + iter: ["/"], + has_root: true, + is_absolute: false, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("c:", + iter: ["c:"], + has_root: false, + is_absolute: false, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("c:\\", + iter: ["c:", "/"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("c:/", + iter: ["c:", "/"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("a\\b\\c", + iter: ["a", "b", "c"], + has_root: false, + is_absolute: false, + parent: Some("a\\b"), + file_name: Some("c"), + file_stem: Some("c"), + extension: None, + file_prefix: Some("c") + ); + + t!("\\a", + iter: ["/", "a"], + has_root: true, + is_absolute: false, + parent: Some("\\"), + file_name: Some("a"), + file_stem: Some("a"), + extension: None, + file_prefix: Some("a") + ); + + t!("c:\\foo.txt", + iter: ["c:", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("c:\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\server\\share\\foo.txt", + iter: ["\\\\server\\share", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\server\\share\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("//server/share\\foo.txt", + iter: ["//server/share", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("//server/share\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("//server/share/foo.txt", + iter: ["/", "server", "share", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("//server/share"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\server\\share", + iter: ["\\\\server\\share", "/"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\server", + iter: ["/", "server"], + has_root: true, + is_absolute: false, + parent: Some("\\"), + file_name: Some("server"), + file_stem: Some("server"), + extension: None, + file_prefix: Some("server") + ); + + t!("\\\\?\\bar\\foo.txt", + iter: ["\\\\?\\bar", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\bar\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\bar", + iter: ["\\\\?\\bar"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\", + iter: ["\\\\?\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\UNC\\server\\share\\foo.txt", + iter: ["\\\\?\\UNC\\server\\share", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\UNC\\server\\share\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\UNC\\server/share\\foo.txt", + iter: ["\\\\?\\UNC\\server/share", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\UNC\\server/share\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("//?/UNC/server\\share/foo.txt", + iter: ["//?/UNC/server\\share", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("//?/UNC/server\\share/"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("//?/UNC/server/share/foo.txt", + iter: ["/", "?", "UNC", "server", "share", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("//?/UNC/server/share"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\UNC\\server", + iter: ["\\\\?\\UNC\\server"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\UNC\\", + iter: ["\\\\?\\UNC\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\C:\\foo.txt", + iter: ["\\\\?\\C:", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("//?/C:\\foo.txt", + iter: ["//?/C:", "/", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("//?/C:\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("//?/C:/foo.txt", + iter: ["/", "?", "C:", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("//?/C:"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\C:\\", + iter: ["\\\\?\\C:", "/"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\C:", + iter: ["\\\\?\\C:"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\foo/bar", + iter: ["\\\\?\\foo", "/", "bar"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\foo/"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("\\\\?\\C:/foo/bar", + iter: ["\\\\?\\C:", "/", "foo", "bar"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:/foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("\\\\.\\foo\\bar", + iter: ["\\\\.\\foo", "/", "bar"], + has_root: true, + is_absolute: true, + parent: Some("\\\\.\\foo\\"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("\\\\.\\foo", + iter: ["\\\\.\\foo", "/"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\.\\foo/bar", + iter: ["\\\\.\\foo", "/", "bar"], + has_root: true, + is_absolute: true, + parent: Some("\\\\.\\foo/"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("\\\\.\\foo\\bar/baz", + iter: ["\\\\.\\foo", "/", "bar", "baz"], + has_root: true, + is_absolute: true, + parent: Some("\\\\.\\foo\\bar"), + file_name: Some("baz"), + file_stem: Some("baz"), + extension: None, + file_prefix: Some("baz") + ); + + t!("\\\\.\\", + iter: ["\\\\.\\", "/"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("//.\\foo/bar", + iter: ["//.\\foo", "/", "bar"], + has_root: true, + is_absolute: true, + parent: Some("//.\\foo/"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("\\\\./foo/bar", + iter: ["\\\\./foo", "/", "bar"], + has_root: true, + is_absolute: true, + parent: Some("\\\\./foo/"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("//./foo\\bar", + iter: ["//./foo", "/", "bar"], + has_root: true, + is_absolute: true, + parent: Some("//./foo\\"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("//./?/C:/foo/bar", + iter: ["/", "?", "C:", "foo", "bar"], + has_root: true, + is_absolute: true, + parent: Some("//./?/C:/foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("//././../././../?/C:/foo/bar", + iter: ["/", "..", "..", "?", "C:", "foo", "bar"], + has_root: true, + is_absolute: true, + parent: Some("//././../././../?/C:/foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("\\\\?\\a\\b\\", + iter: ["\\\\?\\a", "/", "b"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\a\\"), + file_name: Some("b"), + file_stem: Some("b"), + extension: None, + file_prefix: Some("b") + ); + + t!("\\\\?\\C:\\foo.txt.zip", + iter: ["\\\\?\\C:", "/", "foo.txt.zip"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:\\"), + file_name: Some("foo.txt.zip"), + file_stem: Some("foo.txt"), + extension: Some("zip"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\C:\\.foo.txt.zip", + iter: ["\\\\?\\C:", "/", ".foo.txt.zip"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:\\"), + file_name: Some(".foo.txt.zip"), + file_stem: Some(".foo.txt"), + extension: Some("zip"), + file_prefix: Some(".foo") + ); + + t!("\\\\?\\C:\\.foo", + iter: ["\\\\?\\C:", "/", ".foo"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:\\"), + file_name: Some(".foo"), + file_stem: Some(".foo"), + extension: None, + file_prefix: Some(".foo") + ); +} + #[test] pub fn test_stem_ext() { t!("foo", @@ -1227,6 +1694,11 @@ pub fn test_push() { tp!("/foo/bar", "/", "/"); tp!("/foo/bar", "/baz", "/baz"); tp!("/foo/bar", "./baz", "/foo/bar/./baz"); + + if cfg!(target_os = "cygwin") { + tp!("c:\\", "windows", "c:\\windows"); + tp!("c:", "windows", "c:windows"); + } } else { tp!("", "foo", "foo"); tp!("foo", "bar", r"foo\bar"); From 71afc6f463f141965c3a7aff70be3e4fe55c15ce Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 16 Jun 2025 14:07:25 +0000 Subject: [PATCH 12/32] Add test. --- .../write_to_borrowed.main.CopyProp.diff | 31 +++++++++++++ tests/mir-opt/copy-prop/write_to_borrowed.rs | 45 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/mir-opt/copy-prop/write_to_borrowed.main.CopyProp.diff create mode 100644 tests/mir-opt/copy-prop/write_to_borrowed.rs diff --git a/tests/mir-opt/copy-prop/write_to_borrowed.main.CopyProp.diff b/tests/mir-opt/copy-prop/write_to_borrowed.main.CopyProp.diff new file mode 100644 index 0000000000000..eb73f8c17558d --- /dev/null +++ b/tests/mir-opt/copy-prop/write_to_borrowed.main.CopyProp.diff @@ -0,0 +1,31 @@ +- // MIR for `main` before CopyProp ++ // MIR for `main` after CopyProp + + fn main() -> () { + let mut _0: (); + let mut _1: *const char; + let mut _2: char; + let mut _3: char; + let mut _4: char; + let mut _5: char; + let mut _6: &char; + let mut _7: (); + + bb0: { + _1 = &raw const _2; + _3 = const 'b'; + _5 = copy _3; + _6 = &_3; +- _4 = copy _5; ++ _3 = copy _5; + (*_1) = copy (*_6); + _6 = &_5; +- _7 = dump_var::(copy _4) -> [return: bb1, unwind unreachable]; ++ _7 = dump_var::(copy _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { + return; + } + } + diff --git a/tests/mir-opt/copy-prop/write_to_borrowed.rs b/tests/mir-opt/copy-prop/write_to_borrowed.rs new file mode 100644 index 0000000000000..dba504fe970d7 --- /dev/null +++ b/tests/mir-opt/copy-prop/write_to_borrowed.rs @@ -0,0 +1,45 @@ +//@ test-mir-pass: CopyProp + +#![feature(custom_mir, core_intrinsics)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +#[custom_mir(dialect = "runtime")] +fn main() { + mir! { + // Both _3 and _5 are borrowed, check that we do not unify them, and that we do not + // introduce a write to any of them. + let _1; + let _2; + let _3; + let _4; + let _5; + let _6; + let _7; + // CHECK: bb0: { + { + // CHECK-NEXT: _1 = &raw const _2; + _1 = core::ptr::addr_of!(_2); + // CHECK-NEXT: _3 = const 'b'; + _3 = 'b'; + // CHECK-NEXT: _5 = copy _3; + _5 = _3; + // CHECK-NEXT: _6 = &_3; + _6 = &_3; + // CHECK-NEXT: _3 = copy _5 + _4 = _5; + // CHECK-NEXT: (*_1) = copy (*_6); + *_1 = *_6; + // CHECK-NEXT: _6 = &_5; + _6 = &_5; + // CHECK-NEXT: _7 = dump_var::(copy _3) + Call(_7 = dump_var(_4), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { Return() } + } +} + +fn dump_var(_: T) {} + +// EMIT_MIR write_to_borrowed.main.CopyProp.diff From c4b67c6d873866d4b2c687306bf2f558a8659246 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 16 Jun 2025 10:05:54 +0000 Subject: [PATCH 13/32] Reason about borrowed classes in CopyProp. --- compiler/rustc_mir_transform/src/copy_prop.rs | 34 ++++--------------- compiler/rustc_mir_transform/src/ssa.rs | 23 +++++++++++++ .../write_to_borrowed.main.CopyProp.diff | 3 +- tests/mir-opt/copy-prop/write_to_borrowed.rs | 4 +-- 4 files changed, 33 insertions(+), 31 deletions(-) diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index 27af5818982d0..fe78a104fa0b6 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -30,6 +30,8 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp { let typing_env = body.typing_env(tcx); let ssa = SsaLocals::new(tcx, body, typing_env); + debug!(borrowed_locals = ?ssa.borrowed_locals()); + debug!(copy_classes = ?ssa.copy_classes()); let fully_moved = fully_moved_locals(&ssa, body); debug!(?fully_moved); @@ -43,14 +45,8 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp { let any_replacement = ssa.copy_classes().iter_enumerated().any(|(l, &h)| l != h); - Replacer { - tcx, - copy_classes: ssa.copy_classes(), - fully_moved, - borrowed_locals: ssa.borrowed_locals(), - storage_to_remove, - } - .visit_body_preserves_cfg(body); + Replacer { tcx, copy_classes: ssa.copy_classes(), fully_moved, storage_to_remove } + .visit_body_preserves_cfg(body); if any_replacement { crate::simplify::remove_unused_definitions(body); @@ -102,7 +98,6 @@ struct Replacer<'a, 'tcx> { tcx: TyCtxt<'tcx>, fully_moved: DenseBitSet, storage_to_remove: DenseBitSet, - borrowed_locals: &'a DenseBitSet, copy_classes: &'a IndexSlice, } @@ -111,34 +106,18 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { self.tcx } + #[tracing::instrument(level = "trace", skip(self))] fn visit_local(&mut self, local: &mut Local, ctxt: PlaceContext, _: Location) { let new_local = self.copy_classes[*local]; - // We must not unify two locals that are borrowed. But this is fine if one is borrowed and - // the other is not. We chose to check the original local, and not the target. That way, if - // the original local is borrowed and the target is not, we do not pessimize the whole class. - if self.borrowed_locals.contains(*local) { - return; - } match ctxt { // Do not modify the local in storage statements. PlaceContext::NonUse(NonUseContext::StorageLive | NonUseContext::StorageDead) => {} - // The local should have been marked as non-SSA. - PlaceContext::MutatingUse(_) => assert_eq!(*local, new_local), // We access the value. _ => *local = new_local, } } - fn visit_place(&mut self, place: &mut Place<'tcx>, _: PlaceContext, loc: Location) { - if let Some(new_projection) = self.process_projection(place.projection, loc) { - place.projection = self.tcx().mk_place_elems(&new_projection); - } - - // Any non-mutating use context is ok. - let ctxt = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy); - self.visit_local(&mut place.local, ctxt, loc) - } - + #[tracing::instrument(level = "trace", skip(self))] fn visit_operand(&mut self, operand: &mut Operand<'tcx>, loc: Location) { if let Operand::Move(place) = *operand // A move out of a projection of a copy is equivalent to a copy of the original @@ -151,6 +130,7 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { self.super_operand(operand, loc); } + #[tracing::instrument(level = "trace", skip(self))] fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, loc: Location) { // When removing storage statements, we need to remove both (#107511). if let StatementKind::StorageLive(l) | StatementKind::StorageDead(l) = stmt.kind diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index edd0cabca49a4..03b6f9b7ff3b8 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -293,6 +293,10 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor<'_, 'tcx> { fn compute_copy_classes(ssa: &mut SsaLocals, body: &Body<'_>) { let mut direct_uses = std::mem::take(&mut ssa.direct_uses); let mut copies = IndexVec::from_fn_n(|l| l, body.local_decls.len()); + // We must not unify two locals that are borrowed. But this is fine if one is borrowed and + // the other is not. This bitset is keyed by *class head* and contains whether any member of + // the class is borrowed. + let mut borrowed_classes = ssa.borrowed_locals().clone(); for (local, rvalue, _) in ssa.assignments(body) { let (Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) @@ -318,6 +322,11 @@ fn compute_copy_classes(ssa: &mut SsaLocals, body: &Body<'_>) { // visited before `local`, and we just have to copy the representing local. let head = copies[rhs]; + // Do not unify two borrowed locals. + if borrowed_classes.contains(local) && borrowed_classes.contains(head) { + continue; + } + if local == RETURN_PLACE { // `_0` is special, we cannot rename it. Instead, rename the class of `rhs` to // `RETURN_PLACE`. This is only possible if the class head is a temporary, not an @@ -330,14 +339,21 @@ fn compute_copy_classes(ssa: &mut SsaLocals, body: &Body<'_>) { *h = RETURN_PLACE; } } + if borrowed_classes.contains(head) { + borrowed_classes.insert(RETURN_PLACE); + } } else { copies[local] = head; + if borrowed_classes.contains(local) { + borrowed_classes.insert(head); + } } direct_uses[rhs] -= 1; } debug!(?copies); debug!(?direct_uses); + debug!(?borrowed_classes); // Invariant: `copies` must point to the head of an equivalence class. #[cfg(debug_assertions)] @@ -346,6 +362,13 @@ fn compute_copy_classes(ssa: &mut SsaLocals, body: &Body<'_>) { } debug_assert_eq!(copies[RETURN_PLACE], RETURN_PLACE); + // Invariant: `borrowed_classes` must be true if any member of the class is borrowed. + #[cfg(debug_assertions)] + for &head in copies.iter() { + let any_borrowed = ssa.borrowed_locals.iter().any(|l| copies[l] == head); + assert_eq!(borrowed_classes.contains(head), any_borrowed); + } + ssa.direct_uses = direct_uses; ssa.copy_classes = copies; } diff --git a/tests/mir-opt/copy-prop/write_to_borrowed.main.CopyProp.diff b/tests/mir-opt/copy-prop/write_to_borrowed.main.CopyProp.diff index eb73f8c17558d..eab06b1ba1e79 100644 --- a/tests/mir-opt/copy-prop/write_to_borrowed.main.CopyProp.diff +++ b/tests/mir-opt/copy-prop/write_to_borrowed.main.CopyProp.diff @@ -17,11 +17,10 @@ _5 = copy _3; _6 = &_3; - _4 = copy _5; -+ _3 = copy _5; (*_1) = copy (*_6); _6 = &_5; - _7 = dump_var::(copy _4) -> [return: bb1, unwind unreachable]; -+ _7 = dump_var::(copy _3) -> [return: bb1, unwind unreachable]; ++ _7 = dump_var::(copy _5) -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/copy-prop/write_to_borrowed.rs b/tests/mir-opt/copy-prop/write_to_borrowed.rs index dba504fe970d7..58809749103e1 100644 --- a/tests/mir-opt/copy-prop/write_to_borrowed.rs +++ b/tests/mir-opt/copy-prop/write_to_borrowed.rs @@ -27,13 +27,13 @@ fn main() { _5 = _3; // CHECK-NEXT: _6 = &_3; _6 = &_3; - // CHECK-NEXT: _3 = copy _5 + // CHECK-NOT: {{_.*}} = {{_.*}}; _4 = _5; // CHECK-NEXT: (*_1) = copy (*_6); *_1 = *_6; // CHECK-NEXT: _6 = &_5; _6 = &_5; - // CHECK-NEXT: _7 = dump_var::(copy _3) + // CHECK-NEXT: _7 = dump_var::(copy _5) Call(_7 = dump_var(_4), ReturnTo(bb1), UnwindUnreachable()) } bb1 = { Return() } From 6da3bf853e2a3a0417de7faa878e4d7907526a96 Mon Sep 17 00:00:00 2001 From: "Celina G. Val" Date: Mon, 16 Jun 2025 12:25:25 -0700 Subject: [PATCH 14/32] Apply suggestions from code review Co-authored-by: Oli Scherer --- compiler/rustc_thread_pool/LICENSE-APACHE | 201 ---------------------- compiler/rustc_thread_pool/LICENSE-MIT | 25 --- compiler/rustc_thread_pool/README.md | 5 +- 3 files changed, 1 insertion(+), 230 deletions(-) delete mode 100644 compiler/rustc_thread_pool/LICENSE-APACHE delete mode 100644 compiler/rustc_thread_pool/LICENSE-MIT diff --git a/compiler/rustc_thread_pool/LICENSE-APACHE b/compiler/rustc_thread_pool/LICENSE-APACHE deleted file mode 100644 index 16fe87b06e802..0000000000000 --- a/compiler/rustc_thread_pool/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/compiler/rustc_thread_pool/LICENSE-MIT b/compiler/rustc_thread_pool/LICENSE-MIT deleted file mode 100644 index 25597d5838fa4..0000000000000 --- a/compiler/rustc_thread_pool/LICENSE-MIT +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2010 The Rust Project Developers - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/compiler/rustc_thread_pool/README.md b/compiler/rustc_thread_pool/README.md index 5b8714f5df48e..a50cc1165b810 100644 --- a/compiler/rustc_thread_pool/README.md +++ b/compiler/rustc_thread_pool/README.md @@ -2,12 +2,9 @@ Note: This is an unstable fork made for use in rustc Rayon-core represents the "core, stable" APIs of Rayon: join, scope, and so forth, as well as the ability to create custom thread-pools with ThreadPool. -Maybe worth mentioning: users are not necessarily intended to directly access rayon-core; all its APIs are mirrored in the rayon crate. To that end, the examples in the docs use rayon::join and so forth rather than rayon_core::join. +Maybe worth mentioning: users are not necessarily intended to directly access rustc_thread_pool; all its APIs are mirrored in the rayon crate. To that end, the examples in the docs use rayon::join and so forth rather than rayon_core::join. -rayon-core aims to never, or almost never, have a breaking change to its API, because each revision of rayon-core also houses the global thread-pool (and hence if you have two simultaneous versions of rayon-core, you have two thread-pools). Please see [Rayon Docs] for details about using Rayon. [Rayon Docs]: https://docs.rs/rayon/ - -Rayon-core currently requires `rustc 1.63.0` or greater. From 5a449fb40bd481652b98dd5299e1dfe91cf71ef2 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Mon, 16 Jun 2025 23:21:23 -0700 Subject: [PATCH 15/32] tests: remove define so dso_local attr does not disrupt test --- tests/codegen/frame-pointer-cli-control.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codegen/frame-pointer-cli-control.rs b/tests/codegen/frame-pointer-cli-control.rs index d1f2e2b685f72..a65dd132763de 100644 --- a/tests/codegen/frame-pointer-cli-control.rs +++ b/tests/codegen/frame-pointer-cli-control.rs @@ -45,7 +45,7 @@ Specific cases where platforms or tools rely on frame pointers for sound or corr extern crate minicore; -// CHECK: define i32 @peach{{.*}}[[PEACH_ATTRS:\#[0-9]+]] { +// CHECK: i32 @peach{{.*}}[[PEACH_ATTRS:\#[0-9]+]] { #[no_mangle] pub fn peach(x: u32) -> u32 { x From 2c2bc10f1bbc2ffbac148c0bc18b92d4ec88f169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Miku=C5=82a?= Date: Tue, 6 May 2025 18:42:53 +0200 Subject: [PATCH 16/32] {aarch64,x86_64}-pc-windows-gnullvm: build host tools --- .../dist-aarch64-windows-gnullvm/Dockerfile | 48 ++++++++++++++++++ .../host-x86_64/dist-various-1/Dockerfile | 8 +-- .../dist-various-1/install-llvm-mingw.sh | 8 --- .../dist-x86_64-windows-gnullvm/Dockerfile | 50 +++++++++++++++++++ .../install-llvm-mingw.sh | 13 +++++ src/ci/github-actions/jobs.yml | 6 +++ src/doc/rustc/src/platform-support.md | 4 +- .../src/platform-support/windows-gnullvm.md | 2 +- 8 files changed, 121 insertions(+), 18 deletions(-) create mode 100644 src/ci/docker/host-x86_64/dist-aarch64-windows-gnullvm/Dockerfile delete mode 100755 src/ci/docker/host-x86_64/dist-various-1/install-llvm-mingw.sh create mode 100644 src/ci/docker/host-x86_64/dist-x86_64-windows-gnullvm/Dockerfile create mode 100755 src/ci/docker/host-x86_64/dist-x86_64-windows-gnullvm/install-llvm-mingw.sh diff --git a/src/ci/docker/host-x86_64/dist-aarch64-windows-gnullvm/Dockerfile b/src/ci/docker/host-x86_64/dist-aarch64-windows-gnullvm/Dockerfile new file mode 100644 index 0000000000000..cdbc1cda02558 --- /dev/null +++ b/src/ci/docker/host-x86_64/dist-aarch64-windows-gnullvm/Dockerfile @@ -0,0 +1,48 @@ +FROM ubuntu:24.04 + +WORKDIR /build + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + cmake \ + curl \ + g++ \ + git \ + make \ + ninja-build \ + python3 \ + xz-utils + +ENV ARCH=aarch64 +COPY host-x86_64/dist-x86_64-windows-gnullvm/install-llvm-mingw.sh /build +RUN ./install-llvm-mingw.sh + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV CC_aarch64_pc_windows_gnullvm=aarch64-w64-mingw32-clang \ + CXX_aarch64_pc_windows_gnullvm=aarch64-w64-mingw32-clang++ + +ENV HOST=aarch64-pc-windows-gnullvm + +# We are bootstrapping this target and cannot use previously built artifacts. +# Without this option Clang is given `"-I/checkout/obj/build/aarch64-pc-windows-gnullvm/ci-llvm/include"` +# despite no such directory existing: +# $ ls obj/dist-windows-gnullvm/build/aarch64-pc-windows-gnullvm/ -1 +# llvm +# stage2 +ENV NO_DOWNLOAD_CI_LLVM 1 + +ENV RUST_CONFIGURE_ARGS \ + --enable-extended \ + --enable-profiler \ + --enable-sanitizers \ + --disable-docs \ + --set llvm.download-ci-llvm=false \ + --set rust.llvm-tools=false +# LLVM cross tools are not installed into expected location so copying fails. +# Probably will solve itself once this target can host itself on Windows. +# --enable-full-tools \ + +ENV SCRIPT python3 ../x.py dist --host $HOST --target $HOST diff --git a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile index 00552db4b0144..af48e9bcfde9d 100644 --- a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile @@ -114,9 +114,6 @@ ENV TARGETS=$TARGETS,armv7r-none-eabi ENV TARGETS=$TARGETS,armv7r-none-eabihf ENV TARGETS=$TARGETS,thumbv7neon-unknown-linux-gnueabihf ENV TARGETS=$TARGETS,armv7a-none-eabi -ENV TARGETS=$TARGETS,aarch64-pc-windows-gnullvm -ENV TARGETS=$TARGETS,i686-pc-windows-gnullvm -ENV TARGETS=$TARGETS,x86_64-pc-windows-gnullvm ENV CFLAGS_armv5te_unknown_linux_musleabi="-march=armv5te -marm -mfloat-abi=soft" \ CFLAGS_arm_unknown_linux_musleabi="-march=armv6 -marm" \ @@ -148,10 +145,7 @@ ENV CFLAGS_armv5te_unknown_linux_musleabi="-march=armv5te -marm -mfloat-abi=soft CC_riscv64imac_unknown_none_elf=riscv64-unknown-elf-gcc \ CFLAGS_riscv64imac_unknown_none_elf=-march=rv64imac -mabi=lp64 \ CC_riscv64gc_unknown_none_elf=riscv64-unknown-elf-gcc \ - CFLAGS_riscv64gc_unknown_none_elf=-march=rv64gc -mabi=lp64 \ - CC_aarch64_pc_windows_gnullvm=aarch64-w64-mingw32-clang \ - CC_i686_pc_windows_gnullvm=i686-w64-mingw32-clang \ - CC_x86_64_pc_windows_gnullvm=x86_64-w64-mingw32-clang + CFLAGS_riscv64gc_unknown_none_elf=-march=rv64gc -mabi=lp64 ENV RUST_CONFIGURE_ARGS \ --musl-root-armv5te=/musl-armv5te \ diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-llvm-mingw.sh b/src/ci/docker/host-x86_64/dist-various-1/install-llvm-mingw.sh deleted file mode 100755 index 95471895fe771..0000000000000 --- a/src/ci/docker/host-x86_64/dist-various-1/install-llvm-mingw.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -release_date=20240404 -archive=llvm-mingw-${release_date}-ucrt-ubuntu-20.04-x86_64.tar.xz -curl -L https://github.com/mstorsjo/llvm-mingw/releases/download/${release_date}/${archive} | \ -tar --extract --lzma --strip 1 --directory /usr/local diff --git a/src/ci/docker/host-x86_64/dist-x86_64-windows-gnullvm/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-windows-gnullvm/Dockerfile new file mode 100644 index 0000000000000..1ee3951beb563 --- /dev/null +++ b/src/ci/docker/host-x86_64/dist-x86_64-windows-gnullvm/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +WORKDIR /build + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + cmake \ + curl \ + g++ \ + git \ + make \ + ninja-build \ + python3 \ + xz-utils + +ENV ARCH='i686 x86_64' +COPY host-x86_64/dist-x86_64-windows-gnullvm/install-llvm-mingw.sh /build +RUN ./install-llvm-mingw.sh + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV CC_i686_pc_windows_gnullvm=i686-w64-mingw32-clang \ + CC_x86_64_pc_windows_gnullvm=x86_64-w64-mingw32-clang \ + CXX_x86_64_pc_windows_gnullvm=x86_64-w64-mingw32-clang++ + +ENV HOST=x86_64-pc-windows-gnullvm +ENV TARGETS=i686-pc-windows-gnullvm,x86_64-pc-windows-gnullvm + +# We are bootstrapping this target and cannot use previously built artifacts. +# Without this option Clang is given `"-I/checkout/obj/build/aarch64-pc-windows-gnullvm/ci-llvm/include"` +# despite no such directory existing: +# $ ls obj/dist-windows-gnullvm/build/aarch64-pc-windows-gnullvm/ -1 +# llvm +# stage2 +ENV NO_DOWNLOAD_CI_LLVM 1 + +ENV RUST_CONFIGURE_ARGS \ + --enable-extended \ + --enable-profiler \ + --enable-sanitizers \ + --disable-docs \ + --set llvm.download-ci-llvm=false \ + --set rust.llvm-tools=false +# LLVM cross tools are not installed into expected location so copying fails. +# Probably will solve itself once these targets can host themselves on Windows. +# --enable-full-tools \ + +ENV SCRIPT python3 ../x.py dist --host $HOST --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-x86_64-windows-gnullvm/install-llvm-mingw.sh b/src/ci/docker/host-x86_64/dist-x86_64-windows-gnullvm/install-llvm-mingw.sh new file mode 100755 index 0000000000000..0ea5dae3ffbd9 --- /dev/null +++ b/src/ci/docker/host-x86_64/dist-x86_64-windows-gnullvm/install-llvm-mingw.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex + +release_date=20250613 +archive=llvm-mingw-${release_date}-ucrt-ubuntu-22.04-x86_64.tar.xz +curl -L https://github.com/mstorsjo/llvm-mingw/releases/download/${release_date}/${archive} | \ +tar --extract --xz --strip 1 --directory /usr/local + +# https://github.com/mstorsjo/llvm-mingw/issues/493 +for arch in $ARCH; do + ln -s $arch-w64-windows-gnu.cfg /usr/local/bin/$arch-pc-windows-gnu.cfg +done diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 217cf764afbb1..e996a8f7139d3 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -237,6 +237,12 @@ auto: - name: dist-s390x-linux <<: *job-linux-4c + - name: dist-aarch64-windows-gnullvm + <<: *job-linux-4c + + - name: dist-x86_64-windows-gnullvm + <<: *job-linux-4c + - name: dist-various-1 <<: *job-linux-4c diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md index 3cab57df75a82..285b1e519b7cc 100644 --- a/src/doc/rustc/src/platform-support.md +++ b/src/doc/rustc/src/platform-support.md @@ -88,6 +88,7 @@ so Rustup may install the documentation for a similar tier 1 target instead. target | notes -------|------- +[`aarch64-pc-windows-gnullvm`](platform-support/windows-gnullvm.md) | ARM64 MinGW (Windows 10+), LLVM ABI [`aarch64-pc-windows-msvc`](platform-support/windows-msvc.md) | ARM64 Windows MSVC `aarch64-unknown-linux-musl` | ARM64 Linux with musl 1.2.3 [`aarch64-unknown-linux-ohos`](platform-support/openharmony.md) | ARM64 OpenHarmony @@ -105,6 +106,7 @@ target | notes [`riscv64gc-unknown-linux-gnu`](platform-support/riscv64gc-unknown-linux-gnu.md) | RISC-V Linux (kernel 4.20+, glibc 2.29) [`riscv64gc-unknown-linux-musl`](platform-support/riscv64gc-unknown-linux-musl.md) | RISC-V Linux (kernel 4.20+, musl 1.2.3) [`s390x-unknown-linux-gnu`](platform-support/s390x-unknown-linux-gnu.md) | S390x Linux (kernel 3.2+, glibc 2.17) +[`x86_64-pc-windows-gnullvm`](platform-support/windows-gnullvm.md) | 64-bit x86 MinGW (Windows 10+), LLVM ABI [`x86_64-unknown-freebsd`](platform-support/freebsd.md) | 64-bit x86 FreeBSD [`x86_64-unknown-illumos`](platform-support/illumos.md) | illumos `x86_64-unknown-linux-musl` | 64-bit Linux with musl 1.2.3 @@ -147,7 +149,6 @@ target | std | notes [`aarch64-apple-ios-macabi`](platform-support/apple-ios-macabi.md) | ✓ | Mac Catalyst on ARM64 [`aarch64-apple-ios-sim`](platform-support/apple-ios.md) | ✓ | Apple iOS Simulator on ARM64 [`aarch64-linux-android`](platform-support/android.md) | ✓ | ARM64 Android -[`aarch64-pc-windows-gnullvm`](platform-support/windows-gnullvm.md) | ✓ | ARM64 MinGW (Windows 10+), LLVM ABI [`aarch64-unknown-fuchsia`](platform-support/fuchsia.md) | ✓ | ARM64 Fuchsia `aarch64-unknown-none` | * | Bare ARM64, hardfloat `aarch64-unknown-none-softfloat` | * | Bare ARM64, softfloat @@ -204,7 +205,6 @@ target | std | notes [`x86_64-apple-ios-macabi`](platform-support/apple-ios-macabi.md) | ✓ | Mac Catalyst on x86_64 [`x86_64-fortanix-unknown-sgx`](platform-support/x86_64-fortanix-unknown-sgx.md) | ✓ | [Fortanix ABI] for 64-bit Intel SGX [`x86_64-linux-android`](platform-support/android.md) | ✓ | 64-bit x86 Android -[`x86_64-pc-windows-gnullvm`](platform-support/windows-gnullvm.md) | ✓ | 64-bit x86 MinGW (Windows 10+), LLVM ABI [`x86_64-unknown-fuchsia`](platform-support/fuchsia.md) | ✓ | 64-bit x86 Fuchsia `x86_64-unknown-linux-gnux32` | ✓ | 64-bit Linux (x32 ABI) (kernel 4.15+, glibc 2.27) [`x86_64-unknown-none`](platform-support/x86_64-unknown-none.md) | * | Freestanding/bare-metal x86_64, softfloat diff --git a/src/doc/rustc/src/platform-support/windows-gnullvm.md b/src/doc/rustc/src/platform-support/windows-gnullvm.md index f3dc73165ac7e..b469af0153130 100644 --- a/src/doc/rustc/src/platform-support/windows-gnullvm.md +++ b/src/doc/rustc/src/platform-support/windows-gnullvm.md @@ -1,6 +1,6 @@ # \*-windows-gnullvm -**Tier: 2 (without host tools)** +**Tier: 2 (with host tools)** Windows targets similar to `*-windows-gnu` but using UCRT as the runtime and various LLVM tools/libraries instead of GCC/Binutils. From 7889332342974a590eb2f755a3035aae6c1ccafc Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Mon, 16 Jun 2025 23:47:24 +0530 Subject: [PATCH 17/32] add deferred command in execution context and update run method --- src/bootstrap/src/utils/execution_context.rs | 137 ++++++++++++------- 1 file changed, 91 insertions(+), 46 deletions(-) diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index a5e1e9bcc07df..778ef1b0fc1b3 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -3,6 +3,7 @@ //! This module provides the [`ExecutionContext`] type, which holds global configuration //! relevant during the execution of commands in bootstrap. This includes dry-run //! mode, verbosity level, and behavior on failure. +use std::process::Child; use std::sync::{Arc, Mutex}; use crate::core::config::DryRun; @@ -80,15 +81,16 @@ impl ExecutionContext { /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to /// execute commands. They internally call this method. #[track_caller] - pub fn run( + pub fn start<'a>( &self, - command: &mut BootstrapCommand, + command: &'a mut BootstrapCommand, stdout: OutputMode, stderr: OutputMode, - ) -> CommandOutput { + ) -> DeferredCommand<'a> { command.mark_as_executed(); + if self.dry_run() && !command.run_always { - return CommandOutput::default(); + return DeferredCommand::new(None, stdout, stderr, command, Arc::new(self.clone())); } #[cfg(feature = "tracing")] @@ -105,7 +107,75 @@ impl ExecutionContext { cmd.stdout(stdout.stdio()); cmd.stderr(stderr.stdio()); - let output = cmd.output(); + let child = cmd.spawn().unwrap(); + + DeferredCommand::new(Some(child), stdout, stderr, command, Arc::new(self.clone())) + } + + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn run( + &self, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> CommandOutput { + self.start(command, stdout, stderr).wait_for_output() + } + + fn fail(&self, message: &str, output: CommandOutput) -> ! { + if self.is_verbose() { + println!("{message}"); + } else { + let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); + // If the command captures output, the user would not see any indication that + // it has failed. In this case, print a more verbose error, since to provide more + // context. + if stdout.is_some() || stderr.is_some() { + if let Some(stdout) = output.stdout_if_present().take_if(|s| !s.trim().is_empty()) { + println!("STDOUT:\n{stdout}\n"); + } + if let Some(stderr) = output.stderr_if_present().take_if(|s| !s.trim().is_empty()) { + println!("STDERR:\n{stderr}\n"); + } + println!("Command has failed. Rerun with -v to see more details."); + } else { + println!("Command has failed. Rerun with -v to see more details."); + } + } + exit!(1); + } +} + +pub struct DeferredCommand<'a> { + process: Option, + command: &'a mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + exec_ctx: Arc, +} + +impl<'a> DeferredCommand<'a> { + pub fn new( + child: Option, + stdout: OutputMode, + stderr: OutputMode, + command: &'a mut BootstrapCommand, + exec_ctx: Arc, + ) -> Self { + DeferredCommand { process: child, stdout, stderr, command, exec_ctx } + } + + pub fn wait_for_output(mut self) -> CommandOutput { + if self.process.is_none() { + return CommandOutput::default(); + } + let output = self.process.take().unwrap().wait_with_output(); + + let created_at = self.command.get_created_location(); + let executed_at = std::panic::Location::caller(); use std::fmt::Write; @@ -113,30 +183,31 @@ impl ExecutionContext { let output: CommandOutput = match output { // Command has succeeded Ok(output) if output.status.success() => { - CommandOutput::from_output(output, stdout, stderr) + CommandOutput::from_output(output, self.stdout, self.stderr) } // Command has started, but then it failed Ok(output) => { writeln!( message, r#" -Command {command:?} did not execute successfully. +Command {:?} did not execute successfully. Expected success, got {} Created at: {created_at} Executed at: {executed_at}"#, - output.status, + self.command, output.status, ) .unwrap(); - let output: CommandOutput = CommandOutput::from_output(output, stdout, stderr); + let output: CommandOutput = + CommandOutput::from_output(output, self.stdout, self.stderr); // If the output mode is OutputMode::Capture, we can now print the output. // If it is OutputMode::Print, then the output has already been printed to // stdout/stderr, and we thus don't have anything captured to print anyway. - if stdout.captures() { + if self.stdout.captures() { writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); } - if stderr.captures() { + if self.stderr.captures() { writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); } output @@ -145,52 +216,26 @@ Executed at: {executed_at}"#, Err(e) => { writeln!( message, - "\n\nCommand {command:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}" + "\n\nCommand {:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}", + self.command ) .unwrap(); - CommandOutput::did_not_start(stdout, stderr) - } - }; - - let fail = |message: &str, output: CommandOutput| -> ! { - if self.is_verbose() { - println!("{message}"); - } else { - let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); - // If the command captures output, the user would not see any indication that - // it has failed. In this case, print a more verbose error, since to provide more - // context. - if stdout.is_some() || stderr.is_some() { - if let Some(stdout) = - output.stdout_if_present().take_if(|s| !s.trim().is_empty()) - { - println!("STDOUT:\n{stdout}\n"); - } - if let Some(stderr) = - output.stderr_if_present().take_if(|s| !s.trim().is_empty()) - { - println!("STDERR:\n{stderr}\n"); - } - println!("Command {command:?} has failed. Rerun with -v to see more details."); - } else { - println!("Command has failed. Rerun with -v to see more details."); - } + CommandOutput::did_not_start(self.stdout, self.stderr) } - exit!(1); }; if !output.is_success() { - match command.failure_behavior { + match self.command.failure_behavior { BehaviorOnFailure::DelayFail => { - if self.fail_fast { - fail(&message, output); + if self.exec_ctx.fail_fast { + self.exec_ctx.fail(&message, output); } - self.add_to_delay_failure(message); + self.exec_ctx.add_to_delay_failure(message); } BehaviorOnFailure::Exit => { - fail(&message, output); + self.exec_ctx.fail(&message, output); } BehaviorOnFailure::Ignore => { // If failures are allowed, either the error has been printed already From 16bc870ee2c05ed56a003c3f0444a067afc54c18 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Mon, 16 Jun 2025 23:47:40 +0530 Subject: [PATCH 18/32] add start methods in exec --- src/bootstrap/src/utils/exec.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index f297300e34a85..85a19ebe36880 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -2,16 +2,16 @@ //! //! This module provides a structured way to execute and manage commands efficiently, //! ensuring controlled failure handling and output management. - +#![allow(warnings)] use std::ffi::OsStr; use std::fmt::{Debug, Formatter}; use std::path::Path; -use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::process::{Child, Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; use build_helper::ci::CiEnv; use build_helper::drop_bomb::DropBomb; -use super::execution_context::ExecutionContext; +use super::execution_context::{DeferredCommand, ExecutionContext}; /// What should be done when the command fails. #[derive(Debug, Copy, Clone)] @@ -158,6 +158,21 @@ impl BootstrapCommand { exec_ctx.as_ref().run(self, OutputMode::Capture, OutputMode::Print) } + /// Spawn the command in background, while capturing and returning all its output. + #[track_caller] + pub fn start_capture(&mut self, exec_ctx: impl AsRef) -> DeferredCommand { + exec_ctx.as_ref().start(self, OutputMode::Capture, OutputMode::Capture) + } + + /// Spawn the command in background, while capturing and returning stdout, and printing stderr. + #[track_caller] + pub fn start_capture_stdout( + &mut self, + exec_ctx: impl AsRef, + ) -> DeferredCommand { + exec_ctx.as_ref().start(self, OutputMode::Capture, OutputMode::Print) + } + /// Provides access to the stdlib Command inside. /// FIXME: This function should be eventually removed from bootstrap. pub fn as_command_mut(&mut self) -> &mut Command { From 2e4f2d2f3bb566a36339bf28b2167b6c68823b5b Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Mon, 16 Jun 2025 23:49:48 +0530 Subject: [PATCH 19/32] move from start process to new start and wait for output api's --- src/bootstrap/src/utils/channel.rs | 41 +++++++++++++++++------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index 38f250af42f08..a7ccdbf6eab07 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -11,7 +11,7 @@ use std::path::Path; use super::execution_context::ExecutionContext; use super::helpers; use crate::Build; -use crate::utils::helpers::{start_process, t}; +use crate::utils::helpers::t; #[derive(Clone, Default)] pub enum GitInfo { @@ -46,7 +46,7 @@ impl GitInfo { let mut git_command = helpers::git(Some(dir)); git_command.arg("rev-parse"); - let output = git_command.allow_failure().run_capture(exec_ctx); + let output = git_command.allow_failure().run_capture(&exec_ctx); if output.is_failure() { return GitInfo::Absent; @@ -59,23 +59,28 @@ impl GitInfo { } // Ok, let's scrape some info - let ver_date = start_process( - helpers::git(Some(dir)) - .arg("log") - .arg("-1") - .arg("--date=short") - .arg("--pretty=format:%cd") - .as_command_mut(), - ); - let ver_hash = - start_process(helpers::git(Some(dir)).arg("rev-parse").arg("HEAD").as_command_mut()); - let short_ver_hash = start_process( - helpers::git(Some(dir)).arg("rev-parse").arg("--short=9").arg("HEAD").as_command_mut(), - ); + let mut git_log_cmd = helpers::git(Some(dir)); + let ver_date = git_log_cmd + .arg("log") + .arg("-1") + .arg("--date=short") + .arg("--pretty=format:%cd") + .start_capture_stdout(&exec_ctx); + + let mut git_hash_cmd = helpers::git(Some(dir)); + let ver_hash = git_hash_cmd.arg("rev-parse").arg("HEAD").start_capture_stdout(&exec_ctx); + + let mut git_short_hash_cmd = helpers::git(Some(dir)); + let short_ver_hash = git_short_hash_cmd + .arg("rev-parse") + .arg("--short=9") + .arg("HEAD") + .start_capture_stdout(&exec_ctx); + GitInfo::Present(Some(Info { - commit_date: ver_date().trim().to_string(), - sha: ver_hash().trim().to_string(), - short_sha: short_ver_hash().trim().to_string(), + commit_date: ver_date.wait_for_output().stdout().trim().to_string(), + sha: ver_hash.wait_for_output().stdout().trim().to_string(), + short_sha: short_ver_hash.wait_for_output().stdout().trim().to_string(), })) } From b16ba4d3ee8c783cf33fb31a8c8a40d81a5316c2 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Mon, 16 Jun 2025 23:50:10 +0530 Subject: [PATCH 20/32] remove start_porcess helper method --- src/bootstrap/src/utils/helpers.rs | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index f4be22f1e649e..2f18fb6031829 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -5,13 +5,11 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use std::sync::OnceLock; use std::thread::panicking; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use std::{env, fs, io, panic, str}; -use build_helper::util::fail; use object::read::archive::ArchiveFile; use crate::LldMode; @@ -282,33 +280,6 @@ pub fn make(host: &str) -> PathBuf { } } -/// Spawn a process and return a closure that will wait for the process -/// to finish and then return its output. This allows the spawned process -/// to do work without immediately blocking bootstrap. -#[track_caller] -pub fn start_process(cmd: &mut Command) -> impl FnOnce() -> String + use<> { - let child = match cmd.stderr(Stdio::inherit()).stdout(Stdio::piped()).spawn() { - Ok(child) => child, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nERROR: {e}")), - }; - - let command = format!("{cmd:?}"); - - move || { - let output = child.wait_with_output().unwrap(); - - if !output.status.success() { - panic!( - "command did not execute successfully: {}\n\ - expected success, got: {}", - command, output.status - ); - } - - String::from_utf8(output.stdout).unwrap() - } -} - /// Returns the last-modified time for `path`, or zero if it doesn't exist. pub fn mtime(path: &Path) -> SystemTime { fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) From 0c60856ed19acd0786bac3543dab42e7253bda31 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 17 Jun 2025 18:13:34 +0530 Subject: [PATCH 21/32] remove new method from command execution --- src/bootstrap/src/utils/execution_context.rs | 26 +++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index 778ef1b0fc1b3..c0544afc5ec3e 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -90,7 +90,13 @@ impl ExecutionContext { command.mark_as_executed(); if self.dry_run() && !command.run_always { - return DeferredCommand::new(None, stdout, stderr, command, Arc::new(self.clone())); + return DeferredCommand { + process: None, + stdout, + stderr, + command, + exec_ctx: Arc::new(self.clone()), + }; } #[cfg(feature = "tracing")] @@ -109,7 +115,13 @@ impl ExecutionContext { let child = cmd.spawn().unwrap(); - DeferredCommand::new(Some(child), stdout, stderr, command, Arc::new(self.clone())) + DeferredCommand { + process: Some(child), + stdout, + stderr, + command, + exec_ctx: Arc::new(self.clone()), + } } /// Execute a command and return its output. @@ -158,16 +170,6 @@ pub struct DeferredCommand<'a> { } impl<'a> DeferredCommand<'a> { - pub fn new( - child: Option, - stdout: OutputMode, - stderr: OutputMode, - command: &'a mut BootstrapCommand, - exec_ctx: Arc, - ) -> Self { - DeferredCommand { process: child, stdout, stderr, command, exec_ctx } - } - pub fn wait_for_output(mut self) -> CommandOutput { if self.process.is_none() { return CommandOutput::default(); From 2270572cb4467f3939ea86fb648794164a8c9054 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 17 Jun 2025 18:17:16 +0530 Subject: [PATCH 22/32] add created at to command execution --- src/bootstrap/src/utils/execution_context.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index c0544afc5ec3e..15b92f93b0426 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -3,6 +3,7 @@ //! This module provides the [`ExecutionContext`] type, which holds global configuration //! relevant during the execution of commands in bootstrap. This includes dry-run //! mode, verbosity level, and behavior on failure. +use std::panic::Location; use std::process::Child; use std::sync::{Arc, Mutex}; @@ -89,6 +90,9 @@ impl ExecutionContext { ) -> DeferredCommand<'a> { command.mark_as_executed(); + let created_at = command.get_created_location(); + let executed_at = std::panic::Location::caller(); + if self.dry_run() && !command.run_always { return DeferredCommand { process: None, @@ -96,15 +100,13 @@ impl ExecutionContext { stderr, command, exec_ctx: Arc::new(self.clone()), + created_at, }; } #[cfg(feature = "tracing")] let _run_span = trace_cmd!(command); - let created_at = command.get_created_location(); - let executed_at = std::panic::Location::caller(); - self.verbose(|| { println!("running: {command:?} (created at {created_at}, executed at {executed_at})") }); @@ -121,6 +123,7 @@ impl ExecutionContext { stderr, command, exec_ctx: Arc::new(self.clone()), + created_at, } } @@ -167,6 +170,7 @@ pub struct DeferredCommand<'a> { stdout: OutputMode, stderr: OutputMode, exec_ctx: Arc, + created_at: Location<'a>, } impl<'a> DeferredCommand<'a> { @@ -176,7 +180,7 @@ impl<'a> DeferredCommand<'a> { } let output = self.process.take().unwrap().wait_with_output(); - let created_at = self.command.get_created_location(); + let created_at = self.created_at; let executed_at = std::panic::Location::caller(); use std::fmt::Write; From dca9fe0d940c61b16d1a8d4a13b9639a7d035458 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 17 Jun 2025 18:27:06 +0530 Subject: [PATCH 23/32] explain reasoning behind spawn API --- src/bootstrap/src/utils/channel.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index a7ccdbf6eab07..3303e0b771534 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -59,6 +59,7 @@ impl GitInfo { } // Ok, let's scrape some info + // We use the command's spawn API to execute these commands concurrently, which leads to performance improvements. let mut git_log_cmd = helpers::git(Some(dir)); let ver_date = git_log_cmd .arg("log") From 05e1ae7bd4d3982a33431912d24623f6c46cd34c Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 17 Jun 2025 18:52:39 +0530 Subject: [PATCH 24/32] move execution context out of deferred command, add as_ref implementation and update wait_for_output usage --- src/bootstrap/src/utils/channel.rs | 6 +-- src/bootstrap/src/utils/execution_context.rs | 40 +++++++++----------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index 3303e0b771534..9a9100a820ba5 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -79,9 +79,9 @@ impl GitInfo { .start_capture_stdout(&exec_ctx); GitInfo::Present(Some(Info { - commit_date: ver_date.wait_for_output().stdout().trim().to_string(), - sha: ver_hash.wait_for_output().stdout().trim().to_string(), - short_sha: short_ver_hash.wait_for_output().stdout().trim().to_string(), + commit_date: ver_date.wait_for_output(&exec_ctx).stdout().trim().to_string(), + sha: ver_hash.wait_for_output(&exec_ctx).stdout().trim().to_string(), + short_sha: short_ver_hash.wait_for_output(&exec_ctx).stdout().trim().to_string(), })) } diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index 15b92f93b0426..5417307e54f84 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -94,14 +94,7 @@ impl ExecutionContext { let executed_at = std::panic::Location::caller(); if self.dry_run() && !command.run_always { - return DeferredCommand { - process: None, - stdout, - stderr, - command, - exec_ctx: Arc::new(self.clone()), - created_at, - }; + return DeferredCommand { process: None, stdout, stderr, command, created_at }; } #[cfg(feature = "tracing")] @@ -117,14 +110,7 @@ impl ExecutionContext { let child = cmd.spawn().unwrap(); - DeferredCommand { - process: Some(child), - stdout, - stderr, - command, - exec_ctx: Arc::new(self.clone()), - created_at, - } + DeferredCommand { process: Some(child), stdout, stderr, command, created_at } } /// Execute a command and return its output. @@ -137,7 +123,7 @@ impl ExecutionContext { stdout: OutputMode, stderr: OutputMode, ) -> CommandOutput { - self.start(command, stdout, stderr).wait_for_output() + self.start(command, stdout, stderr).wait_for_output(self) } fn fail(&self, message: &str, output: CommandOutput) -> ! { @@ -164,20 +150,28 @@ impl ExecutionContext { } } +impl AsRef for ExecutionContext { + fn as_ref(&self) -> &ExecutionContext { + self + } +} + pub struct DeferredCommand<'a> { process: Option, command: &'a mut BootstrapCommand, stdout: OutputMode, stderr: OutputMode, - exec_ctx: Arc, created_at: Location<'a>, } impl<'a> DeferredCommand<'a> { - pub fn wait_for_output(mut self) -> CommandOutput { + pub fn wait_for_output(mut self, exec_ctx: impl AsRef) -> CommandOutput { if self.process.is_none() { return CommandOutput::default(); } + + let exec_ctx = exec_ctx.as_ref(); + let output = self.process.take().unwrap().wait_with_output(); let created_at = self.created_at; @@ -234,14 +228,14 @@ Executed at: {executed_at}"#, if !output.is_success() { match self.command.failure_behavior { BehaviorOnFailure::DelayFail => { - if self.exec_ctx.fail_fast { - self.exec_ctx.fail(&message, output); + if exec_ctx.fail_fast { + exec_ctx.fail(&message, output); } - self.exec_ctx.add_to_delay_failure(message); + exec_ctx.add_to_delay_failure(message); } BehaviorOnFailure::Exit => { - self.exec_ctx.fail(&message, output); + exec_ctx.fail(&message, output); } BehaviorOnFailure::Ignore => { // If failures are allowed, either the error has been printed already From 55e2c2681e30d8274e0c00a57f506f3dc2c3d36d Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 17 Jun 2025 18:55:49 +0530 Subject: [PATCH 25/32] add run_always to recently migrated start_process command --- src/bootstrap/src/utils/channel.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index 9a9100a820ba5..b28ab57377408 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -66,16 +66,19 @@ impl GitInfo { .arg("-1") .arg("--date=short") .arg("--pretty=format:%cd") + .run_always() .start_capture_stdout(&exec_ctx); let mut git_hash_cmd = helpers::git(Some(dir)); - let ver_hash = git_hash_cmd.arg("rev-parse").arg("HEAD").start_capture_stdout(&exec_ctx); + let ver_hash = + git_hash_cmd.arg("rev-parse").arg("HEAD").run_always().start_capture_stdout(&exec_ctx); let mut git_short_hash_cmd = helpers::git(Some(dir)); let short_ver_hash = git_short_hash_cmd .arg("rev-parse") .arg("--short=9") .arg("HEAD") + .run_always() .start_capture_stdout(&exec_ctx); GitInfo::Present(Some(Info { From 186f5887725f72a73ed62f8bed95b7bb7b047739 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 17 Jun 2025 20:58:19 +0530 Subject: [PATCH 26/32] change to executed at --- src/bootstrap/src/utils/exec.rs | 14 ++++++++------ src/bootstrap/src/utils/execution_context.rs | 10 +++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 85a19ebe36880..eb9802bf2e1ba 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -2,11 +2,10 @@ //! //! This module provides a structured way to execute and manage commands efficiently, //! ensuring controlled failure handling and output management. -#![allow(warnings)] use std::ffi::OsStr; use std::fmt::{Debug, Formatter}; use std::path::Path; -use std::process::{Child, Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; use build_helper::ci::CiEnv; use build_helper::drop_bomb::DropBomb; @@ -73,7 +72,7 @@ pub struct BootstrapCommand { drop_bomb: DropBomb, } -impl BootstrapCommand { +impl<'a> BootstrapCommand { #[track_caller] pub fn new>(program: S) -> Self { Command::new(program).into() @@ -160,16 +159,19 @@ impl BootstrapCommand { /// Spawn the command in background, while capturing and returning all its output. #[track_caller] - pub fn start_capture(&mut self, exec_ctx: impl AsRef) -> DeferredCommand { + pub fn start_capture( + &'a mut self, + exec_ctx: impl AsRef, + ) -> DeferredCommand<'a> { exec_ctx.as_ref().start(self, OutputMode::Capture, OutputMode::Capture) } /// Spawn the command in background, while capturing and returning stdout, and printing stderr. #[track_caller] pub fn start_capture_stdout( - &mut self, + &'a mut self, exec_ctx: impl AsRef, - ) -> DeferredCommand { + ) -> DeferredCommand<'a> { exec_ctx.as_ref().start(self, OutputMode::Capture, OutputMode::Print) } diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index 5417307e54f84..a1b6ff94ca742 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -94,7 +94,7 @@ impl ExecutionContext { let executed_at = std::panic::Location::caller(); if self.dry_run() && !command.run_always { - return DeferredCommand { process: None, stdout, stderr, command, created_at }; + return DeferredCommand { process: None, stdout, stderr, command, executed_at }; } #[cfg(feature = "tracing")] @@ -110,7 +110,7 @@ impl ExecutionContext { let child = cmd.spawn().unwrap(); - DeferredCommand { process: Some(child), stdout, stderr, command, created_at } + DeferredCommand { process: Some(child), stdout, stderr, command, executed_at } } /// Execute a command and return its output. @@ -161,7 +161,7 @@ pub struct DeferredCommand<'a> { command: &'a mut BootstrapCommand, stdout: OutputMode, stderr: OutputMode, - created_at: Location<'a>, + executed_at: &'a Location<'a>, } impl<'a> DeferredCommand<'a> { @@ -174,8 +174,8 @@ impl<'a> DeferredCommand<'a> { let output = self.process.take().unwrap().wait_with_output(); - let created_at = self.created_at; - let executed_at = std::panic::Location::caller(); + let created_at = self.command.get_created_location(); + let executed_at = self.executed_at; use std::fmt::Write; From 05122808a39ad64aa629368984ae2693e4f66c11 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 17 Jun 2025 23:05:50 +0530 Subject: [PATCH 27/32] allow spawn to behave according to failure behavior --- src/bootstrap/src/utils/execution_context.rs | 88 ++++++++++++-------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index a1b6ff94ca742..5b9fef3f8248b 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -108,7 +108,7 @@ impl ExecutionContext { cmd.stdout(stdout.stdio()); cmd.stderr(stderr.stdio()); - let child = cmd.spawn().unwrap(); + let child = cmd.spawn(); DeferredCommand { process: Some(child), stdout, stderr, command, executed_at } } @@ -157,7 +157,7 @@ impl AsRef for ExecutionContext { } pub struct DeferredCommand<'a> { - process: Option, + process: Option>, command: &'a mut BootstrapCommand, stdout: OutputMode, stderr: OutputMode, @@ -166,61 +166,77 @@ pub struct DeferredCommand<'a> { impl<'a> DeferredCommand<'a> { pub fn wait_for_output(mut self, exec_ctx: impl AsRef) -> CommandOutput { - if self.process.is_none() { - return CommandOutput::default(); - } - let exec_ctx = exec_ctx.as_ref(); - let output = self.process.take().unwrap().wait_with_output(); + let process = match self.process.take() { + Some(p) => p, + None => return CommandOutput::default(), + }; let created_at = self.command.get_created_location(); let executed_at = self.executed_at; - use std::fmt::Write; - let mut message = String::new(); - let output: CommandOutput = match output { - // Command has succeeded - Ok(output) if output.status.success() => { - CommandOutput::from_output(output, self.stdout, self.stderr) - } - // Command has started, but then it failed - Ok(output) => { - writeln!( - message, - r#" + + let output = match process { + Ok(child) => match child.wait_with_output() { + Ok(result) if result.status.success() => { + // Successful execution + CommandOutput::from_output(result, self.stdout, self.stderr) + } + Ok(result) => { + // Command ran but failed + use std::fmt::Write; + + writeln!( + message, + r#" Command {:?} did not execute successfully. Expected success, got {} Created at: {created_at} Executed at: {executed_at}"#, - self.command, output.status, - ) - .unwrap(); + self.command, result.status, + ) + .unwrap(); + + let output = CommandOutput::from_output(result, self.stdout, self.stderr); - let output: CommandOutput = - CommandOutput::from_output(output, self.stdout, self.stderr); + if self.stdout.captures() { + writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); + } + if self.stderr.captures() { + writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); + } - // If the output mode is OutputMode::Capture, we can now print the output. - // If it is OutputMode::Print, then the output has already been printed to - // stdout/stderr, and we thus don't have anything captured to print anyway. - if self.stdout.captures() { - writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); + output } - if self.stderr.captures() { - writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); + Err(e) => { + // Failed to wait for output + use std::fmt::Write; + + writeln!( + message, + "\n\nCommand {:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}", + self.command + ) + .unwrap(); + + CommandOutput::did_not_start(self.stdout, self.stderr) } - output - } - // The command did not even start + }, Err(e) => { + // Failed to spawn the command + use std::fmt::Write; + writeln!( message, "\n\nCommand {:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}", + \nIt was not possible to execute the command: {e:?}", self.command ) .unwrap(); + CommandOutput::did_not_start(self.stdout, self.stderr) } }; @@ -231,7 +247,6 @@ Executed at: {executed_at}"#, if exec_ctx.fail_fast { exec_ctx.fail(&message, output); } - exec_ctx.add_to_delay_failure(message); } BehaviorOnFailure::Exit => { @@ -244,6 +259,7 @@ Executed at: {executed_at}"#, } } } + output } } From 0ac37ffb7f5bf5b8a3da01a3f56d80203a07f5d1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Sat, 14 Jun 2025 22:01:12 +1000 Subject: [PATCH 28/32] Add a test for the size of important types. A lot of these are large! Lots of room for improvement in the future. --- src/rustdoc-json-types/tests.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/rustdoc-json-types/tests.rs b/src/rustdoc-json-types/tests.rs index b9363fcf1b714..d17c550bf05b8 100644 --- a/src/rustdoc-json-types/tests.rs +++ b/src/rustdoc-json-types/tests.rs @@ -38,3 +38,28 @@ fn test_union_info_roundtrip() { let decoded: ItemEnum = bincode::deserialize(&encoded).unwrap(); assert_eq!(u, decoded); } + +// The memory used by a `Crate` can get large. These types are the ones that +// contribute the most to its size. +#[test] +#[cfg(target_pointer_width = "64")] +fn test_type_sizes() { + // tidy-alphabetical-start + assert_eq!(size_of::(), 208); + assert_eq!(size_of::(), 184); + assert_eq!(size_of::(), 48); + assert_eq!(size_of::(), 168); + assert_eq!(size_of::(), 80); + assert_eq!(size_of::(), 104); + assert_eq!(size_of::(), 72); + assert_eq!(size_of::(), 136); + assert_eq!(size_of::(), 304); + assert_eq!(size_of::(), 552); + assert_eq!(size_of::(), 32); + assert_eq!(size_of::(), 64); + assert_eq!(size_of::(), 32); + assert_eq!(size_of::(), 80); + assert_eq!(size_of::(), 80); + assert_eq!(size_of::(), 160); + // tidy-alphabetical-end +} From 68ad7305680585a00142a0c0826297f41eb4e67f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 16 Jun 2025 09:52:23 +1000 Subject: [PATCH 29/32] rustdoc_json: Add a test for some `GenericArgs` cases. --- tests/rustdoc-json/generic-args.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/rustdoc-json/generic-args.rs diff --git a/tests/rustdoc-json/generic-args.rs b/tests/rustdoc-json/generic-args.rs new file mode 100644 index 0000000000000..e48c3329f56ba --- /dev/null +++ b/tests/rustdoc-json/generic-args.rs @@ -0,0 +1,20 @@ +pub struct MyStruct(u32); + +pub trait MyTrait { + type MyType; + fn my_fn(&self); +} + +impl MyTrait for MyStruct { + type MyType = u32; + fn my_fn(&self) {} +} + +//@ is "$.index[?(@.name=='my_fn1')].inner.function.sig.inputs[0][1].qualified_path.args" {\"angle_bracketed\":{\"args\":[],\"constraints\":[]}} +//@ is "$.index[?(@.name=='my_fn1')].inner.function.sig.inputs[0][1].qualified_path.self_type.resolved_path.args" {\"angle_bracketed\":{\"args\":[],\"constraints\":[]}} +pub fn my_fn1(_: ::MyType) {} + +//@ is "$.index[?(@.name=='my_fn2')].inner.function.sig.inputs[0][1].dyn_trait.traits[0].trait.args.angle_bracketed.constraints[0].args" {\"angle_bracketed\":{\"args\":[],\"constraints\":[]}} +pub fn my_fn2(_: IntoIterator) {} + +fn main() {} From 62bec9059e4559b73ac4f67330c8953abd1e75f5 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Sat, 14 Jun 2025 20:14:51 +1000 Subject: [PATCH 30/32] rustdoc_json: Fix handling of paths with no generic args. A path without generic args, like `Reader`, currently has JSON produced like this: ``` {"path":"Reader","id":286,"args":{"angle_bracketed":{"args":[],"constraints":[]}}} ``` Even though `types::Path::args` is `Option` and allows for "no args", instead it gets represented as "empty args". (More like `Reader<>` than `Reader`.) This is due to a problem in `clean::Path::from_clean`. It only produces `None` if the path is an empty string. This commit changes it to also produce `None` if there are no generic args. The example above becomes: ``` {"path":"Reader","id":286,"args":null} ``` I looked at a few examples and saw this reduce the size of the JSON output by 3-9%. The commit also adds an assertion that non-final segments don't have any generics; something the old code was implicitly relying on. Note: the original sin here is that `clean::PathSegment::args` is not an `Option`, unlike `{ast,hir}::PathSegment::args`. I want to fix that, but it can be done separately. --- src/librustdoc/json/conversions.rs | 19 ++++++++++++++++++- tests/rustdoc-json/generic-args.rs | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 6bdf3b5fe3876..3b270d111a17e 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -580,7 +580,24 @@ impl FromClean for Path { Path { path: path.whole_name(), id: renderer.id_from_item_default(path.def_id().into()), - args: path.segments.last().map(|args| Box::new(args.args.into_json(renderer))), + args: { + if let Some((final_seg, rest_segs)) = path.segments.split_last() { + // In general, `clean::Path` can hold things like + // `std::vec::Vec::::new`, where generic args appear + // in a middle segment. But for the places where `Path` is + // used by rustdoc-json-types, generic args can only be + // used in the final segment, e.g. `std::vec::Vec`. So + // check that the non-final segments have no generic args. + assert!(rest_segs.iter().all(|seg| seg.args.is_empty())); + if final_seg.args.is_empty() { + None + } else { + Some(Box::new(final_seg.args.into_json(renderer))) + } + } else { + None // no generics on any segments because there are no segments + } + }, } } } diff --git a/tests/rustdoc-json/generic-args.rs b/tests/rustdoc-json/generic-args.rs index e48c3329f56ba..e87c1e6230432 100644 --- a/tests/rustdoc-json/generic-args.rs +++ b/tests/rustdoc-json/generic-args.rs @@ -11,7 +11,7 @@ impl MyTrait for MyStruct { } //@ is "$.index[?(@.name=='my_fn1')].inner.function.sig.inputs[0][1].qualified_path.args" {\"angle_bracketed\":{\"args\":[],\"constraints\":[]}} -//@ is "$.index[?(@.name=='my_fn1')].inner.function.sig.inputs[0][1].qualified_path.self_type.resolved_path.args" {\"angle_bracketed\":{\"args\":[],\"constraints\":[]}} +//@ is "$.index[?(@.name=='my_fn1')].inner.function.sig.inputs[0][1].qualified_path.self_type.resolved_path.args" null pub fn my_fn1(_: ::MyType) {} //@ is "$.index[?(@.name=='my_fn2')].inner.function.sig.inputs[0][1].dyn_trait.traits[0].trait.args.angle_bracketed.constraints[0].args" {\"angle_bracketed\":{\"args\":[],\"constraints\":[]}} From 203793ebcdd3f1273f8e5b425625a2b80a566048 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Sat, 14 Jun 2025 20:22:30 +1000 Subject: [PATCH 31/32] Fix some comments. As per the previous commit, generic args here can only appear on the final segment. So make the comments obey that constraint. --- src/rustdoc-json-types/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rustdoc-json-types/lib.rs b/src/rustdoc-json-types/lib.rs index 2c94b8f914f7d..e47a697243dd6 100644 --- a/src/rustdoc-json-types/lib.rs +++ b/src/rustdoc-json-types/lib.rs @@ -277,8 +277,8 @@ pub struct PolyTrait { /// A set of generic arguments provided to a path segment, e.g. /// /// ```text -/// std::option::Option::::None -/// ^^^^^ +/// std::option::Option +/// ^^^^^ /// ``` #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -331,7 +331,7 @@ pub enum GenericArg { Const(Constant), /// A generic argument that's explicitly set to be inferred. /// ```text - /// std::vec::Vec::<_>::new() + /// std::vec::Vec::<_> /// ^ /// ``` Infer, From 81a2aecfda381ff50b170e63821e9ff97f22fd03 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Sat, 14 Jun 2025 21:39:09 +1000 Subject: [PATCH 32/32] rustdoc_json: represent generic args consistently. They show up in three places: once as `Option>`, once as `Box`, and once as `GenericArgs`. The first option is best. It is more compact because generic args are often missing. This commit changes the latter two to the former. Example output, before and after, for the `AssocItemConstraint` change: ``` {"name":"Offset","args":{"angle_bracketed":{"args":[],"constraints":[]}},"binding":{...}} {"name":"Offset","args":null,"binding":{...}} ``` Example output, before and after, for the `Type::QualifiedPath` change: ``` {"qualified_path":{"name":"Offset","args":{"angle_bracketed":{"args":[],"constraints":[]}}, ...}} {"qualified_path":{"name":"Offset","args":null, ...}} ``` This reduces JSON output size, but not by much (e.g. 0.5%), because `AssocItemConstraint` and `Type::QualifiedPath` are uncommon. --- src/librustdoc/json/conversions.rs | 17 ++++++++--------- src/rustdoc-json-types/lib.rs | 8 ++++---- src/rustdoc-json-types/tests.rs | 2 +- src/tools/jsondoclint/src/validator.rs | 13 ++++++------- tests/rustdoc-json/generic-args.rs | 4 ++-- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 3b270d111a17e..b9df594dcca96 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -170,10 +170,13 @@ pub(crate) fn from_deprecation(deprecation: attrs::Deprecation) -> Deprecation { Deprecation { since, note: note.map(|s| s.to_string()) } } -impl FromClean for GenericArgs { +impl FromClean for Option> { fn from_clean(args: &clean::GenericArgs, renderer: &JsonRenderer<'_>) -> Self { use clean::GenericArgs::*; - match args { + if args.is_empty() { + return None; + } + Some(Box::new(match args { AngleBracketed { args, constraints } => GenericArgs::AngleBracketed { args: args.into_json(renderer), constraints: constraints.into_json(renderer), @@ -183,7 +186,7 @@ impl FromClean for GenericArgs { output: output.as_ref().map(|a| a.as_ref().into_json(renderer)), }, ReturnTypeNotation => GenericArgs::ReturnTypeNotation, - } + })) } } @@ -589,11 +592,7 @@ impl FromClean for Path { // used in the final segment, e.g. `std::vec::Vec`. So // check that the non-final segments have no generic args. assert!(rest_segs.iter().all(|seg| seg.args.is_empty())); - if final_seg.args.is_empty() { - None - } else { - Some(Box::new(final_seg.args.into_json(renderer))) - } + final_seg.args.into_json(renderer) } else { None // no generics on any segments because there are no segments } @@ -608,7 +607,7 @@ impl FromClean for Type { Self::QualifiedPath { name: assoc.name.to_string(), - args: Box::new(assoc.args.into_json(renderer)), + args: assoc.args.into_json(renderer), self_type: Box::new(self_type.into_json(renderer)), trait_: trait_.as_ref().map(|trait_| trait_.into_json(renderer)), } diff --git a/src/rustdoc-json-types/lib.rs b/src/rustdoc-json-types/lib.rs index e47a697243dd6..cf9b86b967c0f 100644 --- a/src/rustdoc-json-types/lib.rs +++ b/src/rustdoc-json-types/lib.rs @@ -37,8 +37,8 @@ pub type FxHashMap = HashMap; // re-export for use in src/librustdoc // will instead cause conflicts. See #94591 for more. (This paragraph and the "Latest feature" line // are deliberately not in a doc comment, because they need not be in public docs.) // -// Latest feature: rustdoc JSON: Don't apply #[repr] privacy heuristics -pub const FORMAT_VERSION: u32 = 46; +// Latest feature: improve handling of generic args +pub const FORMAT_VERSION: u32 = 47; /// The root of the emitted JSON blob. /// @@ -362,7 +362,7 @@ pub struct AssocItemConstraint { /// The name of the associated type/constant. pub name: String, /// Arguments provided to the associated type/constant. - pub args: GenericArgs, + pub args: Option>, /// The kind of bound applied to the associated type/constant. pub binding: AssocItemConstraintKind, } @@ -1118,7 +1118,7 @@ pub enum Type { /// as BetterIterator>::Item<'static> /// // ^^^^^^^^^ /// ``` - args: Box, + args: Option>, /// The type with which this type is associated. /// /// ```ignore (incomplete expression) diff --git a/src/rustdoc-json-types/tests.rs b/src/rustdoc-json-types/tests.rs index d17c550bf05b8..c50eb61650126 100644 --- a/src/rustdoc-json-types/tests.rs +++ b/src/rustdoc-json-types/tests.rs @@ -45,7 +45,7 @@ fn test_union_info_roundtrip() { #[cfg(target_pointer_width = "64")] fn test_type_sizes() { // tidy-alphabetical-start - assert_eq!(size_of::(), 208); + assert_eq!(size_of::(), 112); assert_eq!(size_of::(), 184); assert_eq!(size_of::(), 48); assert_eq!(size_of::(), 168); diff --git a/src/tools/jsondoclint/src/validator.rs b/src/tools/jsondoclint/src/validator.rs index 8c9e4c8bb3a60..0a4051fcbe8cd 100644 --- a/src/tools/jsondoclint/src/validator.rs +++ b/src/tools/jsondoclint/src/validator.rs @@ -271,7 +271,7 @@ impl<'a> Validator<'a> { Type::RawPointer { is_mutable: _, type_ } => self.check_type(&**type_), Type::BorrowedRef { lifetime: _, is_mutable: _, type_ } => self.check_type(&**type_), Type::QualifiedPath { name: _, args, self_type, trait_ } => { - self.check_generic_args(&**args); + self.check_opt_generic_args(&args); self.check_type(&**self_type); if let Some(trait_) = trait_ { self.check_path(trait_, PathKind::Trait); @@ -309,13 +309,12 @@ impl<'a> Validator<'a> { self.fail(&x.id, ErrorKind::Custom(format!("No entry in '$.paths' for {x:?}"))); } - if let Some(args) = &x.args { - self.check_generic_args(&**args); - } + self.check_opt_generic_args(&x.args); } - fn check_generic_args(&mut self, x: &'a GenericArgs) { - match x { + fn check_opt_generic_args(&mut self, x: &'a Option>) { + let Some(x) = x else { return }; + match &**x { GenericArgs::AngleBracketed { args, constraints } => { args.iter().for_each(|arg| self.check_generic_arg(arg)); constraints.iter().for_each(|bind| self.check_assoc_item_constraint(bind)); @@ -355,7 +354,7 @@ impl<'a> Validator<'a> { } fn check_assoc_item_constraint(&mut self, bind: &'a AssocItemConstraint) { - self.check_generic_args(&bind.args); + self.check_opt_generic_args(&bind.args); match &bind.binding { AssocItemConstraintKind::Equality(term) => self.check_term(term), AssocItemConstraintKind::Constraint(bounds) => { diff --git a/tests/rustdoc-json/generic-args.rs b/tests/rustdoc-json/generic-args.rs index e87c1e6230432..0f588820da75d 100644 --- a/tests/rustdoc-json/generic-args.rs +++ b/tests/rustdoc-json/generic-args.rs @@ -10,11 +10,11 @@ impl MyTrait for MyStruct { fn my_fn(&self) {} } -//@ is "$.index[?(@.name=='my_fn1')].inner.function.sig.inputs[0][1].qualified_path.args" {\"angle_bracketed\":{\"args\":[],\"constraints\":[]}} +//@ is "$.index[?(@.name=='my_fn1')].inner.function.sig.inputs[0][1].qualified_path.args" null //@ is "$.index[?(@.name=='my_fn1')].inner.function.sig.inputs[0][1].qualified_path.self_type.resolved_path.args" null pub fn my_fn1(_: ::MyType) {} -//@ is "$.index[?(@.name=='my_fn2')].inner.function.sig.inputs[0][1].dyn_trait.traits[0].trait.args.angle_bracketed.constraints[0].args" {\"angle_bracketed\":{\"args\":[],\"constraints\":[]}} +//@ is "$.index[?(@.name=='my_fn2')].inner.function.sig.inputs[0][1].dyn_trait.traits[0].trait.args.angle_bracketed.constraints[0].args" null pub fn my_fn2(_: IntoIterator) {} fn main() {}