bpart: Fully switch to partitioned semantics

This is the final PR in the binding partitions series (modulo bugs and tweaks),
i.e. it closes #54654 and thus closes #40399, which was the original design
sketch.

This thus activates the full designed semantics for binding partitions,
in particular allowing safe replacement of const bindings. It in particular
allows struct redefinitions. This thus closes timholy/Revise.jl#18 and
also closes #38584.

The biggest semantic change here is probably that this gets rid of the
notion of "resolvedness" of a binding. Previously, a lot of the behavior
of our implementation depended on when bindings were "resolved", which
could happen at basically an arbitrary point (in the compiler, in REPL
completion, in a different thread), making a lot of the semantics around
bindings ill- or at least implementation-defined. There are several related
issues in the bugtracker, so this closes #14055 #44604 #46354 #30277

It is also the last step to close #24569.
It also supports bindings for undef->defined transitions and thus
closes #53958 #54733 - however, this is not activated yet for performance
reasons and may need some further optimization.

Since resolvedness no longer exists, we need to replace it with some
hopefully more well-defined semantics. I will describe the semantics
below, but before I do I will make two notes:

1. There are a number of cases where these semantics will behave
   slightly differently than the old semantics absent some other
   task going around resolving random bindings.
2. The new behavior (except for the replacement stuff) was generally
   permissible under the old semantics if the bindings
   happened to be resolved at the right time.

With all that said, there are essentially three "strengths" of bindings:

1. Implicit Bindings: Anything implicitly obtained from `using Mod`, "no binding",
   plus slightly more exotic corner cases around conflicts

2. Weakly declared bindings: Declared using `global sym` and nothing else

3. Strongly declared bindings: Declared using `global sym::T`, `const sym=val`,
   `import Mod: sym`, `using Mod: sym` or as an implicit strong global declaration
   in `sym=val`, where `sym` is known to be global (either by being at toplevle
   or as `global sym=val` inside a function).

In general, you always allowed to syntactically replace a weaker binding
by a stronger one (although the runtime permits arbitrary binding deletion
now, this is just a syntactic constraint to catch errors).
Second, any implicit binding can be replaced by other implicit
bindings as the result of changing the `using`'ed module.
And lastly, any constants may be replaced by any other constants
(irrespective of type).

We do not currently allow replacing globals, but may consider changing that
in 1.13.

This is mostly how things used to work, as well in the absence of any stray
external binding resolutions. The most prominent difference is probably this one:

```
set_foo!() = global foo = 1
```

In the above terminology, this now always declares a "strongly declared binding",
whereas before it declared a "weakly declared binding" that would become
strongly declared on first write to the global (unless of course somebody
had created a different strongly declared global in the meantime). To see
the difference, this is now disallowed:

```
julia> set_foo!() = global foo = 1
set_foo! (generic function with 1 method)

julia> const foo = 1
ERROR: cannot declare Main.foo constant; it was already declared global
Stacktrace:
 [1] top-level scope
   @ REPL[2]:1
```

Before it would depend on the order of binding resolution (although it
just crashes on current master for some reason - whoops, probably my
fault).

Another major change is the ambiguousness of imports. In:
```
module M1; export x; x=1; end
module M2; export x; x=2; end
using .M1, .M2
```
the binding `Main.x` is now always ambiguous and will throw on access.
Before which binding you get, would depend on resolution order. To choose
one, use an explicit import (which was the behavior you would previously
get if neither binding was resolved before both imports).

Author: Keno

Compiler/src/Compiler.jl

@@ -49,7 +49,7 @@ using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstanc
 
 using Base
 using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospecializeinfer,
-    BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST,
+    BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST, BINDING_KIND_DECLARED,
     Base, BitVector, Bottom, Callable, DataTypeFieldDesc,
     EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES,
     OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME,

Compiler/src/abstractinterpretation.jl

@@ -3464,14 +3464,7 @@ world_range(ci::CodeInfo) = WorldRange(ci.min_world, ci.max_world)
 world_range(ci::CodeInstance) = WorldRange(ci.min_world, ci.max_world)
 world_range(compact::IncrementalCompact) = world_range(compact.ir)
 
-function force_binding_resolution!(g::GlobalRef, world::UInt)
-    # Force resolution of the binding
-    # TODO: This will go away once we switch over to fully partitioned semantics
-    ccall(:jl_force_binding_resolution, Cvoid, (Any, Csize_t), g, world)
-    return nothing
-end
-
-function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}, retry_after_resolve::Bool=true)
+function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact})
     worlds = world_range(src)
     partition = lookup_binding_partition(min_world(worlds), g)
     partition.max_world < max_world(worlds) && return Any
@@ -3480,25 +3473,18 @@ function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode,
         partition = lookup_binding_partition(min_world(worlds), imported_binding)
         partition.max_world < max_world(worlds) && return Any
     end
-    if is_some_guard(binding_kind(partition))
-        if retry_after_resolve
-            # This method is surprisingly hot. For performance, don't ask the runtime to resolve
-            # the binding unless necessary - doing so triggers an additional lookup, which though
-            # not super expensive is hot enough to show up in benchmarks.
-            force_binding_resolution!(g, min_world(worlds))
-            return abstract_eval_globalref_type(g, src, false)
-        end
+    kind = binding_kind(partition)
+    if is_some_guard(kind)
         # return Union{}
         return Any
     end
-    if is_some_const_binding(binding_kind(partition))
+    if is_some_const_binding(kind)
         return Const(partition_restriction(partition))
     end
-    return partition_restriction(partition)
+    return kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition)
 end
 
 function lookup_binding_partition!(interp::AbstractInterpreter, g::GlobalRef, sv::AbsIntState)
-    force_binding_resolution!(g, get_inference_world(interp))
     partition = lookup_binding_partition(get_inference_world(interp), g)
     update_valid_age!(sv, WorldRange(partition.min_world, partition.max_world))
     partition
@@ -3541,7 +3527,11 @@ function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Co
         return RTEffects(rt, Union{}, Effects(EFFECTS_TOTAL, inaccessiblememonly=is_mutation_free_argtype(rt) ? ALWAYS_TRUE : ALWAYS_FALSE))
     end
 
-    rt = partition_restriction(partition)
+    if kind == BINDING_KIND_DECLARED
+        rt = Any
+    else
+        rt = partition_restriction(partition)
+    end
     return RTEffects(rt, UndefVarError, generic_getglobal_effects)
 end
 
@@ -3580,7 +3570,7 @@ function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partitio
     elseif is_some_const_binding(kind)
         return Pair{Any,Any}(Bottom, ErrorException)
     end
-    ty = partition_restriction(partition)
+    ty = kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition)
     wnewty = widenconst(newty)
     if !hasintersect(wnewty, ty)
         return Pair{Any,Any}(Bottom, TypeError)

Compiler/src/ssair/verify.jl

@@ -61,7 +61,6 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int,
             raise_error()
         end
     elseif isa(op, GlobalRef)
-        force_binding_resolution!(op, min_world(ir.valid_worlds))
         bpart = lookup_binding_partition(min_world(ir.valid_worlds), op)
         while is_some_imported(binding_kind(bpart)) && max_world(ir.valid_worlds) <= bpart.max_world
             imported_binding = partition_restriction(bpart)::Core.Binding

Compiler/src/validation.jl

@@ -22,7 +22,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}(
     :copyast => 1:1,
     :meta => 0:typemax(Int),
     :global => 1:1,
-    :globaldecl => 2:2,
+    :globaldecl => 1:2,
     :foreigncall => 5:typemax(Int), # name, RT, AT, nreq, (cconv, effects), args..., roots...
     :cfunction => 5:5,
     :isdefined => 1:2,

Compiler/test/codegen.jl

@@ -689,6 +689,10 @@ U41096 = Term41096{:U}(Modulate41096(:U, false))
 
 # test that we can start julia with libjulia-codegen removed; PR #41936
 mktempdir() do pfx
+    # First check that ordinary running won't try to compile things. If it does that indicates an insufficient precompile or invalidation.
+    @test success(`$(Base.julia_cmd()) --startup-file=no --trace-compile="$(pfx)/compiles.txt" -e 'print("no codegen!\n")'`)
+    @test !isfile(joinpath(pfx, "compiles.txt"))
+
     cp(dirname(Sys.BINDIR), pfx; force=true)
     libpath = relpath(dirname(dlpath(libjulia_codegen_name())), dirname(Sys.BINDIR))
     libs_deleted = 0

Compiler/test/effects.jl

@@ -378,32 +378,27 @@ let effects = Base.infer_effects(; optimize=false) do
 end
 
 # we should taint `nothrow` if the binding doesn't exist and isn't fixed yet,
-# as the cached effects can be easily wrong otherwise
-# since the inference currently doesn't track "world-age" of global variables
-@eval global_assignment_undefinedyet() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET)) = 42
 setglobal!_nothrow_undefinedyet() = setglobal!(@__MODULE__, :UNDEFINEDYET, 42)
-let effects = Base.infer_effects() do
-        global_assignment_undefinedyet()
-    end
+let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet)
     @test !Compiler.is_nothrow(effects)
 end
-let effects = Base.infer_effects() do
-        setglobal!_nothrow_undefinedyet()
-    end
-    @test !Compiler.is_nothrow(effects)
+@test_throws ErrorException setglobal!_nothrow_undefinedyet()
+# This declares the binding as ::Any
+@eval global_assignment_undefinedyet() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET)) = 42
+let effects = Base.infer_effects(global_assignment_undefinedyet)
+    @test Compiler.is_nothrow(effects)
 end
-global UNDEFINEDYET::String = "0"
-let effects = Base.infer_effects() do
-        global_assignment_undefinedyet()
-    end
+# Again with type mismatch
+global UNDEFINEDYET2::String = "0"
+setglobal!_nothrow_undefinedyet2() = setglobal!(@__MODULE__, :UNDEFINEDYET2, 42)
+@eval global_assignment_undefinedyet2() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET2)) = 42
+let effects = Base.infer_effects(global_assignment_undefinedyet2)
     @test !Compiler.is_nothrow(effects)
 end
-let effects = Base.infer_effects() do
-        setglobal!_nothrow_undefinedyet()
-    end
+let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet2)
     @test !Compiler.is_nothrow(effects)
 end
-@test_throws Union{ErrorException,TypeError} setglobal!_nothrow_undefinedyet() # TODO: what kind of error should this be?
+@test_throws TypeError setglobal!_nothrow_undefinedyet2()
 
 # Nothrow for setfield!
 mutable struct SetfieldNothrow

Compiler/test/inference.jl

@@ -6160,7 +6160,7 @@ end === Int
     swapglobal!(@__MODULE__, :swapglobal!_xxx, x)
 end === Union{}
 
-global swapglobal!_must_throw
+eval(Expr(:const, :swapglobal!_must_throw))
 @newinterp SwapGlobalInterp
 Compiler.InferenceParams(::SwapGlobalInterp) = Compiler.InferenceParams(; assume_bindings_static=true)
 function func_swapglobal!_must_throw(x)

base/boot.jl

@@ -229,7 +229,7 @@ export
     Expr, QuoteNode, LineNumberNode, GlobalRef,
     # object model functions
     fieldtype, getfield, setfield!, swapfield!, modifyfield!, replacefield!, setfieldonce!,
-    nfields, throw, tuple, ===, isdefined, eval,
+    nfields, throw, tuple, ===, isdefined,
     # access to globals
     getglobal, setglobal!, swapglobal!, modifyglobal!, replaceglobal!, setglobalonce!, isdefinedglobal,
     # ifelse, sizeof    # not exported, to avoid conflicting with Base

base/client.jl

@@ -267,9 +267,8 @@ function exec_options(opts)
         let Distributed = require(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed"))
             Core.eval(MainInclude, :(const Distributed = $Distributed))
             Core.eval(Main, :(using Base.MainInclude.Distributed))
+            invokelatest(Distributed.process_opts, opts)
         end
-
-        invokelatest(Main.Distributed.process_opts, opts)
     end
 
     interactiveinput = (repl || is_interactive::Bool) && isa(stdin, TTY)

base/deprecated.jl

@@ -531,4 +531,29 @@ end
 
 # BEGIN 1.12 deprecations
 
+@deprecate isbindingresolved(m::Module, var::Symbol) true false
+
+"""
+    isbindingresolved(m::Module, s::Symbol) -> Bool
+
+Returns whether the binding of a symbol in a module is resolved.
+
+See also: [`isexported`](@ref), [`ispublic`](@ref), [`isdeprecated`](@ref)
+
+```jldoctest
+julia> module Mod
+           foo() = 17
+       end
+Mod
+
+julia> Base.isbindingresolved(Mod, :foo)
+true
+```
+
+!!! warning
+    This function is deprecated. The concept of binding "resolvedness" was removed in Julia 1.12.
+    The function now always returns `true`.
+"""
+isbindingresolved
+
 # END 1.12 deprecations

base/invalidation.jl

@@ -113,32 +113,56 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid
     end
 end
 
-function invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_max_world::UInt)
-    b = convert(Core.Binding, gr)
-    try
-        valid_in_valuepos = false
-        foreach_module_mtable(gr.mod, new_max_world) do mt::Core.MethodTable
-            for method in MethodList(mt)
-                invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world)
+const BINDING_FLAG_EXPORTP = 0x2
+
+function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Union{Core.BindingPartition, Nothing}, new_max_world::UInt)
+    gr = b.globalref
+    if is_some_guard(binding_kind(invalidated_bpart))
+        # TODO: We may want to invalidate for these anyway, since they have performance implications
+        return
+    end
+    foreach_module_mtable(gr.mod, new_max_world) do mt::Core.MethodTable
+        for method in MethodList(mt)
+            invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world)
+        end
+        return true
+    end
+    if isdefined(b, :backedges)
+        for edge in b.backedges
+            if isa(edge, CodeInstance)
+                ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world)
+            elseif isa(edge, Core.Binding)
+                isdefined(edge, :partitions) || continue
+                latest_bpart = edge.partitions
+                latest_bpart.max_world == typemax(UInt) || continue
+                is_some_imported(binding_kind(latest_bpart)) || continue
+                partition_restriction(latest_bpart) === b || continue
+                invalidate_code_for_globalref!(edge, latest_bpart, nothing, new_max_world)
+            else
+                invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world)
             end
-            return true
         end
-        b = convert(Core.Binding, gr)
-        if isdefined(b, :backedges)
-            for edge in b.backedges
-                if isa(edge, CodeInstance)
-                    ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world)
-                else
-                    invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world)
-                end
+    end
+    if (b.flags & BINDING_FLAG_EXPORTP) != 0
+        # This binding was exported - we need to check all modules that `using` us to see if they
+        # have an implicit binding to us.
+        usings_backedges = ccall(:jl_get_module_usings_backedges, Any, (Any,), gr.mod)
+        if usings_backedges !== nothing
+            for user in usings_backedges::Vector{Any}
+                user_binding = ccall(:jl_get_module_binding_or_nothing, Any, (Any, Any), user, gr.name)
+                user_binding === nothing && continue
+                isdefined(user_binding, :partitions) || continue
+                latest_bpart = user_binding.partitions
+                latest_bpart.max_world == typemax(UInt) || continue
+                is_some_imported(binding_kind(latest_bpart)) || continue
+                partition_restriction(latest_bpart) === b || continue
+                invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, nothing, new_max_world)
             end
         end
-    catch err
-        bt = catch_backtrace()
-        invokelatest(Base.println, "Internal Error during invalidation:")
-        invokelatest(Base.display_error, err, bt)
     end
 end
+invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) =
+    invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world)
 
 gr_needs_backedge_in_module(gr::GlobalRef, mod::Module) = gr.mod !== mod
 

base/reflection.jl

@@ -46,92 +46,6 @@ function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool=
     return ret
 end
 
-# high-level, more convenient method lookup functions
-
-function visit(f, mt::Core.MethodTable)
-    mt.defs !== nothing && visit(f, mt.defs)
-    nothing
-end
-function visit(f, mc::Core.TypeMapLevel)
-    function avisit(f, e::Memory{Any})
-        for i in 2:2:length(e)
-            isassigned(e, i) || continue
-            ei = e[i]
-            if ei isa Memory{Any}
-                for j in 2:2:length(ei)
-                    isassigned(ei, j) || continue
-                    visit(f, ei[j])
-                end
-            else
-                visit(f, ei)
-            end
-        end
-    end
-    if mc.targ !== nothing
-        avisit(f, mc.targ::Memory{Any})
-    end
-    if mc.arg1 !== nothing
-        avisit(f, mc.arg1::Memory{Any})
-    end
-    if mc.tname !== nothing
-        avisit(f, mc.tname::Memory{Any})
-    end
-    if mc.name1 !== nothing
-        avisit(f, mc.name1::Memory{Any})
-    end
-    mc.list !== nothing && visit(f, mc.list)
-    mc.any !== nothing && visit(f, mc.any)
-    nothing
-end
-function visit(f, d::Core.TypeMapEntry)
-    while d !== nothing
-        f(d.func)
-        d = d.next
-    end
-    nothing
-end
-struct MethodSpecializations
-    specializations::Union{Nothing, Core.MethodInstance, Core.SimpleVector}
-end
-"""
-    specializations(m::Method) → itr
-
-Return an iterator `itr` of all compiler-generated specializations of `m`.
-"""
-specializations(m::Method) = MethodSpecializations(isdefined(m, :specializations) ? m.specializations : nothing)
-function iterate(specs::MethodSpecializations)
-    s = specs.specializations
-    s === nothing && return nothing
-    isa(s, Core.MethodInstance) && return (s, nothing)
-    return iterate(specs, 0)
-end
-iterate(specs::MethodSpecializations, ::Nothing) = nothing
-function iterate(specs::MethodSpecializations, i::Int)
-    s = specs.specializations::Core.SimpleVector
-    n = length(s)
-    i >= n && return nothing
-    item = nothing
-    while i < n && item === nothing
-        item = s[i+=1]
-    end
-    item === nothing && return nothing
-    return (item, i)
-end
-length(specs::MethodSpecializations) = count(Returns(true), specs)
-
-function length(mt::Core.MethodTable)
-    n = 0
-    visit(mt) do m
-        n += 1
-    end
-    return n::Int
-end
-isempty(mt::Core.MethodTable) = (mt.defs === nothing)
-
-uncompressed_ir(m::Method) = isdefined(m, :source) ? _uncompressed_ir(m) :
-                             isdefined(m, :generator) ? error("Method is @generated; try `code_lowered` instead.") :
-                             error("Code for this Method is not available.")
-
 # for backwards compat
 const uncompressed_ast = uncompressed_ir
 const _uncompressed_ast = _uncompressed_ir