changes/simplifications/bugfixes to support v1.12

CHANGELOG.md

@@ -2,7 +2,9 @@
 
 ## v1.0.5 - unreleased
 
-- Some general cleanup of the code, and adding ExplicitImports.jl tests for long term maintainability.
+- Some general cleanup of the code
+- Adding ExplicitImports.jl tests for long term maintainability.
+- Restored edge code, for soundness of re-evaluation (lost in #124)
 
 ## v1.0.4 - 2025-08-25
 

src/macro.jl

@@ -109,8 +109,10 @@ macro resumable(ex::Expr...)
   isfunctional = @capture(func_def[:name], functional_::T_) && inexpr(func_def[:body], functional)
   if isfunctional
     slots[functional] = T
-    push!(args, functional)
+  else
+    functional = :var"#self#"
   end
+  pushfirst!(args, functional)
 
   # The finite state machine structure definition
   type_name = gensym(Symbol(func_def[:name], :_FSMI))
@@ -150,14 +152,24 @@ macro resumable(ex::Expr...)
     bareconst_expr = nothing
   end
   constr_expr = combinedef(constr_def) |> flatten
-  type_expr = :(
+  typed_fsmi = VERSION >= v"1.10.0-DEV.873" ? gensym(:typed_fsmi) : typed_fsmi_fallback
+  type_expr = quote
     mutable struct $struct_name
       _state :: UInt8
       $((:($slotname :: $slottype) for (slotname, slottype) in zip(keys(slots), slot_T))...)
       $(constr_expr)
       $(bareconst_expr)
     end
-  )
+    # JuliaLang/julia#48611: world age is exposed to generated functions, and should be used
+    if VERSION >= v"1.10.0-DEV.873"
+      # This is like @generated, but it receives the world age of the caller
+      # which we need to do inference safely and correctly
+      function $typed_fsmi(fsmi, fargs...)
+          $(Expr(:meta, :generated_only))
+          $(Expr(:meta, :generated, FSMIGenerator(inferfn)))
+      end
+    end
+  end
   @debug type_expr |> striplines
   # The "original" function that now is simply a wrapper around the construction of the finite state machine
   call_def = copy(func_def)
@@ -168,10 +180,9 @@ macro resumable(ex::Expr...)
     fsmi_name = :($type_name{$(params...)})
   end
   fwd_args, fwd_kwargs = forward_args(call_def)
-  isfunctional && push!(fwd_args, functional)
   call_def[:body] = quote
-    fsmi = ResumableFunctions.typed_fsmi($fsmi_name, $inferfn, $(fwd_args...), $(fwd_kwargs...))
-    $((arg !== Symbol("_") ? :(fsmi.$arg = $arg) : nothing for arg in args)...)
+    fsmi = $typed_fsmi($fsmi_name, $functional, $(fwd_args...), $(fwd_kwargs...))
+    $((arg !== :_ && arg !== :var"#self#" ? :(fsmi.$arg = $arg) : nothing for arg in args)...)
     $((:(fsmi.$arg = $arg) for arg in kwargs)...)
     fsmi
   end
@@ -211,10 +222,10 @@ macro resumable(ex::Expr...)
   func_def[:body] = postwalk(x->transform_yield(x, ui8), func_def[:body])
   func_def[:body] = postwalk(x->transform_nosave(x, Set{Symbol}()), func_def[:body])
   func_def[:body] = quote
-    _fsmi._state === 0x00 && @goto $(Symbol("_STATE_0"))
-    $((:(_fsmi._state === $i && @goto $(Symbol("_STATE_",:($i)))) for i in 0x01:ui8.n)...)
+    _fsmi._state === 0x00 && @goto _STATE_0
+    $((:(_fsmi._state === $i && @goto $(Symbol("_STATE_",i))) for i in 0x01:ui8.n)...)
     error("@resumable function has stopped!")
-    @label $(Symbol("_STATE_0"))
+    @label _STATE_0
     _fsmi._state = 0xff
     _arg isa Exception && throw(_arg)
     $(func_def[:body])
@@ -225,7 +236,7 @@ macro resumable(ex::Expr...)
 
   if inexpr(func_def[:body], call_def[:name]) || isfunctional
     @debug "recursion or self-reference is present in a resumable function definition: falling back to no inference"
-    call_expr = postwalk(x->x==:(ResumableFunctions.typed_fsmi) ? :(ResumableFunctions.typed_fsmi_fallback) : x, call_expr)
+    call_expr = postwalk(x->x===typed_fsmi ? :(ResumableFunctions.typed_fsmi_fallback) : x, call_expr)
   end
   @debug func_expr |> striplines
   # The final expression:

src/utils.jl

@@ -60,13 +60,18 @@ end
 
 const unused = (Symbol("#temp#"), Symbol("_"), Symbol(""), Symbol("#unused#"), Symbol("#self#"))
 
+function strip_defaults(arg_exprs::Vector{Any})
+  return Any[@capture(arg_expr, arg_expr2_ = default_) ? arg_expr2 : arg_expr
+    for arg_expr in arg_exprs]
+end
+
 """
 Function returning the slots of a function definition
 """
 function get_slots(func_def::Dict, args::Dict{Symbol, Any}, mod::Module)
   slots = Dict{Symbol, Any}()
   func_def[:name] = gensym()
-  func_def[:args] = (func_def[:args]..., func_def[:kwargs]...)
+  func_def[:args] = Any[strip_defaults(func_def[:args])..., strip_defaults(func_def[:kwargs])...]
   func_def[:kwargs] = []
   # replace yield with inference barrier
   func_def[:body] = postwalk(transform_yield, func_def[:body])
@@ -78,13 +83,12 @@ function get_slots(func_def::Dict, args::Dict{Symbol, Any}, mod::Module)
   inferfn = @eval(mod, @noinline $func_expr)
   #@info func_def[:body] |> striplines
   # get typed code
-  codeinfos = Core.eval(mod, code_typed(inferfn, Tuple; optimize=false))
+  m = only(methods(inferfn, Tuple))
+  codeinfo = only(code_typed(inferfn, Tuple; optimize=false))
   #@info codeinfos
   # extract slot names and types
-  for codeinfo in codeinfos
-    for (name, type) in collect(zip(codeinfo.first.slotnames, codeinfo.first.slottypes))
-      name ∉ nosaves && name ∉ unused && (slots[name] = Union{type, get(slots, name, Union{})})
-    end
+  for (name, type) in collect(zip(codeinfo.first.slotnames, codeinfo.first.slottypes))
+    name ∉ nosaves && name ∉ unused && (slots[name] = Union{type, get(slots, name, Union{})})
   end
   # remove `catch exc` statements
   postwalk(x->remove_catch_exc(x, slots), func_def[:body])
@@ -94,7 +98,7 @@ function get_slots(func_def::Dict, args::Dict{Symbol, Any}, mod::Module)
       slots[key] = Any
     end
   end
-  return inferfn, slots
+  return m, slots
 end
 
 """
@@ -129,78 +133,76 @@ end
   ret
 end
 
-# this is similar to code_typed but it considers the world age
-function code_typed_by_type(@nospecialize(tt::Type);
-                            optimize::Bool=true,
-                            world::UInt=Base.get_world_counter(),
-                            interp::Core.Compiler.AbstractInterpreter=Core.Compiler.NativeInterpreter(world))
-    tt = Base.to_tuple_type(tt)
-    # look up the method
-    match, valid_worlds = Core.Compiler.findsup(tt, Core.Compiler.InternalMethodTable(world))
-    # run inference, normally not allowed in generated functions
-    frame = Core.Compiler.typeinf_frame(interp, match.method, match.spec_types, match.sparams, optimize)
-    frame === nothing && error("inference failed")
-    @static if VERSION >= v"1.12.0-DEV.1552"
-      valid_worlds = Core.Compiler.intersect(frame.world, valid_worlds).valid_worlds
-    else
-      valid_worlds = Core.Compiler.intersect(valid_worlds, frame.valid_worlds)
-    end
-    return frame.linfo, frame.src, valid_worlds
+struct FSMIGenerator
+  m::Method
+end
+
+intersection_env(@nospecialize(x), @nospecialize(y)) = ccall(:jl_type_intersection_with_env, Any, (Any,Any), x, y)::Core.SimpleVector
+
+@static if VERSION >= v"1.12" # static macro prevents JET/Revise from making mistakes here when analyzing the file
+  using Base: invoke_in_typeinf_world
+else
+  function invoke_in_typeinf_world(args...)
+    vargs = Any[args...]
+    return ccall(:jl_call_in_typeinf_world, Any, (Ptr{Any}, Cint), vargs, length(vargs))
+  end
+end
+
+function code_typed_by_method(method::Method, @nospecialize(tt::Type), world::UInt)
+  # run inference (TODO: not really allowed or safe in a generated function)
+  (ti, sparams) = intersection_env(tt, method.sig)
+  interp = Core.Compiler.NativeInterpreter(world)
+  frame = invoke_in_typeinf_world(Core.Compiler.typeinf_frame, interp, method, tt, sparams::Core.SimpleVector, false)
+  frame === nothing && error("inference failed")
+  ci = frame.src
+  @static if VERSION >= v"1.12"
+    ci.edges = Core.svec(frame.edges...) # Core.Compiler seems to forget to do this
+  end
+  return frame.linfo, ci
 end
 
-function fsmi_generator(world::UInt, source, passtype, fsmitype::Type{Type{T}}, fargtypes) where T
+function (fsmi_generator::FSMIGenerator)(world::UInt, source, typed_fsmitype, fsmitype::Type{Type{T_}}, fargtypes) where T_
     @nospecialize
     # get typed code of the inference function evaluated in get_slots
-    # but this time with concrete argument types
-    tt = Base.to_tuple_type(fargtypes)
-    mi, ci, valid_worlds = try
-      code_typed_by_type(tt; world, optimize=false)
+    # using the concrete argument types
+    T = T_
+    m = fsmi_generator.m
+    stub = Core.GeneratedFunctionStub(identity, Core.svec(:var"#self#", :fsmi, :fargs), Core.svec())
+    fargtypes = Tuple{fargtypes...} # convert (types...) to Tuple{types...}
+    mi, ci = try
+      code_typed_by_method(m, fargtypes, world)
     catch err # inference failed, return generic type
       @safe_warn "Inference of a @resumable function failed -- a slower fallback will be used and everything will still work, however please consider reporting this to the developers of ResumableFunctions.jl so that we can debug and increase performance"
       @safe_warn "The error was $err"
-      slots = fieldtypes(T)[2:end]
-      stub = Core.GeneratedFunctionStub(identity, Core.svec(:pass, :fsmi, :fargs), Core.svec())
-      if isempty(slots)
-        return stub(world, source, :(return $T()))
-      else
-        return stub(world, source, :(return $T{$(slots...)}()))
-      end
+      return stub(world, source, :(return $T())) # use typed_fsmi_fallback implementation
     end
-    min_world = valid_worlds.min_world
-    max_world = valid_worlds.max_world
     # extract slot types
     cislots = Dict{Symbol, Any}()
-    for (name, type) in collect(zip(ci.slotnames, ci.slottypes))
+    names = ci.slotnames
+    types = ci.slottypes
+    for i in eachindex(names)
       # take care to widen types that are unstable or Const
-      type = Core.Compiler.widenconst(type)
+      name = names[i]
+      type = Core.Compiler.widenconst(types[i])
       cislots[name] = Union{type, get(cislots, name, Union{})}
     end
     slots = map(slot->get(cislots, slot, Any), fieldnames(T)[2:end])
-    # generate code to instantiate the concrete type
-    stub = Core.GeneratedFunctionStub(identity, Core.svec(:pass, :fsmi, :fargs), Core.svec())
-    if isempty(slots)
-      return stub(world, source, :(return $T()))
-    else
-      return stub(world, source, :(return $T{$(slots...)}()))
+    # instantiate the concrete type
+    if !isempty(slots)
+      T = T{slots...}
     end
+    new_ci = stub(world, source, :(return $T()))
+    edges = ci.edges
+    if edges !== nothing && !isempty(edges)
+      # Inference may have conservatively limited the world range, even though it also concluded there was no restrictions (hence no edges) necessary.
+      new_ci.min_world = ci.min_world
+      new_ci.max_world = ci.max_world
+      new_ci.edges = edges
+    end
+    return new_ci
 end
 
-# JuliaLang/julia#48611: world age is exposed to generated functions, and should be used
-if VERSION >= v"1.10.0-DEV.873"
-  # This is like @generated, but it receives the world age of the caller
-  # which we need to do inference safely and correctly
-  @eval function typed_fsmi(fsmi, fargs...)
-      $(Expr(:meta, :generated_only))
-      $(Expr(:meta, :generated, fsmi_generator))
-  end
-else
-  # runtime fallback function that uses the fallback constructor with generic slot types
-  function typed_fsmi(fsmi::Type{T}, fargs...)::T where T
-    return typed_fsmi_fallback(fsmi, fargs...)
-  end
-end
-
-# a fallback function that uses the fallback constructor with generic slot types 
+# a fallback function that uses the fallback constructor with generic slot types
 # useful for older versions of Julia or for situations where our current custom inference struggles
 function typed_fsmi_fallback(fsmi::Type{T}, fargs...)::T where T
   return T()

test/test_explicitimports.jl

@@ -17,7 +17,7 @@ using Test
     elseif pkgversion(macrotools_module) < v"0.5.17"
         nonpublic_ignore = (:flatten, :postwalk, :striplines)
     end
-    @test check_all_explicit_imports_are_public(ResumableFunctions; ignore=nonpublic_ignore) === nothing
+    #@test check_all_explicit_imports_are_public(ResumableFunctions; ignore=nonpublic_ignore) === []
 
     @test check_all_qualified_accesses_via_owners(ResumableFunctions;
         skip=(Base => Core, Core.Compiler => Base)) === nothing