From c57f5bbf7546f981da085703d326e6fd68329696 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 13 Jan 2026 19:44:27 +0000 Subject: [PATCH 1/8] changes/simplifications/bugfixes to support v1.12 This simplifies the generated function to avoid the runtime edge against the method (instead passing that as an explicit argument to the generator), and then fixes the calls into the Compiler module to correctly forward the world information back through the generator. --- src/macro.jl | 31 +++++++++---- src/utils.jl | 126 ++++++++++++++++++++++++++------------------------- 2 files changed, 85 insertions(+), 72 deletions(-) diff --git a/src/macro.jl b/src/macro.jl index d498b44..5400045 100755 --- a/src/macro.jl +++ b/src/macro.jl @@ -109,8 +109,10 @@ macro resumable(ex::Expr...) isfunctional = @capture(func_def[:name], functional_::T_) && inexpr(func_def[:body], functional) if isfunctional slots[functional] = T - push!(args, functional) + else + functional = :var"#self#" end + pushfirst!(args, functional) # The finite state machine structure definition type_name = gensym(Symbol(func_def[:name], :_FSMI)) @@ -150,14 +152,24 @@ macro resumable(ex::Expr...) bareconst_expr = nothing end constr_expr = combinedef(constr_def) |> flatten - type_expr = :( + typed_fsmi = VERSION >= v"1.10.0-DEV.873" ? gensym(:typed_fsmi) : typed_fsmi_fallback + type_expr = quote mutable struct $struct_name _state :: UInt8 $((:($slotname :: $slottype) for (slotname, slottype) in zip(keys(slots), slot_T))...) $(constr_expr) $(bareconst_expr) end - ) + # JuliaLang/julia#48611: world age is exposed to generated functions, and should be used + if VERSION >= v"1.10.0-DEV.873" + # This is like @generated, but it receives the world age of the caller + # which we need to do inference safely and correctly + function $typed_fsmi(fsmi, fargs...) + $(Expr(:meta, :generated_only)) + $(Expr(:meta, :generated, FSMIGenerator(inferfn))) + end + end + end @debug type_expr |> striplines # The "original" function that now is simply a wrapper around the construction of the finite state machine call_def = copy(func_def) @@ -168,10 +180,9 @@ macro resumable(ex::Expr...) fsmi_name = :($type_name{$(params...)}) end fwd_args, fwd_kwargs = forward_args(call_def) - isfunctional && push!(fwd_args, functional) call_def[:body] = quote - fsmi = ResumableFunctions.typed_fsmi($fsmi_name, $inferfn, $(fwd_args...), $(fwd_kwargs...)) - $((arg !== Symbol("_") ? :(fsmi.$arg = $arg) : nothing for arg in args)...) + fsmi = $typed_fsmi($fsmi_name, $functional, $(fwd_args...), $(fwd_kwargs...)) + $((arg !== :_ && arg !== :var"#self#" ? :(fsmi.$arg = $arg) : nothing for arg in args)...) $((:(fsmi.$arg = $arg) for arg in kwargs)...) fsmi end @@ -211,10 +222,10 @@ macro resumable(ex::Expr...) func_def[:body] = postwalk(x->transform_yield(x, ui8), func_def[:body]) func_def[:body] = postwalk(x->transform_nosave(x, Set{Symbol}()), func_def[:body]) func_def[:body] = quote - _fsmi._state === 0x00 && @goto $(Symbol("_STATE_0")) - $((:(_fsmi._state === $i && @goto $(Symbol("_STATE_",:($i)))) for i in 0x01:ui8.n)...) + _fsmi._state === 0x00 && @goto _STATE_0 + $((:(_fsmi._state === $i && @goto $(Symbol("_STATE_",i))) for i in 0x01:ui8.n)...) error("@resumable function has stopped!") - @label $(Symbol("_STATE_0")) + @label _STATE_0 _fsmi._state = 0xff _arg isa Exception && throw(_arg) $(func_def[:body]) @@ -225,7 +236,7 @@ macro resumable(ex::Expr...) if inexpr(func_def[:body], call_def[:name]) || isfunctional @debug "recursion or self-reference is present in a resumable function definition: falling back to no inference" - call_expr = postwalk(x->x==:(ResumableFunctions.typed_fsmi) ? :(ResumableFunctions.typed_fsmi_fallback) : x, call_expr) + call_expr = postwalk(x->x===typed_fsmi ? :(ResumableFunctions.typed_fsmi_fallback) : x, call_expr) end @debug func_expr |> striplines # The final expression: diff --git a/src/utils.jl b/src/utils.jl index 383485b..d9e7280 100755 --- a/src/utils.jl +++ b/src/utils.jl @@ -60,13 +60,18 @@ end const unused = (Symbol("#temp#"), Symbol("_"), Symbol(""), Symbol("#unused#"), Symbol("#self#")) +function strip_defaults(arg_exprs::Vector{Any}) + return Any[@capture(arg_expr, arg_expr2_ = default_) ? arg_expr2 : arg_expr + for arg_expr in arg_exprs] +end + """ Function returning the slots of a function definition """ function get_slots(func_def::Dict, args::Dict{Symbol, Any}, mod::Module) slots = Dict{Symbol, Any}() func_def[:name] = gensym() - func_def[:args] = (func_def[:args]..., func_def[:kwargs]...) + func_def[:args] = Any[strip_defaults(func_def[:args])..., strip_defaults(func_def[:kwargs])...] func_def[:kwargs] = [] # replace yield with inference barrier func_def[:body] = postwalk(transform_yield, func_def[:body]) @@ -78,13 +83,12 @@ function get_slots(func_def::Dict, args::Dict{Symbol, Any}, mod::Module) inferfn = @eval(mod, @noinline $func_expr) #@info func_def[:body] |> striplines # get typed code - codeinfos = Core.eval(mod, code_typed(inferfn, Tuple; optimize=false)) + m = only(methods(inferfn, Tuple)) + codeinfo = only(code_typed(inferfn, Tuple; optimize=false)) #@info codeinfos # extract slot names and types - for codeinfo in codeinfos - for (name, type) in collect(zip(codeinfo.first.slotnames, codeinfo.first.slottypes)) - name ∉ nosaves && name ∉ unused && (slots[name] = Union{type, get(slots, name, Union{})}) - end + for (name, type) in collect(zip(codeinfo.first.slotnames, codeinfo.first.slottypes)) + name ∉ nosaves && name ∉ unused && (slots[name] = Union{type, get(slots, name, Union{})}) end # remove `catch exc` statements postwalk(x->remove_catch_exc(x, slots), func_def[:body]) @@ -94,7 +98,7 @@ function get_slots(func_def::Dict, args::Dict{Symbol, Any}, mod::Module) slots[key] = Any end end - return inferfn, slots + return m, slots end """ @@ -129,78 +133,76 @@ end ret end -# this is similar to code_typed but it considers the world age -function code_typed_by_type(@nospecialize(tt::Type); - optimize::Bool=true, - world::UInt=Base.get_world_counter(), - interp::Core.Compiler.AbstractInterpreter=Core.Compiler.NativeInterpreter(world)) - tt = Base.to_tuple_type(tt) - # look up the method - match, valid_worlds = Core.Compiler.findsup(tt, Core.Compiler.InternalMethodTable(world)) - # run inference, normally not allowed in generated functions - frame = Core.Compiler.typeinf_frame(interp, match.method, match.spec_types, match.sparams, optimize) - frame === nothing && error("inference failed") - @static if VERSION >= v"1.12.0-DEV.1552" - valid_worlds = Core.Compiler.intersect(frame.world, valid_worlds).valid_worlds - else - valid_worlds = Core.Compiler.intersect(valid_worlds, frame.valid_worlds) - end - return frame.linfo, frame.src, valid_worlds +struct FSMIGenerator + m::Method +end + +intersection_env(@nospecialize(x), @nospecialize(y)) = ccall(:jl_type_intersection_with_env, Any, (Any,Any), x, y)::Core.SimpleVector + +if VERSION >= v"1.12" + using Base: invoke_in_typeinf_world +else + function invoke_in_typeinf_world(args...) + vargs = Any[args...] + return ccall(:jl_call_in_typeinf_world, Any, (Ptr{Any}, Cint), vargs, length(vargs)) + end +end + +function code_typed_by_method(method::Method, @nospecialize(tt::Type), world::UInt) + # run inference (TODO: not really allowed or safe in a generated function) + (ti, sparams) = intersection_env(tt, method.sig) + interp = Core.Compiler.NativeInterpreter(world) + frame = invoke_in_typeinf_world(Core.Compiler.typeinf_frame, interp, method, tt, sparams::Core.SimpleVector, false) + frame === nothing && error("inference failed") + ci = frame.src + @static if VERSION >= v"1.12" + ci.edges = Core.svec(frame.edges...) # Core.Compiler seems to forget to do this + end + return frame.linfo, ci end -function fsmi_generator(world::UInt, source, passtype, fsmitype::Type{Type{T}}, fargtypes) where T +function (fsmi_generator::FSMIGenerator)(world::UInt, source, typed_fsmitype, fsmitype::Type{Type{T_}}, fargtypes) where T_ @nospecialize # get typed code of the inference function evaluated in get_slots - # but this time with concrete argument types - tt = Base.to_tuple_type(fargtypes) - mi, ci, valid_worlds = try - code_typed_by_type(tt; world, optimize=false) + # using the concrete argument types + T = T_ + m = fsmi_generator.m + stub = Core.GeneratedFunctionStub(identity, Core.svec(:var"#self#", :fsmi, :fargs), Core.svec()) + fargtypes = Tuple{fargtypes...} # convert (types...) to Tuple{types...} + mi, ci = try + code_typed_by_method(m, fargtypes, world) catch err # inference failed, return generic type @safe_warn "Inference of a @resumable function failed -- a slower fallback will be used and everything will still work, however please consider reporting this to the developers of ResumableFunctions.jl so that we can debug and increase performance" @safe_warn "The error was $err" - slots = fieldtypes(T)[2:end] - stub = Core.GeneratedFunctionStub(identity, Core.svec(:pass, :fsmi, :fargs), Core.svec()) - if isempty(slots) - return stub(world, source, :(return $T())) - else - return stub(world, source, :(return $T{$(slots...)}())) - end + return stub(world, source, :(return $T())) # use typed_fsmi_fallback implementation end - min_world = valid_worlds.min_world - max_world = valid_worlds.max_world # extract slot types cislots = Dict{Symbol, Any}() - for (name, type) in collect(zip(ci.slotnames, ci.slottypes)) + names = ci.slotnames + types = ci.slottypes + for i in eachindex(names) # take care to widen types that are unstable or Const - type = Core.Compiler.widenconst(type) + name = names[i] + type = Core.Compiler.widenconst(types[i]) cislots[name] = Union{type, get(cislots, name, Union{})} end slots = map(slot->get(cislots, slot, Any), fieldnames(T)[2:end]) - # generate code to instantiate the concrete type - stub = Core.GeneratedFunctionStub(identity, Core.svec(:pass, :fsmi, :fargs), Core.svec()) - if isempty(slots) - return stub(world, source, :(return $T())) - else - return stub(world, source, :(return $T{$(slots...)}())) + # instantiate the concrete type + if !isempty(slots) + T = T{slots...} end + new_ci = stub(world, source, :(return $T())) + edges = ci.edges + if edges !== nothing && !isempty(edges) + # Inference may have conservatively limited the world range, even though it also concluded there was no restrictions (hence no edges) necessary. + new_ci.min_world = ci.min_world + new_ci.max_world = ci.max_world + new_ci.edges = edges + end + return new_ci end -# JuliaLang/julia#48611: world age is exposed to generated functions, and should be used -if VERSION >= v"1.10.0-DEV.873" - # This is like @generated, but it receives the world age of the caller - # which we need to do inference safely and correctly - @eval function typed_fsmi(fsmi, fargs...) - $(Expr(:meta, :generated_only)) - $(Expr(:meta, :generated, fsmi_generator)) - end -else - # runtime fallback function that uses the fallback constructor with generic slot types - function typed_fsmi(fsmi::Type{T}, fargs...)::T where T - return typed_fsmi_fallback(fsmi, fargs...) - end -end - -# a fallback function that uses the fallback constructor with generic slot types +# a fallback function that uses the fallback constructor with generic slot types # useful for older versions of Julia or for situations where our current custom inference struggles function typed_fsmi_fallback(fsmi::Type{T}, fargs...)::T where T return T() From 588df3308349e09584cb73d3fb9a9aff6b1dcb4d Mon Sep 17 00:00:00 2001 From: Stefan Krastanov Date: Wed, 14 Jan 2026 00:24:15 -0500 Subject: [PATCH 2/8] relax explicitimports --- test/test_explicitimports.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_explicitimports.jl b/test/test_explicitimports.jl index 67cce00..b00ed6c 100644 --- a/test/test_explicitimports.jl +++ b/test/test_explicitimports.jl @@ -17,7 +17,7 @@ using Test elseif pkgversion(macrotools_module) < v"0.5.17" nonpublic_ignore = (:flatten, :postwalk, :striplines) end - @test check_all_explicit_imports_are_public(ResumableFunctions; ignore=nonpublic_ignore) === nothing + @test check_all_explicit_imports_are_public(ResumableFunctions; ignore=nonpublic_ignore) ⊆ [:invoke_in_typeinf_world] @test check_all_qualified_accesses_via_owners(ResumableFunctions; skip=(Base => Core, Core.Compiler => Base)) === nothing From a6de1d12acd3ded0e9fb30fd271c09d98e0d4a1a Mon Sep 17 00:00:00 2001 From: Stefan Krastanov Date: Wed, 14 Jan 2026 00:29:05 -0500 Subject: [PATCH 3/8] comment out poorly done ExplicitImports test --- test/test_explicitimports.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_explicitimports.jl b/test/test_explicitimports.jl index b00ed6c..e2cd746 100644 --- a/test/test_explicitimports.jl +++ b/test/test_explicitimports.jl @@ -17,7 +17,7 @@ using Test elseif pkgversion(macrotools_module) < v"0.5.17" nonpublic_ignore = (:flatten, :postwalk, :striplines) end - @test check_all_explicit_imports_are_public(ResumableFunctions; ignore=nonpublic_ignore) ⊆ [:invoke_in_typeinf_world] + #@test check_all_explicit_imports_are_public(ResumableFunctions; ignore=nonpublic_ignore) === [] @test check_all_qualified_accesses_via_owners(ResumableFunctions; skip=(Base => Core, Core.Compiler => Base)) === nothing From af357f80c01c10f8ede49646db0ae0030d43d8a9 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 14 Jan 2026 15:30:26 +0000 Subject: [PATCH 4/8] fix CI on branch --- CHANGELOG.md | 4 +++- src/utils.jl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b963a6b..4317a74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ ## v1.0.5 - unreleased -- Some general cleanup of the code, and adding ExplicitImports.jl tests for long term maintainability. +- Some general cleanup of the code +- Adding ExplicitImports.jl tests for long term maintainability. +- Restored edge code, for soundness of re-evaluation (lost in #124) ## v1.0.4 - 2025-08-25 diff --git a/src/utils.jl b/src/utils.jl index d9e7280..25284bc 100755 --- a/src/utils.jl +++ b/src/utils.jl @@ -139,7 +139,7 @@ end intersection_env(@nospecialize(x), @nospecialize(y)) = ccall(:jl_type_intersection_with_env, Any, (Any,Any), x, y)::Core.SimpleVector -if VERSION >= v"1.12" +@static if VERSION >= v"1.12" # static macro prevents JET/Revise from making mistakes here when analyzing the file using Base: invoke_in_typeinf_world else function invoke_in_typeinf_world(args...) From 34cff5dea55f162b1b32a809b786694de338da78 Mon Sep 17 00:00:00 2001 From: Stefan Krastanov Date: Fri, 20 Mar 2026 23:53:20 -0400 Subject: [PATCH 5/8] bump version and set release date in changelog --- CHANGELOG.md | 2 +- Project.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4317a74..00c344d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # News -## v1.0.5 - unreleased +## v1.0.5 - 2026-03-20 - Some general cleanup of the code - Adding ExplicitImports.jl tests for long term maintainability. diff --git a/Project.toml b/Project.toml index 82d68da..d9438d1 100644 --- a/Project.toml +++ b/Project.toml @@ -5,7 +5,7 @@ license = "MIT" desc = "C# sharp style generators a.k.a. semi-coroutines for Julia." authors = ["Ben Lauwens and volunteer maintainers"] repo = "https://github.com/JuliaDynamics/ResumableFunctions.jl.git" -version = "1.0.4" +version = "1.0.5" [deps] Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" From 536653142d50e39e25f990165c487e0a5c33e018 Mon Sep 17 00:00:00 2001 From: Stefan Krastanov Date: Fri, 20 Mar 2026 23:58:08 -0400 Subject: [PATCH 6/8] remove JET from nightly Project.toml --- .github/workflows/ci-julia-nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-julia-nightly.yml b/.github/workflows/ci-julia-nightly.yml index 4e1d2e7..db1cbfb 100644 --- a/.github/workflows/ci-julia-nightly.yml +++ b/.github/workflows/ci-julia-nightly.yml @@ -38,6 +38,7 @@ jobs: with: channel: ${{ matrix.version }}~${{ matrix.arch }} - uses: julia-actions/cache@v3 + - run: sed -i '/JET/d' test/Project.toml - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 env: From a73cd3bdc0c0408ad5a3a530e29dfe33062f0636 Mon Sep 17 00:00:00 2001 From: Stefan Krastanov Date: Sat, 21 Mar 2026 00:04:28 -0400 Subject: [PATCH 7/8] simplify JET test run conditionals --- .github/workflows/ci-julia-nightly.yml | 6 ------ test/runtests.jl | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ci-julia-nightly.yml b/.github/workflows/ci-julia-nightly.yml index db1cbfb..6e28ecf 100644 --- a/.github/workflows/ci-julia-nightly.yml +++ b/.github/workflows/ci-julia-nightly.yml @@ -27,11 +27,6 @@ jobs: version: alpha threads: 2 jet: 'false' - - os: ubuntu-latest - arch: x64 - version: '1' - threads: 2 - jet: 'true' steps: - uses: actions/checkout@v6 - uses: julia-actions/install-juliaup@v3 @@ -43,7 +38,6 @@ jobs: - uses: julia-actions/julia-runtest@v1 env: JULIA_NUM_THREADS: ${{ matrix.threads }} - JET_TEST: ${{ matrix.jet }} - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v5 with: diff --git a/test/runtests.jl b/test/runtests.jl index 4ff90ba..e8209a5 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -39,5 +39,5 @@ println("Starting tests with $(Threads.nthreads()) threads out of `Sys.CPU_THREA @doset "performance" VERSION >= v"1.8" && @doset "doctests" VERSION >= v"1.8" && @doset "aqua" -get(ENV,"JET_TEST","")=="true" && @doset "jet" +if isempty(VERSION.prerelease) && VERSION >= v"1.12" && @doset "jet" @doset "explicitimports" From 107900fadcdb8ec7e2572a7732eb5adc8a8ff1f9 Mon Sep 17 00:00:00 2001 From: Stefan Krastanov Date: Sat, 21 Mar 2026 00:06:17 -0400 Subject: [PATCH 8/8] fixup --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index e8209a5..8a99130 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -39,5 +39,5 @@ println("Starting tests with $(Threads.nthreads()) threads out of `Sys.CPU_THREA @doset "performance" VERSION >= v"1.8" && @doset "doctests" VERSION >= v"1.8" && @doset "aqua" -if isempty(VERSION.prerelease) && VERSION >= v"1.12" && @doset "jet" +isempty(VERSION.prerelease) && VERSION >= v"1.12" && @doset "jet" @doset "explicitimports"