Puzzling failure to set the value of a symbol

[timholy]

Looking into timholy/Revise.jl#429, I applied the following patch to JuliaInterpreter (the key part is the first block of changes to do_assignment!, the rest is just me being paranoid):

diff --git a/src/interpret.jl b/src/interpret.jl
index 1700ae6..4e446d2 100644
--- a/src/interpret.jl
+++ b/src/interpret.jl
@@ -349,7 +349,13 @@ function do_assignment!(frame, @nospecialize(lhs), @nospecialize(rhs))
     elseif isa(lhs, GlobalRef)
         Core.eval(lhs.mod, :($(lhs.name) = $(QuoteNode(rhs))))
     elseif isa(lhs, Symbol)
-        Core.eval(moduleof(code), :($lhs = $(QuoteNode(rhs))))
+        @show lhs rhs typeof(rhs) moduleof(code)
+        dump(lhs)
+        dump(rhs)
+        newex = :($lhs = $(QuoteNode(rhs)))
+        @show newex
+        ret = Core.eval(moduleof(code), :($lhs = $(QuoteNode(rhs))))
+        @show ret isdefined(moduleof(code), lhs)
     end
 end
 
@@ -435,7 +441,7 @@ function step_expr!(@nospecialize(recurse), frame, @nospecialize(node), istoplev
     # For profiling:
     # location_key = (scopeof(frame), pc)
     # _location[location_key] = get(_location, location_key, 0) + 1
-    try
+    # try
         if isa(node, Expr)
             if node.head == :(=)
                 lhs, rhs = node.args
@@ -532,13 +538,14 @@ function step_expr!(@nospecialize(recurse), frame, @nospecialize(node), istoplev
             # FIXME: undefine the slot?
         elseif istoplevel && isa(node, LineNumberNode)
         elseif istoplevel && isa(node, Symbol)
+            @show moduleof(frame) node
             rhs = getfield(moduleof(frame), node)
         else
             rhs = @lookup(frame, node)
         end
-    catch err
-        return handle_err(recurse, frame, err)
-    end
+    # catch err
+    #     return handle_err(recurse, frame, err)
+    # end
     @isdefined(rhs) && isa(rhs, BreakpointRef) && return rhs
     if isassign(frame, pc)
         if !@isdefined(rhs)

Then I run the example (I named the module Manu) and get this:

julia> 1+1   # trigger a revision
moduleof(frame) = Manu
node = Symbol("#coinflip#11#16")
moduleof(frame) = Manu
node = Symbol("##inner_function#446#17")
moduleof(frame) = Manu
node = Symbol("##inner_function#446#17")
moduleof(frame) = Manu
node = Symbol("##inner_function#446#17")
moduleof(frame) = Manu
node = Symbol("#coinflip#15")
lhs = Symbol("#s114")
rhs = #s114
typeof(rhs) = TypeVar
moduleof(code) = Manu
Symbol #s114
TypeVar
  name: Symbol #s114
  lb: Core.TypeofBottom Union{}
  ub: Any
newex = :(var"#s114" = $(QuoteNode(#s114)))
ret = #s114
isdefined(moduleof(code), lhs) = false          # HERE'S THE BIZARRE BIT. WHY IS THIS FALSE?
moduleof(frame) = Manu
node = Symbol("#s114")
┌ Error: evaluation error starting at /home/tim/.julia/dev/Revise/test/Manu.jl:8
│   mod = Manu
│   ex =
│    quote
│        #= /home/tim/.julia/dev/Revise/test/Manu.jl:8 =#
│        function test_fun()
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:10 =#
│            p_true = 0.5
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:13 =#
│            Ns = 0:100
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:16 =#
│            Random.seed!(12)
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:17 =#
│            data = rand(Bernoulli(p_true), last(Ns))
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:21 =#
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:21 =# @model (coinflip(y, ::Type{T}=Float64) where T) = begin
│                        #= /home/tim/.julia/dev/Revise/test/Manu.jl:21 =#
│                        #= /home/tim/.julia/dev/Revise/test/Manu.jl:23 =#
│                        p ~ Beta(1, 1)
│                        #= /home/tim/.julia/dev/Revise/test/Manu.jl:26 =#
│                        N = length(y)
│                        #= /home/tim/.julia/dev/Revise/test/Manu.jl:27 =#
│                        for n = 1:N
│                            #= /home/tim/.julia/dev/Revise/test/Manu.jl:29 =#
│                            y[n] ~ Bernoulli(p)
│                        end
│                    end
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:34 =#
│            iterations = 1000
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:35 =#
│            ϵ = 0.05
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:36 =#
│            τ = 10
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:39 =#
│            chain = sample(coinflip(data), HMC(ϵ, τ), iterations)
│            #= /home/tim/.julia/dev/Revise/test/Manu.jl:40 =#
│            return chain
│        end
│    end
│   exception =
│    UndefVarError: #s114 not defined
│    Stacktrace:
│     [1] step_expr!(::Any, ::JuliaInterpreter.Frame, ::Any, ::Bool) at /home/tim/.julia/dev/JuliaInterpreter/src/interpret.jl:542
└ @ Revise ~/.julia/packages/Revise/ZOWOa/src/lowered.jl:105
2

In case you missed it, the strange part is the isdefined(moduleof(code), lhs) = false; why, right after Core.evaling a symbol assignment, is that symbol not defined?

Just to make sure I'm not crazy, with the same patch applied:

julia> using JuliaInterpreter

julia> frame = JuliaInterpreter.prepare_thunk(Main, :(var"#s90" = TypeVar(Symbol("#s90"), Any)))
Frame for Main
  1 0  1 ─ %1 = Symbol("#s90")
  2 0  │   %2 = TypeVar(%1, Any)
  3 0  │        var"#s90" = %2
⋮

julia> JuliaInterpreter.finish!(frame, true)
lhs = Symbol("#s90")
rhs = #s90
typeof(rhs) = TypeVar
moduleof(code) = Main
Symbol #s90
TypeVar
  name: Symbol #s90
  lb: Core.TypeofBottom Union{}
  ub: Any
newex = :(var"#s90" = $(QuoteNode(#s90)))
ret = #s90
isdefined(moduleof(code), lhs) = true

I confess extreme bafflement. Any idea what's wrong? Note this warning:

[ Info: Precompiling Manu [top-level]
WARNING: Method definition (::getfield(Manu, Symbol("#coinflip#5")){#coinflip#1})() in module Manu at /home/tim/.julia/packages/Turing/xFzfF/src/core/compiler.jl:406 overwritten at /home/tim/.julia/packages/Turing/xFzfF/src/core/compiler.jl:412.
  ** incremental compilation may be fatally broken for this module **

I wonder if Turing.jl is breaking Julia?

Anyway, I'd really appreciate feedback, even if it's "That's weird. No clue."

[timholy]

Here's a MWE: call this /tmp/lv.jl

function _test(x,y)
    function f!(::Type{<:Array{<:Union{Base.HWReal, Bool}}}, ℛ::AbstractArray{𝒯}) where 𝒯
        nothing
    end
end

and with

diff --git a/src/interpret.jl b/src/interpret.jl
index f1ae283..8f54f9d 100644
--- a/src/interpret.jl
+++ b/src/interpret.jl
@@ -349,7 +349,13 @@ function do_assignment!(frame, @nospecialize(lhs), @nospecialize(rhs))
     elseif isa(lhs, GlobalRef)
         Core.eval(lhs.mod, :($(lhs.name) = $(QuoteNode(rhs))))
     elseif isa(lhs, Symbol)
+        println("assignment: ", :($lhs = $(QuoteNode(rhs))))
+        println("in module ", moduleof(code))
+        @show typeof(rhs)
         Core.eval(moduleof(code), :($lhs = $(QuoteNode(rhs))))
+        # @show getfield(moduleof(code), lhs)   # did it work?
+        print("check that it worked: ")
+        println(getfield(moduleof(code), lhs))
     end
 end

then

[ Info: Precompiling JuliaInterpreter [aa1ae85d-cabe-5617-a682-6adf51b2e16a]

julia> ex = Base.parse_input_line(read("/tmp/lv.jl", String)).args[end];

julia> frame = Frame(Main, ex)
Frame for Main
   1 0  1 ─       $(Expr(:thunk, CodeInfo(
    @ none within `top-level scope'
1 ─     return $(Expr(:method, :_test))
)))
   2 0  │         $(Expr(:method, :_test))
   3 0  │         $(Expr(:thunk, CodeInfo(
    @ none within `top-level scope'
1 ─      global var"#f!#1"
│        const var"#f!#1"
│   %3 = Core._structtype(Main, Symbol("#f!#1"), Core.svec(), Core.svec(), false, 0)
│        var"#f!#1" = %3
│        Core._setsuper!(var"#f!#1", Core.Function)
│        Core._typebody!(var"#f!#1", Core.svec())
└──      return nothing
)))
⋮

julia> debug_command(frame, :c, true)
assignment: var"#f!#1" = $(QuoteNode(Main.var"#f!#1"))
in module Main
typeof(rhs) = DataType
check that it worked: 
signal (11): Segmentation fault
in expression starting at REPL[4]:1
mtcache_hash_lookup at /home/tim/src/julia-master/src/typemap.c:287 [inlined]
jl_typemap_intersection_visitor at /home/tim/src/julia-master/src/typemap.c:564
jl_typemap_intersection_visitor at /home/tim/src/julia-master/src/typemap.c:598
jl_typemap_intersection_visitor at /home/tim/src/julia-master/src/typemap.c:542
ml_matches at /home/tim/src/julia-master/src/gf.c:2677
_gf_invoke_lookup at /home/tim/src/julia-master/src/gf.c:2357 [inlined]
jl_mt_assoc_by_type at /home/tim/src/julia-master/src/gf.c:1191
jl_lookup_generic_ at /home/tim/src/julia-master/src/gf.c:2316 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2341
print at ./strings/io.jl:35
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
print at ./strings/io.jl:46
println at ./strings/io.jl:73
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
println at ./coreio.jl:4
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
do_assignment! at /home/tim/.julia/dev/JuliaInterpreter/src/interpret.jl:358
step_expr! at /home/tim/.julia/dev/JuliaInterpreter/src/interpret.jl:456
step_expr! at /home/tim/.julia/dev/JuliaInterpreter/src/interpret.jl:588
finish! at /home/tim/.julia/dev/JuliaInterpreter/src/commands.jl:14
step_expr! at /home/tim/.julia/dev/JuliaInterpreter/src/interpret.jl:507
step_expr! at /home/tim/.julia/dev/JuliaInterpreter/src/interpret.jl:588
finish! at /home/tim/.julia/dev/JuliaInterpreter/src/commands.jl:14
finish_and_return! at /home/tim/.julia/dev/JuliaInterpreter/src/commands.jl:29
unknown function (ip: 0x7f6c7979b53d)
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
finish_stack! at /home/tim/.julia/dev/JuliaInterpreter/src/commands.jl:59
unknown function (ip: 0x7f6c7979b4bd)
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
#debug_command#79 at /home/tim/.julia/dev/JuliaInterpreter/src/commands.jl:473
unknown function (ip: 0x7f6c7979ad93)
debug_command at /home/tim/.julia/dev/JuliaInterpreter/src/commands.jl:415 [inlined]
#debug_command#81 at /home/tim/.julia/dev/JuliaInterpreter/src/commands.jl:487 [inlined]
debug_command at /home/tim/.julia/dev/JuliaInterpreter/src/commands.jl:487
unknown function (ip: 0x7f6c797790fd)
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
jl_apply at /home/tim/src/julia-master/src/julia.h:1682 [inlined]
do_call at /home/tim/src/julia-master/src/interpreter.c:115
eval_value at /home/tim/src/julia-master/src/interpreter.c:204
eval_stmt_value at /home/tim/src/julia-master/src/interpreter.c:155 [inlined]
eval_body at /home/tim/src/julia-master/src/interpreter.c:575
jl_interpret_toplevel_thunk at /home/tim/src/julia-master/src/interpreter.c:669
jl_toplevel_eval_flex at /home/tim/src/julia-master/src/toplevel.c:837
jl_toplevel_eval_flex at /home/tim/src/julia-master/src/toplevel.c:785
jl_toplevel_eval_in at /home/tim/src/julia-master/src/toplevel.c:880
eval at ./boot.jl:345
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
eval_user_input at /home/tim/src/julia-master/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:139
repl_backend_loop at /home/tim/src/julia-master/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:200
start_repl_backend at /home/tim/src/julia-master/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:185
#run_repl#42 at /home/tim/src/julia-master/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:316
run_repl at /home/tim/src/julia-master/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:304
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
#849 at ./client.jl:387
jfptr_YY.849_34659 at /home/tim/src/julia-master/usr/lib/julia/sys.so (unknown line)
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
jl_apply at /home/tim/src/julia-master/src/julia.h:1682 [inlined]
do_apply at /home/tim/src/julia-master/src/builtins.c:672
jl_f__apply_latest at /home/tim/src/julia-master/src/builtins.c:722
#invokelatest#2 at ./essentials.jl:709 [inlined]
invokelatest at ./essentials.jl:708 [inlined]
run_main_repl at ./client.jl:372
exec_options at ./client.jl:302
_start at ./client.jl:485
jfptr__start_39731 at /home/tim/src/julia-master/usr/lib/julia/sys.so (unknown line)
_jl_invoke at /home/tim/src/julia-master/src/gf.c:2163 [inlined]
jl_apply_generic at /home/tim/src/julia-master/src/gf.c:2345
jl_apply at /home/tim/src/julia-master/ui/../src/julia.h:1682 [inlined]
true_main at /home/tim/src/julia-master/ui/repl.c:106
main at /home/tim/src/julia-master/ui/repl.c:227
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
_start at /home/tim/src/julia-master/julia (unknown line)
Allocations: 7954364 (Pool: 7952021; Big: 2343); GC: 10
/home/tim/bin/julia-master: line 2: 12249 Segmentation fault      (core dumped) ~/src/julia-master/julia "$@"

xref timholy/Revise.jl#535

[timholy]

Closed by JuliaLang/julia#37717