eliminate Undef and Top types. fixes #8631

thanks to @vtjnash for the test case.

It's hard to tell whether this is the real fix for the problem, but
it works, and having a type that's not a subtype of Any around was
just a huge pain. Things are simpler without it.

Inside the compiler, undefinedness is no longer part of a variable's
type. Instead there is a bit per variable telling whether it is ever
used when undefined. This has the disadvantage of being coarser, since
it doesn't give the undefinedness of each variable use. However it has
the advantage of giving better type info: if a variable is either a
Float64 or undefined, its type is just "Float64" and we can optimize
accordingly.

This combines well with the future optimization of storing
possibly-undefined variables unboxed (#6914).
For that we will add a run time 1-bit flag to track definedness, and
then LLVM can hopefully eliminate checks along paths where the
flag is known to be set, gaining back the previous granularity.

Author: JeffBezanson

base/base.jl

@@ -271,7 +271,7 @@ call{T}(::Type{Array{T}}, m::Integer, n::Integer, o::Integer) =
 
 # TODO: possibly turn these into deprecations
 Array{T,N}(::Type{T}, d::NTuple{N,Int}) = Array{T}(d)
-Array(T::Type, d::Integer...)           = Array{T}(convert((Int...), d))
+Array{T}(::Type{T}, d::Integer...)      = Array{T}(convert((Int...), d))
 Array{T}(::Type{T}, m::Integer)                       = Array{T}(m)
 Array{T}(::Type{T}, m::Integer,n::Integer)            = Array{T}(m,n)
 Array{T}(::Type{T}, m::Integer,n::Integer,o::Integer) = Array{T}(m,n,o)

base/boot.jl

@@ -117,7 +117,7 @@ import Core.Intrinsics.ccall
 
 export
     # key types
-    Any, DataType, Vararg, ANY, NTuple, Top,
+    Any, DataType, Vararg, ANY, NTuple,
     Tuple, Type, TypeConstructor, TypeName, TypeVar, Union, UnionType, Void,
     AbstractArray, DenseArray,
     # special objects

base/inference.jl

@@ -15,6 +15,7 @@ type StaticVarInfo
     vars::Array{Any,1}   # names of args and locals
     gensym_types::Array{Any,1} # types of the GenSym's in this function
     label_counter::Int   # index of the current highest label for this function
+    fedbackvars::ObjectIdDict
 end
 
 type EmptyCallStack
@@ -342,7 +343,7 @@ const getfield_tfunc = function (A, s0, name)
             return abstract_eval_constant(eval(A1,fld))
         end
         if s === Module
-            return Top
+            return Any
         end
         if isType(s0)
             sp = s0.parameters[1]
@@ -939,9 +940,6 @@ end
 
 function abstract_eval_arg(a::ANY, vtypes::ANY, sv::StaticVarInfo)
     t = abstract_eval(a, vtypes, sv)
-    if isa(a,Symbol) || isa(a,SymbolNode) || isa(a,GenSym)
-        t = typeintersect(t,Any)  # remove Undef
-    end
     if isa(t,TypeVar) && t.lb == Bottom && isleaftype(t.ub)
         t = t.ub
     end
@@ -1018,19 +1016,18 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
         abstract_eval(e.args[1], vtypes, sv)
         t = Any
     elseif is(e.head,:static_typeof)
-        t0 = abstract_eval(e.args[1], vtypes, sv)
-        # intersect with Any to remove Undef
-        t = typeintersect(t0, Any)
+        var = e.args[1]
+        t = t0 = abstract_eval(var, vtypes, sv)
         if isa(t,DataType) && typeseq(t,t.name.primary)
             # remove unnecessary typevars
             t = t.name.primary
         end
-        if is(t,Bottom) && Undef<:t0
-            # the first time we see this statement the variable will probably
-            # be Undef; return Bottom so this doesn't contribute to the type
-            # we eventually pick.
-        elseif is(t,Bottom)
-            t = Type{Bottom}
+        if is(t,Bottom)
+            # if we haven't gotten fed-back type info yet, return Bottom. otherwise
+            # Bottom is the actual type of the variable, so return Type{Bottom}.
+            if haskey(sv.fedbackvars, var)
+                t = Type{Bottom}
+            end
         elseif isleaftype(t)
             t = Type{t}
         elseif isleaftype(inference_stack.types)
@@ -1059,7 +1056,7 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
     else
         t = Any
     end
-    if isa(t,TypeVar) && t.lb === Bottom
+    if isa(t,TypeVar)
         # no need to use a typevar as the type of an expression
         t = t.ub
     end
@@ -1082,30 +1079,20 @@ function abstract_eval_constant(x::ANY)
     return typeof(x)
 end
 
-# Undef is the static type of a value location (e.g. variable) that is
-# undefined. The corresponding run-time type is Bottom, since accessing an
-# undefined location is an error. A non-lvalue expression cannot have
-# type Undef, only Bottom.
-# typealias Top Union(Any,Undef)
-
 abstract_eval_global(s::Symbol) =
     abstract_eval_global((inference_stack::CallStack).mod, s)
 
 function abstract_eval_global(M, s::Symbol)
     if isconst(M,s)
         return abstract_eval_constant(eval(M,s))
     end
-    if !isdefined(M,s)
-        return Top
-    end
-    # TODO: change to Undef if there's a way to clear variables
     return Any
 end
 
 function abstract_eval_gensym(s::GenSym, sv::StaticVarInfo)
     typ = sv.gensym_types[s.id+1]
     if typ === NF
-        return Undef
+        return Bottom
     end
     return typ
 end
@@ -1137,7 +1124,7 @@ function abstract_eval_symbol(s::Symbol, vtypes::ObjectIdDict, sv::StaticVarInfo
         end
         if s in sv.vars
             # local variable use not reached
-            return Top
+            return Bottom
         end
         # global
         return abstract_eval_global(s)
@@ -1208,18 +1195,10 @@ function type_too_complex(t::ANY, d)
 end
 
 function tmerge(typea::ANY, typeb::ANY)
-    if is(typea,NF)
-        return typeb
-    end
-    if is(typeb,NF)
-        return typea
-    end
-    if typea <: typeb
-        return typeb
-    end
-    if typeb <: typea
-        return typea
-    end
+    is(typea, NF ) && return typeb
+    is(typeb, NF)  && return typea
+    typea <: typeb && return typeb
+    typeb <: typea && return typea
     if isa(typea, Tuple) && isa(typeb, Tuple)
         if length(typea) == length(typeb) && !isvatuple(typea) && !isvatuple(typeb)
             return typejoin(typea, typeb)
@@ -1230,7 +1209,7 @@ function tmerge(typea::ANY, typeb::ANY)
     if length(u.types) > MAX_TYPEUNION_LEN || type_too_complex(u, 0)
         # don't let type unions get too big
         # TODO: something smarter, like a common supertype
-        return Undef<:u ? Top : Any
+        return Any
     end
     return u
 end
@@ -1252,13 +1231,19 @@ function stupdate(state::ObjectIdDict, changes::Union(StateUpdate,VarTable), var
     state
 end
 
-function stchanged(new::Union(StateUpdate,VarTable), old, vars)
+function stchanged(new::Union(StateUpdate,VarTable), old, vinflist)
     if is(old,())
         return true
     end
-    for i = 1:length(vars)
-        v = vars[i]
-        if tchanged(new[v], get(old,v,NF))
+    for vi in vinflist
+        v = vi[1]
+        newtype = new[v]
+        oldtype = get(old,v,NF)
+        if (newtype === Bottom && oldtype !== NF && oldtype !== Bottom) ||
+            (oldtype === Bottom && newtype !== NF && newtype !== Bottom)
+            vi[3] |= 32
+        end
+        if tchanged(newtype, oldtype)
             return true
         end
     end
@@ -1457,7 +1442,7 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop)
     # initial types
     s[1] = ObjectIdDict()
     for v in vars
-        s[1][v] = Undef
+        s[1][v] = Bottom
     end
     if la > 0
         lastarg = ast.args[1][la]
@@ -1507,7 +1492,8 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop)
         cenv[vname] = vtype
         s[1][vname] = vtype
     end
-    for vi = ((ast.args[2][2])::Array{Any,1})
+    vinflist = ast.args[2][2]::Array{Any,1}
+    for vi in vinflist
         vi::Array{Any,1}
         if (vi[3]&4)!=0
             # variables assigned by inner functions are treated like
@@ -1523,7 +1509,7 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop)
     gensym_init = Any[ NF for i = 1:length(gensym_uses) ]
     gensym_types = copy(gensym_init)
 
-    sv = StaticVarInfo(sparams, cenv, vars, gensym_types, length(labels))
+    sv = StaticVarInfo(sparams, cenv, vars, gensym_types, length(labels), ObjectIdDict())
     frame.sv = sv
 
     recpts = IntSet()  # statements that depend recursively on our value
@@ -1565,7 +1551,7 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop)
             if !is(cur_hand,())
                 # propagate type info to exception handler
                 l = cur_hand[1]::Int
-                if stchanged(changes, s[l], vars)
+                if stchanged(changes, s[l], vinflist)
                     push!(W, l)
                     s[l] = stupdate(s[l], changes, vars)
                 end
@@ -1598,7 +1584,7 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop)
                     else
                         # general case
                         handler_at[l] = cur_hand
-                        if stchanged(changes, s[l], vars)
+                        if stchanged(changes, s[l], vinflist)
                             push!(W, l)
                             s[l] = stupdate(s[l], changes, vars)
                         end
@@ -1619,6 +1605,7 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop)
                             gensym_init[id] = vt
                             typegotoredo = true
                         end
+                        sv.fedbackvars[var] = true
                     end
                 elseif is(hd,:return)
                     pc´ = n+1
@@ -1663,7 +1650,7 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop)
                 end
             end
             if pc´<=n && (handler_at[pc´] = cur_hand; true) &&
-               stchanged(changes, s[pc´], vars)
+               stchanged(changes, s[pc´], vinflist)
                 s[pc´] = stupdate(s[pc´], changes, vars)
                 pc = pc´
             elseif pc´ in W
@@ -1752,7 +1739,7 @@ function record_var_type(e::Symbol, t::ANY, decls)
     end
 end
 
-function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, clo)
+function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, clo, undefs)
     if isa(e, Symbol)
         e = e::Symbol
 
@@ -1761,13 +1748,19 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, clo)
             return e
         end
         t = abstract_eval(e, vtypes, sv)
+        if t === Bottom
+            undefs[e] = true
+        end
         record_var_type(e, t, decls)
         return (is(t,Any) || is(t,IntrinsicFunction)) ? e : SymbolNode(e, t)
     end
 
     if isa(e, SymbolNode)
         e = e::SymbolNode
         curtype = e.typ
+        if curtype === Bottom
+            undefs[e.name] = true
+        end
         t = abstract_eval(e.name, vtypes, sv)
         if !(curtype <: t) || typeseq(curtype, t)
             record_var_type(e.name, t, decls)
@@ -1800,7 +1793,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, clo)
             # we don't use types on assignment LHS
             s = s.name
         end
-        e.args[2] = eval_annotate(e.args[2], vtypes, sv, decls, clo)
+        e.args[2] = eval_annotate(e.args[2], vtypes, sv, decls, clo, undefs)
         if isa(s,Symbol)
             # TODO: if this def does not reach any uses, maybe don't do this
             rhstype = exprtype(e.args[2], sv)
@@ -1814,7 +1807,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, clo)
     for i=i0:length(e.args)
         subex = e.args[i]
         if !(isa(subex,Number) || isa(subex,AbstractString))
-            e.args[i] = eval_annotate(subex, vtypes, sv, decls, clo)
+            e.args[i] = eval_annotate(subex, vtypes, sv, decls, clo, undefs)
         end
     end
     if (head === :call || head === :call1) && isa(e.args[1],LambdaStaticData)
@@ -1832,6 +1825,7 @@ end
 function type_annotate(ast::Expr, states::Array{Any,1}, sv::ANY, rettype::ANY,
                        args)
     decls = ObjectIdDict()
+    undefs = ObjectIdDict()
     # initialize decls with argument types
     for arg in args
         decls[arg] = states[1][arg]
@@ -1842,7 +1836,7 @@ function type_annotate(ast::Expr, states::Array{Any,1}, sv::ANY, rettype::ANY,
         st_i = states[i]
         if st_i !== ()
             # st_i === ()  =>  unreached statement  (see issue #7836)
-            body[i] = eval_annotate(body[i], st_i, sv, decls, closures)
+            body[i] = eval_annotate(body[i], st_i, sv, decls, closures, undefs)
         end
     end
     ast.args[3].typ = rettype
@@ -1852,11 +1846,17 @@ function type_annotate(ast::Expr, states::Array{Any,1}, sv::ANY, rettype::ANY,
         if (vi[3]&4)==0
             vi[2] = get(decls, vi[1], vi[2])
         end
+        if haskey(undefs, vi[1])
+            vi[3] |= 32
+        end
     end
     for vi in ast.args[2][3]::Array{Any,1}
         if (vi[3]&4)==0
             vi[2] = get(decls, vi[1], vi[2])
         end
+        if haskey(undefs, vi[1])
+            vi[3] |= 32
+        end
     end
     ast.args[2][4] = sv.gensym_types
 
@@ -2350,7 +2350,6 @@ function inlineable(f::ANY, e::Expr, atypes::Tuple, sv::StaticVarInfo, enclosing
             for i = 1:numarg
                 name = newnames[i]
                 argtype = exprtype(argexprs[i],sv)
-                argtype = typeintersect(argtype,Any)  # remove Undef
                 push!(locals, Any[name,argtype,0])
                 push!(newcall.args, argtype===Any ? name : SymbolNode(name, argtype))
             end
@@ -3044,7 +3043,7 @@ function remove_redundant_temp_vars(ast, sa)
             # this transformation is not valid for vars used before def.
             # we need to preserve the point of assignment to know where to
             # throw errors (issue #4645).
-            if !occurs_undef(v, ast.args[3])
+            if !occurs_undef(v, ast.args[3], varinfo)
 
                 # the transformation is not ideal if the assignment
                 # is present for the auto-unbox functionality
@@ -3063,18 +3062,22 @@ end
 
 function local_typeof(v, varinfo)
     for (v2, typ, info) in varinfo
-        if v === v2
-            return typ
-        end
+        v === v2 && return typ
+    end
+    @assert false "v not in varinfo"
+end
+function var_infobits(v, varinfo)
+    for (v2, typ, info) in varinfo
+        v === v2 && return info
     end
     @assert false "v not in varinfo"
 end
 
-occurs_undef(var::GenSym, expr) = false
+occurs_undef(var::GenSym, expr, varinfo) = false
 
-occurs_undef(var, expr) =
-    occurs_more(expr,
-                e->(isa(e,SymbolNode) && symequal(var,e) && issubtype(Undef,e.typ)), 0)>0
+occurs_undef(var, expr, varinfo) =
+    occurs_more(expr, e->(isa(e,SymbolNode) && symequal(var,e) &&
+                          ((var_infobits(e.name,varinfo)&32)!=0)), 0)>0
 
 # compute set of vars assigned once
 function find_sa_vars(ast)

base/serialize.jl

@@ -22,7 +22,7 @@ let i = 2
              UTF16String, UTF32String, Float16,
              :reserved9, :reserved10, :reserved11, :reserved12,
 
-             (), Bool, Any, :Any, Bottom, Top, Undef, Type,
+             (), Bool, Any, :Any, Bottom, :reserved21, :reserved22, Type,
              :Array, :TypeVar, :Box,
              :lambda, :body, :return, :call, symbol("::"),
              :(=), :null, :gotoifnot, :A, :B, :C, :M, :N, :T, :S, :X, :Y,

base/show.jl

@@ -66,13 +66,7 @@ function show(io::IO, x::IntrinsicFunction)
     print(io, "(intrinsic function #", box(Int32,unbox(IntrinsicFunction,x)), ")")
 end
 
-function show(io::IO, x::UnionType)
-    if is(x,Top)
-        print(io, "Top")
-    else
-        print(io, "Union", x.types)
-    end
-end
+show(io::IO, x::UnionType) = print(io, "Union", x.types)
 
 show(io::IO, x::TypeConstructor) = show(io, x.body)