WIP: redesign comprehensions

base/abstractarray.jl

@@ -1296,7 +1296,7 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray)
     return dest
 end
 
-function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A::AbstractArray)
+function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A)
     # map to dest array, checking the type of each result. if a result does not
     # match, widen the result type and re-dispatch.
     i = offs
@@ -1318,9 +1318,12 @@ function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A::AbstractArray)
     return dest
 end
 
+_default_eltype(f::Type) = f
+_default_eltype(f) = Union{}
+
 function map(f, A::AbstractArray)
     if isempty(A)
-        return isa(f,Type) ? similar(A,f) : similar(A)
+        return similar(A, _default_eltype(f))
     end
     st = start(A)
     A1, st = next(A, st)
@@ -1338,7 +1341,7 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray)
     return dest
 end
 
-function map_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray, B::AbstractArray)
+function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A, B)
     for i = offs:length(A)
         @inbounds Ai, Bi = A[i], B[i]
         el = f(Ai, Bi)
@@ -1358,12 +1361,12 @@ end
 function map(f, A::AbstractArray, B::AbstractArray)
     shp = promote_shape(size(A),size(B))
     if prod(shp) == 0
-        return similar(A, promote_type(eltype(A),eltype(B)), shp)
+        return similar(A, _default_eltype(f), shp)
     end
     first = f(A[1], B[1])
     dest = similar(A, typeof(first), shp)
     dest[1] = first
-    return map_to!(f, 2, dest, A, B)
+    return map_to!(f, 2, nothing, dest, A, B)
 end
 
 ## N argument
@@ -1400,7 +1403,7 @@ end
 function map(f, As::AbstractArray...)
     shape = mapreduce(size, promote_shape, As)
     if prod(shape) == 0
-        return similar(As[1], promote_eltype(As...), shape)
+        return similar(As[1], Union{}, shape)
     end
     first = f(map(a->a[1], As)...)
     dest = similar(As[1], typeof(first), shape)

base/array.jl

@@ -387,7 +387,9 @@ end
 
 function push!{T}(a::Array{T,1}, item)
     # convert first so we don't grow the array if the assignment won't work
-    itemT = convert(T, item)
+    if T !== Union{}
+        itemT = convert(T, item)
+    end
     ccall(:jl_array_grow_end, Void, (Any, UInt), a, 1)
     a[end] = itemT
     return a

base/boot.jl

@@ -341,6 +341,7 @@ unsafe_convert{T}(::Type{T}, x::T) = x
 (::Type{Array{T}}){T}(m::Int, n::Int, o::Int) = Array{T,3}(m, n, o)
 
 # TODO: possibly turn these into deprecations
+Array{T,N}(::Type{T}, d::NTuple{N,Int}) = Array{T}(d)
 Array{T}(::Type{T}, d::Int...) = Array{T}(d)
 Array{T}(::Type{T}, m::Int)               = Array{T,1}(m)
 Array{T}(::Type{T}, m::Int,n::Int)        = Array{T,2}(m,n)

base/essentials.jl

@@ -193,3 +193,17 @@ const (:) = Colon()
 # For passing constants through type inference
 immutable Val{T}
 end
+
+immutable Generator{F,I}
+    f::F
+    iter::I
+end
+
+start(g::Generator) = start(g.iter)
+done(g::Generator, s) = done(g.iter, s)
+function next(g::Generator, s)
+    v, s2 = next(g.iter, s)
+    g.f(v), s2
+end
+
+collect(g::Generator) = map(g.f, g.iter)

base/inference.jl

@@ -25,7 +25,6 @@ type StaticVarInfo
     gensym_types::Array{Any,1} # types of the GenSym's in this function
     vinfo::Array{Any,1}  # variable properties
     label_counter::Int   # index of the current highest label for this function
-    fedbackvars::ObjectIdDict
     mod::Module
 end
 
@@ -53,7 +52,7 @@ function StaticVarInfo(linfo::LambdaStaticData, ast=linfo.ast)
     else
         sp = svec()
     end
-    StaticVarInfo(sp, vars, gensym_types, vinflist, nl, ObjectIdDict(), linfo.module)
+    StaticVarInfo(sp, vars, gensym_types, vinflist, nl, linfo.module)
 end
 
 type VarState
@@ -1070,8 +1069,7 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
         return abstract_eval_constant(e)
     end
     e = e::Expr
-    # handle:
-    # call  null  new  &  static_typeof
+    # handle: call  null  new  &
     if is(e.head,:call)
         t = abstract_eval_call(e, vtypes, sv)
     elseif is(e.head,:null)
@@ -1089,40 +1087,6 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
     elseif is(e.head,:&)
         abstract_eval(e.args[1], vtypes, sv)
         t = Any
-    elseif is(e.head,:static_typeof)
-        var = e.args[1]
-        t = abstract_eval(var, vtypes, sv)
-        if isa(t,DataType) && typeseq(t,t.name.primary)
-            # remove unnecessary typevars
-            t = t.name.primary
-        end
-        if is(t,Bottom)
-            # if we haven't gotten fed-back type info yet, return Bottom. otherwise
-            # Bottom is the actual type of the variable, so return Type{Bottom}.
-            if haskey(sv.fedbackvars, var)
-                t = Type{Bottom}
-            end
-        elseif isleaftype(t)
-            t = Type{t}
-        elseif isleaftype(inference_stack.types)
-            if isa(t,TypeVar)
-                t = Type{t.ub}
-            else
-                t = Type{t}
-            end
-        else
-            # if there is any type uncertainty in the arguments, we are
-            # effectively predicting what static_typeof will say when
-            # the function is compiled with actual arguments. in that case
-            # abstract types yield Type{<:T} instead of Type{T}.
-            # this doesn't really model the situation perfectly, but
-            # "isleaftype(inference_stack.types)" should be good enough.
-            if isa(t,TypeVar)
-                t = Type{t}
-            else
-                t = Type{TypeVar(:_,t)}
-            end
-        end
     elseif is(e.head,:method)
         t = (length(e.args) == 1) ? Any : Void
     elseif is(e.head,:copyast)
@@ -1678,10 +1642,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
     recpts = IntSet()  # statements that depend recursively on our value
     W = IntSet()
 
-    @label typeinf_top
-
-    typegotoredo = false
-
     # exception handlers
     cur_hand = ()
     handler_at = Any[ () for i=1:n ]
@@ -1753,24 +1713,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
                             s[l] = stupdate(s[l], changes, vars)
                         end
                     end
-                elseif is(hd,:type_goto)
-                    for i = 2:length(stmt.args)
-                        var = stmt.args[i]::GenSym
-                        # Store types that need to be fed back via type_goto
-                        # in gensym_init. After finishing inference, if any
-                        # of these types changed, start over with the fed-back
-                        # types known from the beginning.
-                        # See issue #3821 (using !typeseq instead of !subtype),
-                        # and issue #7810.
-                        id = var.id+1
-                        vt = gensym_types[id]
-                        ot = gensym_init[id]
-                        if ot===NF || !typeseq(vt,ot)
-                            gensym_init[id] = vt
-                            typegotoredo = true
-                        end
-                        sv.fedbackvars[var] = true
-                    end
                 elseif is(hd,:return)
                     pc´ = n+1
                     rt = abstract_eval(stmt.args[1], s[pc], sv)
@@ -1835,16 +1777,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
         end
     end
 
-    if typegotoredo
-        # if any type_gotos changed, clear state and restart.
-        for ll = 2:length(s)
-            s[ll] = ()
-        end
-        empty!(W)
-        gensym_types[:] = gensym_init
-        frame.result = curtype
-        @goto typeinf_top
-    end
     for i = 1:length(gensym_types)
         if gensym_types[i] === NF
             gensym_types[i] = Union{}
@@ -1931,7 +1863,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, undefs)
 
     e = e::Expr
     head = e.head
-    if is(head,:static_typeof) || is(head,:line) || is(head,:const)
+    if is(head,:line) || is(head,:const)
         return e
     #elseif is(head,:gotoifnot) || is(head,:return)
     #    e.typ = Any
@@ -2168,9 +2100,6 @@ function effect_free(e::ANY, sv, allow_volatile::Bool)
     end
     if isa(e,Expr)
         e = e::Expr
-        if e.head === :static_typeof
-            return true
-        end
         ea = e.args
         if e.head === :call
             if is_known_call_p(e, is_pure_builtin, sv)

base/iterator.jl

@@ -292,3 +292,55 @@ eltype{I1,I2}(::Type{Prod{I1,I2}}) = tuple_type_cons(eltype(I1), eltype(I2))
     x = prod_next(p, st)
     ((x[1][1],x[1][2]...), x[2])
 end
+
+immutable GeneratorND{F,I<:AbstractProdIterator}
+    f::F
+    iter::I
+
+    (::Type{GeneratorND}){F}(f::F, iters...) = (P = product(iters...); new{F,typeof(P)}(f, P))
+end
+
+start(g::GeneratorND) = start(g.iter)
+done(g::GeneratorND, s) = done(g.iter, s)
+function next(g::GeneratorND, s)
+    v, s2 = next(g.iter, s)
+    g.f(v...), s2
+end
+
+_size(p::Prod2) = (length(p.a), length(p.b))
+_size(p::Prod) = (length(p.a), _size(p.b)...)
+
+size(g::GeneratorND) = _size(g.iter)
+
+function collect(g::GeneratorND)
+    sz = size(g)
+    if prod(sz) == 0
+        return Array(Union{}, sz)
+    end
+    st = start(g.iter)
+    A1, st = next(g.iter, st)
+    first = g.f(A1...)
+    dest = Array(typeof(first), sz)
+    dest[1] = first
+    return map_to!(xs->g.f(xs...), 2, st, dest, g.iter)
+end
+
+# special case for 2d
+function collect{F,P<:Prod2}(g::GeneratorND{F,P})
+    f = g.f
+    a = g.iter.a
+    b = g.iter.b
+    sz = size(g)
+    if prod(sz) == 0
+        return Array(Union{}, sz)
+    end
+    fst = f(first(a), first(b))  # TODO: don't recompute this in the loop
+    dest = Array(typeof(fst), sz)
+    for j in b
+        for i in a
+            val = f(i, j)  # TODO: handle type changes
+            @inbounds dest[i, j] = val
+        end
+    end
+    return dest
+end

base/sparse/sparsematrix.jl

@@ -2436,8 +2436,8 @@ end
 
 function vcat(X::SparseMatrixCSC...)
     num = length(X)
-    mX = [ size(x, 1) for x in X ]
-    nX = [ size(x, 2) for x in X ]
+    mX = Int[ size(x, 1) for x in X ]
+    nX = Int[ size(x, 2) for x in X ]
     m = sum(mX)
     n = nX[1]
 
@@ -2454,7 +2454,7 @@ function vcat(X::SparseMatrixCSC...)
         Ti = promote_type(Ti, eltype(X[i].rowval))
     end
 
-    nnzX = [ nnz(x) for x in X ]
+    nnzX = Int[ nnz(x) for x in X ]
     nnz_res = sum(nnzX)
     colptr = Array(Ti, n + 1)
     rowval = Array(Ti, nnz_res)
@@ -2496,8 +2496,8 @@ end
 
 function hcat(X::SparseMatrixCSC...)
     num = length(X)
-    mX = [ size(x, 1) for x in X ]
-    nX = [ size(x, 2) for x in X ]
+    mX = Int[ size(x, 1) for x in X ]
+    nX = Int[ size(x, 2) for x in X ]
     m = mX[1]
     for i = 2 : num
         if mX[i] != m; throw(DimensionMismatch("")); end
@@ -2508,7 +2508,7 @@ function hcat(X::SparseMatrixCSC...)
     Ti = promote_type(map(x->eltype(x.rowval), X)...)
 
     colptr = Array(Ti, n + 1)
-    nnzX = [ nnz(x) for x in X ]
+    nnzX = Int[ nnz(x) for x in X ]
     nnz_res = sum(nnzX)
     rowval = Array(Ti, nnz_res)
     nzval = Array(Tv, nnz_res)
@@ -2551,16 +2551,16 @@ Concatenate matrices block-diagonally. Currently only implemented for sparse mat
 """
 function blkdiag(X::SparseMatrixCSC...)
     num = length(X)
-    mX = [ size(x, 1) for x in X ]
-    nX = [ size(x, 2) for x in X ]
+    mX = Int[ size(x, 1) for x in X ]
+    nX = Int[ size(x, 2) for x in X ]
     m = sum(mX)
     n = sum(nX)
 
     Tv = promote_type(map(x->eltype(x.nzval), X)...)
     Ti = promote_type(map(x->eltype(x.rowval), X)...)
 
     colptr = Array(Ti, n + 1)
-    nnzX = [ nnz(x) for x in X ]
+    nnzX = Int[ nnz(x) for x in X ]
     nnz_res = sum(nnzX)
     rowval = Array(Ti, nnz_res)
     nzval = Array(Tv, nnz_res)
@@ -2761,7 +2761,7 @@ function trace{Tv}(A::SparseMatrixCSC{Tv})
     s
 end
 
-diag(A::SparseMatrixCSC) = [d for d in SpDiagIterator(A)]
+diag{Tv}(A::SparseMatrixCSC{Tv}) = Tv[d for d in SpDiagIterator(A)]
 
 function diagm{Tv,Ti}(v::SparseMatrixCSC{Tv,Ti})
     if (size(v,1) != 1 && size(v,2) != 1)

deps/Makefile

@@ -670,6 +670,7 @@ $(eval $(call LLVM_PATCH,llvm-3.7.1))
 else ifeq ($(LLVM_VER),3.7.1)
 $(eval $(call LLVM_PATCH,llvm-3.7.1))
 $(eval $(call LLVM_PATCH,llvm-3.7.1_2))
+$(eval $(call LLVM_PATCH,llvm-3.7.1_3))
 $(LLVM_SRC_DIR)/llvm-3.7.1_2.patch-applied: $(LLVM_SRC_DIR)/llvm-3.7.1.patch-applied
 endif # LLVM_VER
 

deps/llvm-3.7.1_3.patch

@@ -0,0 +1,16 @@
+Index: /lib/Analysis/ScalarEvolutionExpander.cpp
+===================================================================
+--- /lib/Analysis/ScalarEvolutionExpander.cpp
++++ /lib/Analysis/ScalarEvolutionExpander.cpp
+@@ -1273,6 +1273,11 @@
+   if (!SE.dominates(Step, L->getHeader())) {
+     PostLoopScale = Step;
+     Step = SE.getConstant(Normalized->getType(), 1);
++    if (!PostLoopOffset) {
++        // otherwise, Start is known to already be a constant zero
++        PostLoopOffset = Start;
++        Start = SE.getConstant(Normalized->getType(), 0);
++    }
+     Normalized =
+       cast<SCEVAddRecExpr>(SE.getAddRecExpr(
+                              Start, Step, Normalized->getLoop(),

doc/requirements.txt

@@ -1,2 +1,2 @@
--e git+https://github.com/JuliaLang/JuliaDoc.git@85c83a5c4ecd1716e0aa858d73322cc2058bc7b1#egg=JuliaDoc
+-e git+https://github.com/JuliaLang/JuliaDoc.git@fe343066983b33e6451c441bbe99a91d97a7e095#egg=JuliaDoc
 -e git+https://github.com/snide/sphinx_rtd_theme.git@21e875d3a53ce897089ad690d897252f6063349d#egg=sphinx_rtd_theme