Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: redesign comprehensions #14782

Closed
wants to merge 10 commits into from
15 changes: 9 additions & 6 deletions base/abstractarray.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1296,7 +1296,7 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray)
return dest
end

function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A::AbstractArray)
function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A)
# map to dest array, checking the type of each result. if a result does not
# match, widen the result type and re-dispatch.
i = offs
Expand All @@ -1318,9 +1318,12 @@ function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A::AbstractArray)
return dest
end

_default_eltype(f::Type) = f
_default_eltype(f) = Union{}

function map(f, A::AbstractArray)
if isempty(A)
return isa(f,Type) ? similar(A,f) : similar(A)
return similar(A, _default_eltype(f))
end
st = start(A)
A1, st = next(A, st)
Expand All @@ -1338,7 +1341,7 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray)
return dest
end

function map_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray, B::AbstractArray)
function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A, B)
for i = offs:length(A)
@inbounds Ai, Bi = A[i], B[i]
el = f(Ai, Bi)
Expand All @@ -1358,12 +1361,12 @@ end
function map(f, A::AbstractArray, B::AbstractArray)
shp = promote_shape(size(A),size(B))
if prod(shp) == 0
return similar(A, promote_type(eltype(A),eltype(B)), shp)
return similar(A, _default_eltype(f), shp)
end
first = f(A[1], B[1])
dest = similar(A, typeof(first), shp)
dest[1] = first
return map_to!(f, 2, dest, A, B)
return map_to!(f, 2, nothing, dest, A, B)
end

## N argument
Expand Down Expand Up @@ -1400,7 +1403,7 @@ end
function map(f, As::AbstractArray...)
shape = mapreduce(size, promote_shape, As)
if prod(shape) == 0
return similar(As[1], promote_eltype(As...), shape)
return similar(As[1], Union{}, shape)
end
first = f(map(a->a[1], As)...)
dest = similar(As[1], typeof(first), shape)
Expand Down
4 changes: 3 additions & 1 deletion base/array.jl
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,9 @@ end

function push!{T}(a::Array{T,1}, item)
# convert first so we don't grow the array if the assignment won't work
itemT = convert(T, item)
if T !== Union{}
itemT = convert(T, item)
end
ccall(:jl_array_grow_end, Void, (Any, UInt), a, 1)
a[end] = itemT
return a
Expand Down
1 change: 1 addition & 0 deletions base/boot.jl
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,7 @@ unsafe_convert{T}(::Type{T}, x::T) = x
(::Type{Array{T}}){T}(m::Int, n::Int, o::Int) = Array{T,3}(m, n, o)

# TODO: possibly turn these into deprecations
Array{T,N}(::Type{T}, d::NTuple{N,Int}) = Array{T}(d)
Array{T}(::Type{T}, d::Int...) = Array{T}(d)
Array{T}(::Type{T}, m::Int) = Array{T,1}(m)
Array{T}(::Type{T}, m::Int,n::Int) = Array{T,2}(m,n)
Expand Down
14 changes: 14 additions & 0 deletions base/essentials.jl
Original file line number Diff line number Diff line change
Expand Up @@ -193,3 +193,17 @@ const (:) = Colon()
# For passing constants through type inference
immutable Val{T}
end

immutable Generator{F,I}
f::F
iter::I
end

start(g::Generator) = start(g.iter)
done(g::Generator, s) = done(g.iter, s)
function next(g::Generator, s)
v, s2 = next(g.iter, s)
g.f(v), s2
end

collect(g::Generator) = map(g.f, g.iter)
77 changes: 3 additions & 74 deletions base/inference.jl
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ type StaticVarInfo
gensym_types::Array{Any,1} # types of the GenSym's in this function
vinfo::Array{Any,1} # variable properties
label_counter::Int # index of the current highest label for this function
fedbackvars::ObjectIdDict
mod::Module
end

Expand Down Expand Up @@ -53,7 +52,7 @@ function StaticVarInfo(linfo::LambdaStaticData, ast=linfo.ast)
else
sp = svec()
end
StaticVarInfo(sp, vars, gensym_types, vinflist, nl, ObjectIdDict(), linfo.module)
StaticVarInfo(sp, vars, gensym_types, vinflist, nl, linfo.module)
end

type VarState
Expand Down Expand Up @@ -1070,8 +1069,7 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
return abstract_eval_constant(e)
end
e = e::Expr
# handle:
# call null new & static_typeof
# handle: call null new &
if is(e.head,:call)
t = abstract_eval_call(e, vtypes, sv)
elseif is(e.head,:null)
Expand All @@ -1089,40 +1087,6 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
elseif is(e.head,:&)
abstract_eval(e.args[1], vtypes, sv)
t = Any
elseif is(e.head,:static_typeof)
var = e.args[1]
t = abstract_eval(var, vtypes, sv)
if isa(t,DataType) && typeseq(t,t.name.primary)
# remove unnecessary typevars
t = t.name.primary
end
if is(t,Bottom)
# if we haven't gotten fed-back type info yet, return Bottom. otherwise
# Bottom is the actual type of the variable, so return Type{Bottom}.
if haskey(sv.fedbackvars, var)
t = Type{Bottom}
end
elseif isleaftype(t)
t = Type{t}
elseif isleaftype(inference_stack.types)
if isa(t,TypeVar)
t = Type{t.ub}
else
t = Type{t}
end
else
# if there is any type uncertainty in the arguments, we are
# effectively predicting what static_typeof will say when
# the function is compiled with actual arguments. in that case
# abstract types yield Type{<:T} instead of Type{T}.
# this doesn't really model the situation perfectly, but
# "isleaftype(inference_stack.types)" should be good enough.
if isa(t,TypeVar)
t = Type{t}
else
t = Type{TypeVar(:_,t)}
end
end
elseif is(e.head,:method)
t = (length(e.args) == 1) ? Any : Void
elseif is(e.head,:copyast)
Expand Down Expand Up @@ -1678,10 +1642,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
recpts = IntSet() # statements that depend recursively on our value
W = IntSet()

@label typeinf_top

typegotoredo = false

# exception handlers
cur_hand = ()
handler_at = Any[ () for i=1:n ]
Expand Down Expand Up @@ -1753,24 +1713,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
s[l] = stupdate(s[l], changes, vars)
end
end
elseif is(hd,:type_goto)
for i = 2:length(stmt.args)
var = stmt.args[i]::GenSym
# Store types that need to be fed back via type_goto
# in gensym_init. After finishing inference, if any
# of these types changed, start over with the fed-back
# types known from the beginning.
# See issue #3821 (using !typeseq instead of !subtype),
# and issue #7810.
id = var.id+1
vt = gensym_types[id]
ot = gensym_init[id]
if ot===NF || !typeseq(vt,ot)
gensym_init[id] = vt
typegotoredo = true
end
sv.fedbackvars[var] = true
end
elseif is(hd,:return)
pc´ = n+1
rt = abstract_eval(stmt.args[1], s[pc], sv)
Expand Down Expand Up @@ -1835,16 +1777,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
end
end

if typegotoredo
# if any type_gotos changed, clear state and restart.
for ll = 2:length(s)
s[ll] = ()
end
empty!(W)
gensym_types[:] = gensym_init
frame.result = curtype
@goto typeinf_top
end
for i = 1:length(gensym_types)
if gensym_types[i] === NF
gensym_types[i] = Union{}
Expand Down Expand Up @@ -1931,7 +1863,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, undefs)

e = e::Expr
head = e.head
if is(head,:static_typeof) || is(head,:line) || is(head,:const)
if is(head,:line) || is(head,:const)
return e
#elseif is(head,:gotoifnot) || is(head,:return)
# e.typ = Any
Expand Down Expand Up @@ -2168,9 +2100,6 @@ function effect_free(e::ANY, sv, allow_volatile::Bool)
end
if isa(e,Expr)
e = e::Expr
if e.head === :static_typeof
return true
end
ea = e.args
if e.head === :call
if is_known_call_p(e, is_pure_builtin, sv)
Expand Down
52 changes: 52 additions & 0 deletions base/iterator.jl
Original file line number Diff line number Diff line change
Expand Up @@ -292,3 +292,55 @@ eltype{I1,I2}(::Type{Prod{I1,I2}}) = tuple_type_cons(eltype(I1), eltype(I2))
x = prod_next(p, st)
((x[1][1],x[1][2]...), x[2])
end

immutable GeneratorND{F,I<:AbstractProdIterator}
f::F
iter::I

(::Type{GeneratorND}){F}(f::F, iters...) = (P = product(iters...); new{F,typeof(P)}(f, P))
end

start(g::GeneratorND) = start(g.iter)
done(g::GeneratorND, s) = done(g.iter, s)
function next(g::GeneratorND, s)
v, s2 = next(g.iter, s)
g.f(v...), s2
end

_size(p::Prod2) = (length(p.a), length(p.b))
_size(p::Prod) = (length(p.a), _size(p.b)...)

size(g::GeneratorND) = _size(g.iter)

function collect(g::GeneratorND)
sz = size(g)
if prod(sz) == 0
return Array(Union{}, sz)
end
st = start(g.iter)
A1, st = next(g.iter, st)
first = g.f(A1...)
dest = Array(typeof(first), sz)
dest[1] = first
return map_to!(xs->g.f(xs...), 2, st, dest, g.iter)
end

# special case for 2d
function collect{F,P<:Prod2}(g::GeneratorND{F,P})
f = g.f
a = g.iter.a
b = g.iter.b
sz = size(g)
if prod(sz) == 0
return Array(Union{}, sz)
end
fst = f(first(a), first(b)) # TODO: don't recompute this in the loop
dest = Array(typeof(fst), sz)
for j in b
for i in a
val = f(i, j) # TODO: handle type changes
@inbounds dest[i, j] = val
end
end
return dest
end
20 changes: 10 additions & 10 deletions base/sparse/sparsematrix.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2436,8 +2436,8 @@ end

function vcat(X::SparseMatrixCSC...)
num = length(X)
mX = [ size(x, 1) for x in X ]
nX = [ size(x, 2) for x in X ]
mX = Int[ size(x, 1) for x in X ]
nX = Int[ size(x, 2) for x in X ]
m = sum(mX)
n = nX[1]

Expand All @@ -2454,7 +2454,7 @@ function vcat(X::SparseMatrixCSC...)
Ti = promote_type(Ti, eltype(X[i].rowval))
end

nnzX = [ nnz(x) for x in X ]
nnzX = Int[ nnz(x) for x in X ]
nnz_res = sum(nnzX)
colptr = Array(Ti, n + 1)
rowval = Array(Ti, nnz_res)
Expand Down Expand Up @@ -2496,8 +2496,8 @@ end

function hcat(X::SparseMatrixCSC...)
num = length(X)
mX = [ size(x, 1) for x in X ]
nX = [ size(x, 2) for x in X ]
mX = Int[ size(x, 1) for x in X ]
nX = Int[ size(x, 2) for x in X ]
m = mX[1]
for i = 2 : num
if mX[i] != m; throw(DimensionMismatch("")); end
Expand All @@ -2508,7 +2508,7 @@ function hcat(X::SparseMatrixCSC...)
Ti = promote_type(map(x->eltype(x.rowval), X)...)

colptr = Array(Ti, n + 1)
nnzX = [ nnz(x) for x in X ]
nnzX = Int[ nnz(x) for x in X ]
nnz_res = sum(nnzX)
rowval = Array(Ti, nnz_res)
nzval = Array(Tv, nnz_res)
Expand Down Expand Up @@ -2551,16 +2551,16 @@ Concatenate matrices block-diagonally. Currently only implemented for sparse mat
"""
function blkdiag(X::SparseMatrixCSC...)
num = length(X)
mX = [ size(x, 1) for x in X ]
nX = [ size(x, 2) for x in X ]
mX = Int[ size(x, 1) for x in X ]
nX = Int[ size(x, 2) for x in X ]
m = sum(mX)
n = sum(nX)

Tv = promote_type(map(x->eltype(x.nzval), X)...)
Ti = promote_type(map(x->eltype(x.rowval), X)...)

colptr = Array(Ti, n + 1)
nnzX = [ nnz(x) for x in X ]
nnzX = Int[ nnz(x) for x in X ]
nnz_res = sum(nnzX)
rowval = Array(Ti, nnz_res)
nzval = Array(Tv, nnz_res)
Expand Down Expand Up @@ -2761,7 +2761,7 @@ function trace{Tv}(A::SparseMatrixCSC{Tv})
s
end

diag(A::SparseMatrixCSC) = [d for d in SpDiagIterator(A)]
diag{Tv}(A::SparseMatrixCSC{Tv}) = Tv[d for d in SpDiagIterator(A)]

function diagm{Tv,Ti}(v::SparseMatrixCSC{Tv,Ti})
if (size(v,1) != 1 && size(v,2) != 1)
Expand Down
1 change: 1 addition & 0 deletions deps/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,7 @@ $(eval $(call LLVM_PATCH,llvm-3.7.1))
else ifeq ($(LLVM_VER),3.7.1)
$(eval $(call LLVM_PATCH,llvm-3.7.1))
$(eval $(call LLVM_PATCH,llvm-3.7.1_2))
$(eval $(call LLVM_PATCH,llvm-3.7.1_3))
$(LLVM_SRC_DIR)/llvm-3.7.1_2.patch-applied: $(LLVM_SRC_DIR)/llvm-3.7.1.patch-applied
endif # LLVM_VER

Expand Down
16 changes: 16 additions & 0 deletions deps/llvm-3.7.1_3.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
Index: /lib/Analysis/ScalarEvolutionExpander.cpp
===================================================================
--- /lib/Analysis/ScalarEvolutionExpander.cpp
+++ /lib/Analysis/ScalarEvolutionExpander.cpp
@@ -1273,6 +1273,11 @@
if (!SE.dominates(Step, L->getHeader())) {
PostLoopScale = Step;
Step = SE.getConstant(Normalized->getType(), 1);
+ if (!PostLoopOffset) {
+ // otherwise, Start is known to already be a constant zero
+ PostLoopOffset = Start;
+ Start = SE.getConstant(Normalized->getType(), 0);
+ }
Normalized =
cast<SCEVAddRecExpr>(SE.getAddRecExpr(
Start, Step, Normalized->getLoop(),
2 changes: 1 addition & 1 deletion doc/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
-e git+https://github.com/JuliaLang/JuliaDoc.git@85c83a5c4ecd1716e0aa858d73322cc2058bc7b1#egg=JuliaDoc
-e git+https://github.com/JuliaLang/JuliaDoc.git@fe343066983b33e6451c441bbe99a91d97a7e095#egg=JuliaDoc
-e git+https://github.com/snide/sphinx_rtd_theme.git@21e875d3a53ce897089ad690d897252f6063349d#egg=sphinx_rtd_theme
Loading