From bd7cdf7c282d2f6c23fd376a66afbe9a3d919eaf Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 20 Jun 2023 21:07:38 +0000 Subject: [PATCH 1/4] RFC: A path forward on --check-bounds In 1.9, `--check-bounds=no` has started causing significant performance regressions (e.g. #50110). This is because we switched a number of functions that used to be `@pure` to new effects-based infrastructure, which very closely tracks the the legality conditions for concrete evaluation. Unfortunately, disabling bounds checking completely invalidates all prior legality analysis, so the only realistic path we have is to completely disable it. In general, we are learning that these kinds of global make-things-faster-but-unsafe flags are highly problematic for a language for several reasons: - Code is written with the assumption of a particular mode being chosen, so it is in general not possible or unsafe to compose libraries (which in a language like julia is a huge problem). - Unsafe semantics are often harder for the compiler to reason about, causing unexpected performance issues (although the 1.9 --check-bounds=no issues are worse than just disabling concrete eval for things that use bounds checking) In general, I'd like to remove the `--check-bounds=` option entirely (#48245), but that proposal has encountered major opposition. This PR implements an alternative proposal: We introduce a new function `Core.should_check_bounds(boundscheck::Bool) = boundscheck`. This function is passed the result of `Expr(:boundscheck)` (which is now purely determined by the inliner based on `@inbounds`, without regard for the command line flag). In this proposal, what the command line flag does is simply redefine this function to either `true` or `false` (unconditionally) depending on the value of the flag. Of course, this causes massive amounts of recompilation, but I think this can be addressed by adding logic to loading that loads a pkgimage with appropriate definitions to cure the invalidations. The special logic we have now now to take account of the --check-bounds flag in .ji selection, would be replaced by automatically injecting the special pkgimage as a dependency to every loaded image. This part isn't implemented in this PR, but I think it's reasonable to do. I think with that, the `--check-bounds` flag remains functional, while having much more well defined behavior, as it relies on the standard world age mechanisms. A major benefit of this approach is that it can be scoped appropriately using overlay tables. For exmaple: ``` julia> using CassetteOverlay julia> @MethodTable AssumeInboundsTable; julia> @overlay AssumeInboundsTable Core.should_check_bounds(b::Bool) = false; julia> assume_inbounds = @overlaypass AssumeInboundsTable julia> assume_inbounds(f, args...) # f(args...) with bounds checking disabled dynamically ``` Similar logic applies to GPUCompiler, which already supports overlay tables. --- base/array.jl | 4 ++-- base/boot.jl | 2 ++ base/client.jl | 9 +++++++++ base/compiler/abstractinterpretation.jl | 7 ------- base/compiler/ssair/inlining.jl | 4 ++-- base/compiler/utilities.jl | 6 ------ base/essentials.jl | 8 ++++---- base/experimental.jl | 4 ++-- base/tuple.jl | 4 ++-- src/cgutils.cpp | 27 ++----------------------- src/codegen.cpp | 4 ++-- test/loading.jl | 16 +++------------ 12 files changed, 30 insertions(+), 65 deletions(-) diff --git a/base/array.jl b/base/array.jl index d3d4750743a91..4c8e4078c228d 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1017,9 +1017,9 @@ Dict{String, Int64} with 2 entries: function setindex! end @eval setindex!(A::Array{T}, x, i1::Int) where {T} = - arrayset($(Expr(:boundscheck)), A, x isa T ? x : convert(T,x)::T, i1) + arrayset(Core.should_check_bounds($(Expr(:boundscheck))), A, x isa T ? x : convert(T,x)::T, i1) @eval setindex!(A::Array{T}, x, i1::Int, i2::Int, I::Int...) where {T} = - (@inline; arrayset($(Expr(:boundscheck)), A, x isa T ? x : convert(T,x)::T, i1, i2, I...)) + (@inline; arrayset(Core.should_check_bounds($(Expr(:boundscheck))), A, x isa T ? x : convert(T,x)::T, i1, i2, I...)) __inbounds_setindex!(A::Array{T}, x, i1::Int) where {T} = arrayset(false, A, convert(T,x)::T, i1) diff --git a/base/boot.jl b/base/boot.jl index 78b7daaf47d64..8a1bd21eb4aed 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -854,4 +854,6 @@ function _hasmethod(@nospecialize(tt)) # this function has a special tfunc return Intrinsics.not_int(ccall(:jl_gf_invoke_lookup, Any, (Any, Any, UInt), tt, nothing, world) === nothing) end +should_check_bounds(boundscheck::Bool) = boundscheck + ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true) diff --git a/base/client.jl b/base/client.jl index 6e30c9991e45e..f7fe7b2d3fad7 100644 --- a/base/client.jl +++ b/base/client.jl @@ -272,6 +272,15 @@ function exec_options(opts) invokelatest(Main.Distributed.process_opts, opts) end + # Maybe redefine bounds checking if requested + if JLOptions().check_bounds != 0 + if JLOptions().check_bounds == 1 + Core.eval(Main, :(Core.should_check_bounds(boundscheck::Bool) = true)) + else + Core.eval(Main, :(Core.should_check_bounds(boundscheck::Bool) = false)) + end + end + interactiveinput = (repl || is_interactive::Bool) && isa(stdin, TTY) is_interactive::Bool |= interactiveinput diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index ed41b43ff95d9..0ec0fb21839b2 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -835,13 +835,6 @@ end function concrete_eval_eligible(interp::AbstractInterpreter, @nospecialize(f), result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState) (;effects) = result - if inbounds_option() === :off - if !is_nothrow(effects) - # Disable concrete evaluation in `--check-bounds=no` mode, - # unless it is known to not throw. - return :none - end - end if !effects.noinbounds && stmt_taints_inbounds_consistency(sv) # If the current statement is @inbounds or we propagate inbounds, the call's consistency # is tainted and not consteval eligible. diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 338016e53517e..9adc30b05ada2 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -652,8 +652,8 @@ function batch_inline!(ir::IRCode, todo::Vector{Pair{Int,Any}}, propagate_inboun end finish_cfg_inline!(state) - boundscheck = inbounds_option() - if boundscheck === :default && propagate_inbounds + boundscheck = :default + if propagate_inbounds boundscheck = :propagate end diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index f3c5694535ce6..79e21f9b9f14c 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -513,9 +513,3 @@ function coverage_enabled(m::Module) end return false end -function inbounds_option() - opt_check_bounds = JLOptions().check_bounds - opt_check_bounds == 0 && return :default - opt_check_bounds == 1 && return :on - return :off -end diff --git a/base/essentials.jl b/base/essentials.jl index 68dd0c06d646f..3fe913f029a2d 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -import Core: CodeInfo, SimpleVector, donotdelete, compilerbarrier, arrayref +import Core: CodeInfo, SimpleVector, donotdelete, compilerbarrier, arrayref, should_check_bounds const Callable = Union{Function,Type} @@ -10,8 +10,8 @@ const Bottom = Union{} length(a::Array) = arraylen(a) # This is more complicated than it needs to be in order to get Win64 through bootstrap -eval(:(getindex(A::Array, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1))) -eval(:(getindex(A::Array, i1::Int, i2::Int, I::Int...) = (@inline; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)))) +eval(:(getindex(A::Array, i1::Int) = arrayref(should_check_bounds($(Expr(:boundscheck))), A, i1))) +eval(:(getindex(A::Array, i1::Int, i2::Int, I::Int...) = (@inline; arrayref(should_check_bounds($(Expr(:boundscheck))), A, i1, i2, I...)))) ==(a::GlobalRef, b::GlobalRef) = a.mod === b.mod && a.name === b.name @@ -698,7 +698,7 @@ julia> f2() implementation after you are certain its behavior is correct. """ macro boundscheck(blk) - return Expr(:if, Expr(:boundscheck), esc(blk)) + return Expr(:if, Expr(:call, GlobalRef(Core, :should_check_bounds), Expr(:boundscheck)), esc(blk)) end """ diff --git a/base/experimental.jl b/base/experimental.jl index cc8d368023b49..4a7b952ede9a9 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -29,9 +29,9 @@ Base.IndexStyle(::Type{<:Const}) = IndexLinear() Base.size(C::Const) = size(C.a) Base.axes(C::Const) = axes(C.a) @eval Base.getindex(A::Const, i1::Int) = - (Base.@inline; Core.const_arrayref($(Expr(:boundscheck)), A.a, i1)) + (Base.@inline; Core.const_arrayref(Core.should_check_bounds($(Expr(:boundscheck))), A.a, i1)) @eval Base.getindex(A::Const, i1::Int, i2::Int, I::Int...) = - (Base.@inline; Core.const_arrayref($(Expr(:boundscheck)), A.a, i1, i2, I...)) + (Base.@inline; Core.const_arrayref(Core.should_check_bounds($(Expr(:boundscheck))), A.a, i1, i2, I...)) """ @aliasscope expr diff --git a/base/tuple.jl b/base/tuple.jl index 59fe2c1e531e1..33aa947c5fd82 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -28,8 +28,8 @@ firstindex(@nospecialize t::Tuple) = 1 lastindex(@nospecialize t::Tuple) = length(t) size(@nospecialize(t::Tuple), d::Integer) = (d == 1) ? length(t) : throw(ArgumentError("invalid tuple dimension $d")) axes(@nospecialize t::Tuple) = (OneTo(length(t)),) -@eval getindex(@nospecialize(t::Tuple), i::Int) = getfield(t, i, $(Expr(:boundscheck))) -@eval getindex(@nospecialize(t::Tuple), i::Integer) = getfield(t, convert(Int, i), $(Expr(:boundscheck))) +@eval getindex(@nospecialize(t::Tuple), i::Int) = getfield(t, i, should_check_bounds($(Expr(:boundscheck)))) +@eval getindex(@nospecialize(t::Tuple), i::Integer) = getfield(t, convert(Int, i), should_check_bounds($(Expr(:boundscheck)))) __inbounds_getindex(@nospecialize(t::Tuple), i::Int) = getfield(t, i, false) __inbounds_getindex(@nospecialize(t::Tuple), i::Integer) = getfield(t, convert(Int, i), false) getindex(t::Tuple, r::AbstractArray{<:Any,1}) = (eltype(t)[t[ri] for ri in r]...,) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 9b3d634c76400..11b976b6f70a4 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1736,26 +1736,10 @@ static void emit_concretecheck(jl_codectx_t &ctx, Value *typ, const std::string error_unless(ctx, emit_isconcrete(ctx, typ), msg); } -#define CHECK_BOUNDS 1 -static bool bounds_check_enabled(jl_codectx_t &ctx, jl_value_t *inbounds) { -#if CHECK_BOUNDS==1 - if (jl_options.check_bounds == JL_OPTIONS_CHECK_BOUNDS_ON) - return 1; - if (jl_options.check_bounds == JL_OPTIONS_CHECK_BOUNDS_OFF) - return 0; - if (inbounds == jl_false) - return 0; - return 1; -#else - return 0; -#endif -} - static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_value_t *ty, Value *i, Value *len, jl_value_t *boundscheck) { Value *im1 = ctx.builder.CreateSub(i, ConstantInt::get(ctx.types().T_size, 1)); -#if CHECK_BOUNDS==1 - if (bounds_check_enabled(ctx, boundscheck)) { + if (boundscheck != jl_false) { ++EmittedBoundschecks; Value *ok = ctx.builder.CreateICmpULT(im1, len); setName(ctx.emission_context, ok, "boundscheck"); @@ -1790,7 +1774,6 @@ static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_v ctx.f->getBasicBlockList().push_back(passBB); ctx.builder.SetInsertPoint(passBB); } -#endif return im1; } @@ -3031,14 +3014,12 @@ static Value *emit_array_nd_index( Value *a = boxed(ctx, ainfo); Value *i = Constant::getNullValue(ctx.types().T_size); Value *stride = ConstantInt::get(ctx.types().T_size, 1); -#if CHECK_BOUNDS==1 - bool bc = bounds_check_enabled(ctx, inbounds); + bool bc = inbounds != jl_false; BasicBlock *failBB = NULL, *endBB = NULL; if (bc) { failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); } -#endif SmallVector idxs(nidxs); for (size_t k = 0; k < nidxs; k++) { idxs[k] = emit_unbox(ctx, ctx.types().T_size, argv[k], (jl_value_t*)jl_long_type); // type asserted by caller @@ -3050,7 +3031,6 @@ static Value *emit_array_nd_index( if (k < nidxs - 1) { assert(nd >= 0); Value *d = emit_arraysize_for_unsafe_dim(ctx, ainfo, ex, k + 1, nd); -#if CHECK_BOUNDS==1 if (bc) { BasicBlock *okBB = BasicBlock::Create(ctx.builder.getContext(), "ib"); // if !(i < d) goto error @@ -3060,12 +3040,10 @@ static Value *emit_array_nd_index( ctx.f->getBasicBlockList().push_back(okBB); ctx.builder.SetInsertPoint(okBB); } -#endif stride = ctx.builder.CreateMul(stride, d); setName(ctx.emission_context, stride, "stride"); } } -#if CHECK_BOUNDS==1 if (bc) { // We have already emitted a bounds check for each index except for // the last one which we therefore have to do here. @@ -3126,7 +3104,6 @@ static Value *emit_array_nd_index( ctx.f->getBasicBlockList().push_back(endBB); ctx.builder.SetInsertPoint(endBB); } -#endif return i; } diff --git a/src/codegen.cpp b/src/codegen.cpp index 669cf326f1a2d..405a01d75b9db 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3817,7 +3817,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, assert(jl_is_datatype(jt)); // This is not necessary for correctness, but allows to omit // the extra code for getting the length of the tuple - if (!bounds_check_enabled(ctx, boundscheck)) { + if (boundscheck == jl_false) { vidx = ctx.builder.CreateSub(vidx, ConstantInt::get(ctx.types().T_size, 1)); } else { @@ -5772,7 +5772,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ jl_errorf("Expr(:%s) in value position", jl_symbol_name(head)); } else if (head == jl_boundscheck_sym) { - return mark_julia_const(ctx, bounds_check_enabled(ctx, jl_true) ? jl_true : jl_false); + return mark_julia_const(ctx, jl_true); } else if (head == jl_gc_preserve_begin_sym) { SmallVector argv(nargs); diff --git a/test/loading.jl b/test/loading.jl index 394c13c5f2962..c432da21476f2 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1096,25 +1096,17 @@ pkgimage(val) = val == 1 ? `--pkgimage=yes` : `--pkgimage=no` opt_level(val) = `-O$val` debug_level(val) = `-g$val` inline(val) = val == 1 ? `--inline=yes` : `--inline=no` -check_bounds(val) = if val == 0 - `--check-bounds=auto` -elseif val == 1 - `--check-bounds=yes` -elseif val == 2 - `--check-bounds=no` -end @testset "CacheFlags" begin cf = Base.CacheFlags() opts = Base.JLOptions() @test cf.use_pkgimages == opts.use_pkgimages @test cf.debug_level == opts.debug_level - @test cf.check_bounds == opts.check_bounds @test cf.inline == opts.can_inline @test cf.opt_level == opts.opt_level # OOICCDDP - for (P, D, C, I, O) in Iterators.product(0:1, 0:2, 0:2, 0:1, 0:3) + for (P, D, I, O) in Iterators.product(0:1, 0:2, 0:1, 0:3) julia = joinpath(Sys.BINDIR, Base.julia_exename()) script = """ let @@ -1122,25 +1114,23 @@ end opts = Base.JLOptions() cf.use_pkgimages == opts.use_pkgimages == $P || error("use_pkgimages") cf.debug_level == opts.debug_level == $D || error("debug_level") - cf.check_bounds == opts.check_bounds == $C || error("check_bounds") cf.inline == opts.can_inline == $I || error("inline") cf.opt_level == opts.opt_level == $O || error("opt_level") end """ - cmd = `$julia $(pkgimage(P)) $(opt_level(O)) $(debug_level(D)) $(check_bounds(C)) $(inline(I)) -e $script` + cmd = `$julia $(pkgimage(P)) $(opt_level(O)) $(debug_level(D)) $(inline(I)) -e $script` @test success(pipeline(cmd; stdout, stderr)) end cf = Base.CacheFlags(255) @test cf.use_pkgimages @test cf.debug_level == 3 - @test cf.check_bounds == 3 @test cf.inline @test cf.opt_level == 3 io = PipeBuffer() show(io, cf) - @test read(io, String) == "use_pkgimages = true, debug_level = 3, check_bounds = 3, inline = true, opt_level = 3" + @test read(io, String) == "use_pkgimages = true, debug_level = 3, is_mandatory = 1, inline = true, opt_level = 3" end empty!(Base.DEPOT_PATH) From 550e5d0baac464b3ac0f53c2a6e5e1f6f746c330 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 18 Jul 2023 19:10:18 +0000 Subject: [PATCH 2/4] Add support for mandatory dependencies --- base/loading.jl | 82 +++++++++++++++++++++++++++++++++--------- pkgimage.mk | 4 +-- src/gf.c | 8 +++-- src/julia_internal.h | 2 +- src/method.c | 2 +- src/staticdata.c | 14 ++++---- src/staticdata_utils.c | 17 ++++++--- 7 files changed, 96 insertions(+), 33 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 3c672d075523d..facc1d4fbd684 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1057,12 +1057,15 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No if isa(sv, Exception) return sv end - restored = register_restored_modules(sv, pkg, path) + ismandatory = sv[3] for M in restored M = M::Module if parentmodule(M) === M && PkgId(M) == pkg + if ismandatory + push!(_mandatory_dependencies, PkgId(M) => module_build_id(M)) + end if timing_imports elapsed = round((time_ns() - t_before) / 1e6, digits = 1) comp_time, recomp_time = cumulative_compile_time_ns() .- t_comp_before @@ -1388,7 +1391,7 @@ function isprecompiled(pkg::PkgId; ) isnothing(sourcepath) && error("Cannot locate source for $(repr(pkg))") for path_to_try in cachepaths - staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true) + staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true, allow_new_mandatory = true) if staledeps === true continue end @@ -1531,7 +1534,7 @@ end assert_havelock(require_lock) paths = find_all_in_cache_path(pkg) for path_to_try in paths::Vector{String} - staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try) + staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try, allow_new_mandatory = true) if staledeps === true continue end @@ -1653,6 +1656,7 @@ const include_callbacks = Any[] # used to optionally track dependencies when requiring a module: const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them +const _mandatory_dependencies = Pair{PkgId,UInt128}[] # Like `_concrete_dependencies`, but required to actually be loaded (in order) in every precompile process const _require_dependencies = Any[] # a list of (mod, path, mtime) tuples that are the file dependencies of the module currently being precompiled const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies function _include_dependency(mod::Module, _path::AbstractString) @@ -1709,6 +1713,17 @@ order to throw an error if Julia attempts to precompile it. nothing end +function __precompile__(setting::Symbol) + if setting == :mandatory + if ccall(:jl_generating_output, Cint, ()) == 0 + @warn "Mandatory precompilation outside of precompile - loading of precompile will be disabled for all future modules." + push!(_mandatory_dependencies, PkgId(__toplevel__)=>UInt128(0)) + else + ccall(:jl_set_ismandatory, Cvoid, (Int8,), 0x1) + end + end +end + # require always works in Main scope and loads files from node 1 const toplevel_load = Ref(true) @@ -2194,7 +2209,7 @@ end # this is called in the external process that generates precompiled package files function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, - concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String}) + concrete_deps::typeof(_concrete_dependencies), mandatory_deps::typeof(_mandatory_dependencies), source::Union{Nothing,String}) append!(empty!(Base.DEPOT_PATH), depot_path) append!(empty!(Base.DL_LOAD_PATH), dl_load_path) append!(empty!(Base.LOAD_PATH), load_path) @@ -2210,6 +2225,10 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto task_local_storage()[:SOURCE_PATH] = source end + for (dep, build_id) in mandatory_deps + Base.require(dep) + end + ccall(:jl_set_newly_inferred, Cvoid, (Any,), Core.Compiler.newly_inferred) Core.Compiler.track_newly_inferred.x = true try @@ -2225,7 +2244,7 @@ end const PRECOMPILE_TRACE_COMPILE = Ref{String}() function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, - concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) + concrete_deps::typeof(_concrete_dependencies), mandatory_deps::typeof(_mandatory_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) @nospecialize internal_stderr internal_stdout rm(output, force=true) # Remove file if it exists output_o === nothing || rm(output_o, force=true) @@ -2236,7 +2255,8 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: any(path -> path_sep in path, load_path) && error("LOAD_PATH entries cannot contain $(repr(path_sep))") - deps_strs = String[] + concrete_deps_strs = String[] + mandatory_deps_strs = String[] function pkg_str(_pkg::PkgId) if _pkg.uuid === nothing "Base.PkgId($(repr(_pkg.name)))" @@ -2245,7 +2265,10 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: end end for (pkg, build_id) in concrete_deps - push!(deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))") + push!(concrete_deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))") + end + for (pkg, build_id) in mandatory_deps + push!(mandatory_deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))") end if output_o !== nothing @@ -2258,7 +2281,8 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: end deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) - deps = deps_eltype * "[" * join(deps_strs, ",") * "]" + concrete_deps_child = deps_eltype * "[" * join(concrete_deps_strs, ",") * "]" + mandatory_deps_child = deps_eltype * "[" * join(mandatory_deps_strs, ",") * "]" trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : `` io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts) --startup-file=no --history-file=no --warn-overwrite=yes @@ -2274,7 +2298,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: empty!(Base.EXT_DORMITORY) # If we have a custom sysimage with `EXT_DORMITORY` prepopulated Base.precompiling_extension = $(loading_extension) Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), - $(repr(load_path)), $deps, $(repr(source_path(nothing)))) + $(repr(load_path)), $concrete_deps_child, $mandatory_deps_child, $(repr(source_path(nothing)))) """) close(io.in) return io @@ -2365,7 +2389,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in close(tmpio_o) close(tmpio_so) end - p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, internal_stderr, internal_stdout) + p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, _mandatory_dependencies, internal_stderr, internal_stdout) if success(p) if cache_objects @@ -2853,17 +2877,17 @@ struct CacheFlags # OOICCDDP - see jl_cache_flags use_pkgimages::Bool debug_level::Int - check_bounds::Int + is_mandatory::Int inline::Bool opt_level::Int function CacheFlags(f::UInt8) use_pkgimages = Bool(f & 1) debug_level = Int((f >> 1) & 3) - check_bounds = Int((f >> 3) & 3) + is_mandatory = Int((f >> 3) & 1) inline = Bool((f >> 5) & 1) opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils - new(use_pkgimages, debug_level, check_bounds, inline, opt_level) + new(use_pkgimages, debug_level, is_mandatory, inline, opt_level) end end CacheFlags(f::Int) = CacheFlags(UInt8(f)) @@ -2872,7 +2896,7 @@ CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ())) function show(io::IO, cf::CacheFlags) print(io, "use_pkgimages = ", cf.use_pkgimages) print(io, ", debug_level = ", cf.debug_level) - print(io, ", check_bounds = ", cf.check_bounds) + print(io, ", is_mandatory = ", cf.is_mandatory) print(io, ", inline = ", cf.inline) print(io, ", opt_level = ", cf.opt_level) end @@ -2917,10 +2941,10 @@ end # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey # otherwise returns the list of dependencies to also check -@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false) +@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, allow_new_mandatory::Bool = false) return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded) end -@constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; ignore_loaded::Bool = false) +@constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; ignore_loaded::Bool = false, allow_new_mandatory::Bool = false) io = open(cachefile, "r") try checksum = isvalid_cache_header(io) @@ -2940,6 +2964,17 @@ end """ return true end + if CacheFlags(flags).is_mandatory != 0x0 && !allow_new_mandatory + # Check whether this is in the current list of mandatory packages + for (mandatory_modkey, mandatory_build_id) in _mandatory_dependencies + if mandatory_modkey == modkey && mandatory_build_id == build_id + @goto mandatory_ok + end + end + @debug "Rejecting cache file $cachefile for $modkey since it is a mandatory package that we have not loaded" + return true + @label mandatory_ok + end pkgimage = !isempty(clone_targets) if pkgimage ocachefile = ocachefile_from_cachefile(cachefile) @@ -2974,11 +3009,21 @@ end id = id.first modules = Dict{PkgId, UInt64}(modules) + mandatory_deps = Dict{PkgId, UInt128}(_mandatory_dependencies) + # Check if transitive dependencies can be fulfilled ndeps = length(required_modules) depmods = Vector{Any}(undef, ndeps) for i in 1:ndeps req_key, req_build_id = required_modules[i] + if haskey(mandatory_deps, req_key) + mandatory_build_id = mandatory_deps[req_key] + if req_build_id != mandatory_build_id + @debug "Rejecting cache file $cachefile because module $req_key build id $req_build_id does not match mandatory build id $mandatory_build_id." + return true # Mandatory dependency has bad version + end + delete!(mandatory_deps, req_key) + end # Module is already loaded if root_module_exists(req_key) M = root_module(req_key) @@ -3002,6 +3047,11 @@ end end end + if !isempty(mandatory_deps) + @debug "Rejecting cache file $cachefile because mandatory dependencies $(keys(mandatory_deps)) are unfulfilled." + return true # Missing mandatory dependency + end + # check if this file is going to provide one of our concrete dependencies # or if it provides a version that conflicts with our concrete dependencies # or neither diff --git a/pkgimage.mk b/pkgimage.mk index 0803a188851bb..c9d1cc826dff1 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -36,12 +36,12 @@ define pkgimg_builder $1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \ $$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1) $$(BUILDDIR)/stdlib/$1.release.image: $$($1_SRCS) $$(addsuffix .release.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys.$(SHLIB_EXT) - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') +# @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') touch $$@ cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image: $$($1_SRCS) $$(addsuffix .debug.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') +# @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image .SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image diff --git a/src/gf.c b/src/gf.c index 935fd1e60db78..807d989442960 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1541,8 +1541,11 @@ static int is_anonfn_typename(char *name) return other > &name[1] && other[1] > '0' && other[1] <= '9'; } +extern int8_t cache_is_mandatory; static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue) { + if (cache_is_mandatory) + return; // method overwritten jl_method_t *method = (jl_method_t*)newentry->func.method; jl_module_t *newmod = method->module; @@ -1980,7 +1983,7 @@ static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_ return 1; } -JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype) +JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype, int nowarn_overwrite) { JL_TIMING(ADD_METHOD, ADD_METHOD); assert(jl_is_method(method)); @@ -2010,7 +2013,8 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method if (replaced) { oldvalue = (jl_value_t*)replaced; invalidated = 1; - method_overwrite(newentry, replaced->func.method); + if (!nowarn_overwrite) + method_overwrite(newentry, replaced->func.method); jl_method_table_invalidate(mt, replaced, max_world); } else { diff --git a/src/julia_internal.h b/src/julia_internal.h index cf65521770681..4027db9b31c6c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -683,7 +683,7 @@ int jl_subtype_invariant(jl_value_t *a, jl_value_t *b, int ta); int jl_has_concrete_subtype(jl_value_t *typ); jl_tupletype_t *jl_inst_arg_tuple_type(jl_value_t *arg1, jl_value_t **args, size_t nargs, int leaf); jl_tupletype_t *jl_lookup_arg_tuple_type(jl_value_t *arg1 JL_PROPAGATES_ROOT, jl_value_t **args, size_t nargs, int leaf); -JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); +JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype, int nowarn_overwrite); jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED; int jl_obviously_unequal(jl_value_t *a, jl_value_t *b); JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v); diff --git a/src/method.c b/src/method.c index 06a05361a927d..0a230574461a9 100644 --- a/src/method.c +++ b/src/method.c @@ -1109,7 +1109,7 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, jl_array_ptr_1d_push(jl_all_methods, (jl_value_t*)m); } - jl_method_table_insert(mt, m, NULL); + jl_method_table_insert(mt, m, NULL, 0); if (jl_newmeth_tracer) jl_call_tracer(jl_newmeth_tracer, (jl_value_t*)m); JL_GC_POP(); diff --git a/src/staticdata.c b/src/staticdata.c index c05422fd10969..e356c5aab86b8 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3344,7 +3344,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl jl_gc_enable(en); } -static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_t *checksum, int64_t *dataendpos, int64_t *datastartpos) +static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_t *checksum, int64_t *dataendpos, int64_t *datastartpos, int8_t *ismandatory) { uint8_t pkgimage = 0; if (ios_eof(f) || 0 == (*checksum = jl_read_verify_header(f, &pkgimage, dataendpos, datastartpos)) || (*checksum >> 32 != 0xfafbfcfd)) { @@ -3355,6 +3355,7 @@ static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_ if (pkgimage && !jl_match_cache_flags(flags)) { return jl_get_exceptionf(jl_errorexception_type, "Pkgimage flags mismatch"); } + *ismandatory |= !!(flags & (1 << 3)); if (!pkgimage) { // skip past the worklist size_t len; @@ -3378,7 +3379,8 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im uint64_t checksum = 0; int64_t dataendpos = 0; int64_t datastartpos = 0; - jl_value_t *verify_fail = jl_validate_cache_file(f, depmods, &checksum, &dataendpos, &datastartpos); + int8_t ismandatory = 0; + jl_value_t *verify_fail = jl_validate_cache_file(f, depmods, &checksum, &dataendpos, &datastartpos, &ismandatory); if (verify_fail) return verify_fail; @@ -3418,7 +3420,7 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im JL_SIGATOMIC_END(); // Insert method extensions - jl_insert_methods(extext_methods); + jl_insert_methods(extext_methods, ismandatory); // No special processing of `new_specializations` is required because recaching handled it // Add roots to methods jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored)); @@ -3430,7 +3432,7 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im arraylist_free(&ccallable_list); if (completeinfo) { - cachesizes_sv = jl_alloc_svec(7); + cachesizes_sv = jl_alloc_svec(8); jl_svecset(cachesizes_sv, 0, jl_box_long(cachesizes.sysdata)); jl_svecset(cachesizes_sv, 1, jl_box_long(cachesizes.isbitsdata)); jl_svecset(cachesizes_sv, 2, jl_box_long(cachesizes.symboldata)); @@ -3438,11 +3440,11 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im jl_svecset(cachesizes_sv, 4, jl_box_long(cachesizes.reloclist)); jl_svecset(cachesizes_sv, 5, jl_box_long(cachesizes.gvarlist)); jl_svecset(cachesizes_sv, 6, jl_box_long(cachesizes.fptrlist)); - restored = (jl_value_t*)jl_svec(8, restored, init_order, extext_methods, new_specializations, method_roots_list, + restored = (jl_value_t*)jl_svec(9, restored, init_order, ismandatory ? jl_true : jl_false, extext_methods, new_specializations, method_roots_list, ext_targets, edges, cachesizes_sv); } else { - restored = (jl_value_t*)jl_svec(2, restored, init_order); + restored = (jl_value_t*)jl_svec(3, restored, init_order, ismandatory ? jl_true : jl_false); } } } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index bf1a830b608de..8af5a2e761fa8 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -597,18 +597,25 @@ static void write_mod_list(ios_t *s, jl_array_t *a) // OPT_LEVEL should always be the upper bits #define OPT_LEVEL 6 +int8_t cache_is_mandatory = 0; JL_DLLEXPORT uint8_t jl_cache_flags(void) { // OOICCDDP uint8_t flags = 0; flags |= (jl_options.use_pkgimages & 1); // 0-bit flags |= (jl_options.debug_level & 3) << 1; // 1-2 bit - flags |= (jl_options.check_bounds & 3) << 3; // 3-4 bit + flags |= (cache_is_mandatory & 0x1) << 3; + // bit 4 unused flags |= (jl_options.can_inline & 1) << 5; // 5-bit flags |= (jl_options.opt_level & 3) << OPT_LEVEL; // 6-7 bit return flags; } +JL_DLLEXPORT void jl_set_ismandatory(int8_t is_mandatory) +{ + cache_is_mandatory = is_mandatory; +} + JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t flags) { // 1. Check which flags are relevant @@ -621,8 +628,8 @@ JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t flags) return 1; } - // 2. Check all flags, execept opt level must be exact - uint8_t mask = (1 << OPT_LEVEL)-1; + // 2. Check all flags, execept opt level and ismandatory must be exact + uint8_t mask = ((1 << OPT_LEVEL)-1) & (~(1 << 3)); if ((flags & mask) != (current_flags & mask)) return 0; // 3. allow for higher optimization flags in cache @@ -806,7 +813,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t // Deserialization // Add methods to external (non-worklist-owned) functions -static void jl_insert_methods(jl_array_t *list) +static void jl_insert_methods(jl_array_t *list, int nowarn_overwrite) { size_t i, l = jl_array_len(list); for (i = 0; i < l; i++) { @@ -815,7 +822,7 @@ static void jl_insert_methods(jl_array_t *list) assert(!meth->is_for_opaque_closure); jl_methtable_t *mt = jl_method_get_table(meth); assert((jl_value_t*)mt != jl_nothing); - jl_method_table_insert(mt, meth, NULL); + jl_method_table_insert(mt, meth, NULL, nowarn_overwrite); } } From 56893dfc5414f4e985f7964258b4cf01c7919c23 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 18 Jul 2023 19:33:19 +0000 Subject: [PATCH 3/4] Import PrecompileTools to Base --- base/Base.jl | 3 ++ base/invalidations.jl | 70 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 base/invalidations.jl diff --git a/base/Base.jl b/base/Base.jl index 0ec70add2a2c4..906b9f82557a1 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -475,6 +475,9 @@ if isdefined(Core, :Compiler) && is_primary_base_module Docs.loaddocs(Core.Compiler.CoreDocs.DOCS) end +# Formerly PrecompileTools +include("invalidations.jl") + # finally, now make `include` point to the full version for m in methods(include) delete_method(m) diff --git a/base/invalidations.jl b/base/invalidations.jl new file mode 100644 index 0000000000000..299db439a11ab --- /dev/null +++ b/base/invalidations.jl @@ -0,0 +1,70 @@ +""" + @recompile_invalidations begin + using PkgA + ⋮ + end + +Recompile any invalidations that occur within the given expression. This is generally intended to be used +by users in creating "Startup" packages to ensure that the code compiled by package authors is not invalidated. +""" +macro recompile_invalidations(expr) + list = gensym(:list) + Expr(:toplevel, + :($list = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1)), + Expr(:tryfinally, + esc(expr), + :(ccall(:jl_debug_method_invalidation, Any, (Cint,), 0)) + ), + :(if ccall(:jl_generating_output, Cint, ()) == 1 + foreach($precompile_mi, $invalidation_leaves($list)) + end) + ) +end + +function precompile_mi(mi) + precompile(mi.specTypes) # TODO: Julia should allow one to pass `mi` directly (would handle `invoke` properly) + return +end + +function invalidation_leaves(invlist) + umis = Set{Core.MethodInstance}() + # `queued` is a queue of length 0 or 1 of invalidated MethodInstances. + # We wait to read the `depth` to find out if it's a leaf. + queued, depth = nothing, 0 + function cachequeued(item, nextdepth) + if queued !== nothing && nextdepth <= depth + push!(umis, queued) + end + queued, depth = item, nextdepth + end + + i, ilast = firstindex(invlist), lastindex(invlist) + while i <= ilast + item = invlist[i] + if isa(item, Core.MethodInstance) + if i < lastindex(invlist) + nextitem = invlist[i+1] + if nextitem == "invalidate_mt_cache" + cachequeued(nothing, 0) + i += 2 + continue + end + if nextitem ∈ ("jl_method_table_disable", "jl_method_table_insert", "verify_methods") + cachequeued(nothing, 0) + push!(umis, item) + end + if isa(nextitem, Integer) + cachequeued(item, nextitem) + i += 2 + continue + end + end + end + if (isa(item, Method) || isa(item, Type)) && queued !== nothing + push!(umis, queued) + queued, depth = nothing, 0 + end + i += 1 + end + return umis +end From d33ceced622fca29b3503aeb4b1ea741cb5f2c14 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 18 Jul 2023 19:34:28 +0000 Subject: [PATCH 4/4] Add checkbounds switch stdlibs --- base/client.jl | 13 +++++--- base/loading.jl | 2 +- pkgimage.mk | 33 ++++++++++++------- stdlib/--check-bounds=no/Project.toml | 2 ++ .../src/--check-bounds=no.jl | 8 +++++ stdlib/--check-bounds=yes/Project.toml | 2 ++ .../src/--check-bounds=yes.jl | 8 +++++ stdlib/Makefile | 2 +- 8 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 stdlib/--check-bounds=no/Project.toml create mode 100644 stdlib/--check-bounds=no/src/--check-bounds=no.jl create mode 100644 stdlib/--check-bounds=yes/Project.toml create mode 100644 stdlib/--check-bounds=yes/src/--check-bounds=yes.jl diff --git a/base/client.jl b/base/client.jl index f7fe7b2d3fad7..7fa89f73ac5ba 100644 --- a/base/client.jl +++ b/base/client.jl @@ -273,11 +273,14 @@ function exec_options(opts) end # Maybe redefine bounds checking if requested - if JLOptions().check_bounds != 0 - if JLOptions().check_bounds == 1 - Core.eval(Main, :(Core.should_check_bounds(boundscheck::Bool) = true)) - else - Core.eval(Main, :(Core.should_check_bounds(boundscheck::Bool) = false)) + if ccall(:jl_generating_output, Cint, ()) == 0 + # Inoperative during output generation. Determined by mandatory deps mechanism. + if JLOptions().check_bounds != 0 + if JLOptions().check_bounds == 1 + require(PkgId(UUID((0xb3a877e5_8181_4b4a, 0x8173_0b9cb13136fe)),"--check-bounds=yes")) + else + require(PkgId(UUID((0x5ece1bc4_2007_43a8, 0xac47_40059be74678)),"--check-bounds=no")) + end end end diff --git a/base/loading.jl b/base/loading.jl index facc1d4fbd684..1c95b93dc5662 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2924,7 +2924,7 @@ function maybe_cachefile_lock(f, pkg::PkgId, srcpath::String; stale_age=300) pid, hostname, age = invokelatest(parse_pidfile_hook, pidfile) verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug if isempty(hostname) || hostname == gethostname() - @logmsg verbosity "Waiting for another process (pid: $pid) to finish precompiling $pkg" + @logmsg verbosity "Waiting for another process (pid: $pid) to finish precompiling $pkg (pidfile: $pidfile)" else @logmsg verbosity "Waiting for another machine (hostname: $hostname, pid: $pid) to finish precompiling $pkg" end diff --git a/pkgimage.mk b/pkgimage.mk index c9d1cc826dff1..f0e83101db92f 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -33,18 +33,24 @@ all-release: $(addprefix cache-release-, $(STDLIBS)) all-debug: $(addprefix cache-debug-, $(STDLIBS)) define pkgimg_builder -$1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \ +PKGIMG_SRCS := $$(shell find "$$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src" -name \*.jl) \ $$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1) -$$(BUILDDIR)/stdlib/$1.release.image: $$($1_SRCS) $$(addsuffix .release.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys.$(SHLIB_EXT) -# @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') +PKGIMG_SENTINEL_NAME = $(subst =,_EQ_,$1) +$$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).release.image: $$(PKGIMG_SRCS) $$(addsuffix .release.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys.$(SHLIB_EXT) @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') touch $$@ -cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image -$$(BUILDDIR)/stdlib/$1.debug.image: $$($1_SRCS) $$(addsuffix .debug.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) -# @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') +$$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).release.checkbounds.image: $$(PKGIMG_SRCS) $$(addsuffix .release.checkbounds.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $$(BUILDDIR)/stdlib/--check-bounds_EQ_yes.release.image $(build_private_libdir)/sys.$(SHLIB_EXT) + @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') + touch $$@ +cache-release-$$(PKGIMG_SENTINEL_NAME): $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.release.checkbounds.image +$$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).debug.image: $$(PKGIMG_SRCS) $$(addsuffix .debug.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') -cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image -.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image + touch $$@ +$$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).debug.checkbounds.image: $$(PKGIMG_SRCS) $$(addsuffix .debug.checkbounds.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $$(BUILDDIR)/stdlib/--check-bounds_EQ_yes.debug.image $(build_private_libdir)/sys-debug.$(SHLIB_EXT) + @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') + touch $$@ +cache-debug-$$(PKGIMG_SENTINEL_NAME): $$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).debug.image $$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).debug.checkbounds.image +.SECONDARY: $$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).release.image $$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).release.checkbounds.image $$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).debug.image $$(BUILDDIR)/stdlib/$$(PKGIMG_SENTINEL_NAME).debug.checkbounds.image endef # Used to just define them in the dependency graph @@ -52,14 +58,19 @@ endef define sysimg_builder $$(BUILDDIR)/stdlib/$1.release.image: touch $$@ -cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image +$$(BUILDDIR)/stdlib/$1.release.checkbounds.image: + touch $$@ +cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.release.checkbounds.image $$(BUILDDIR)/stdlib/$1.debug.image: touch $$@ -cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image -.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image +$$(BUILDDIR)/stdlib/$1.debug.checkbounds.image: + touch $$@ +cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image $$(BUILDDIR)/stdlib/$1.debug.checkbounds.image +.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.release.checkbounds.image $$(BUILDDIR)/stdlib/$1.debug.image $$(BUILDDIR)/stdlib/$1.debug.checkbounds.image endef # no dependencies +$(eval $(call pkgimg_builder,--check-bounds=yes,)) $(eval $(call pkgimg_builder,MozillaCACerts_jll,)) $(eval $(call sysimg_builder,ArgTools,)) $(eval $(call sysimg_builder,Artifacts,)) diff --git a/stdlib/--check-bounds=no/Project.toml b/stdlib/--check-bounds=no/Project.toml new file mode 100644 index 0000000000000..f5eb68491a096 --- /dev/null +++ b/stdlib/--check-bounds=no/Project.toml @@ -0,0 +1,2 @@ +name = "--check-bounds=no" +uuid = "5ece1bc4-2007-43a8-ac47-40059be74678" diff --git a/stdlib/--check-bounds=no/src/--check-bounds=no.jl b/stdlib/--check-bounds=no/src/--check-bounds=no.jl new file mode 100644 index 0000000000000..161ac88fdded4 --- /dev/null +++ b/stdlib/--check-bounds=no/src/--check-bounds=no.jl @@ -0,0 +1,8 @@ +__precompile__(:mandatory) +module var"--check-bounds=no" + +@Base.recompile_invalidations begin + Core.should_check_bounds(boundscheck::Bool) = false +end + +end diff --git a/stdlib/--check-bounds=yes/Project.toml b/stdlib/--check-bounds=yes/Project.toml new file mode 100644 index 0000000000000..81f1391f1478a --- /dev/null +++ b/stdlib/--check-bounds=yes/Project.toml @@ -0,0 +1,2 @@ +name = "--check-bounds=yes" +uuid = "b3a877e5-8181-4b4a-8173-0b9cb13136fe" \ No newline at end of file diff --git a/stdlib/--check-bounds=yes/src/--check-bounds=yes.jl b/stdlib/--check-bounds=yes/src/--check-bounds=yes.jl new file mode 100644 index 0000000000000..4a8dba504fee5 --- /dev/null +++ b/stdlib/--check-bounds=yes/src/--check-bounds=yes.jl @@ -0,0 +1,8 @@ +__precompile__(:mandatory) +module var"--check-bounds=yes" + +@Base.recompile_invalidations begin + Core.should_check_bounds(boundscheck::Bool) = true +end + +end diff --git a/stdlib/Makefile b/stdlib/Makefile index e42061d593905..5dccd4b628931 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -43,7 +43,7 @@ $(foreach jll,$(JLLS),$(eval $(call download-artifacts-toml,$(jll)))) STDLIBS = Artifacts Base64 CRC32c Dates Distributed FileWatching \ Future InteractiveUtils LazyArtifacts Libdl LibGit2 LinearAlgebra Logging \ Markdown Mmap Printf Profile Random REPL Serialization \ - SharedArrays Sockets Test TOML Unicode UUIDs \ + SharedArrays Sockets Test TOML Unicode UUIDs --check-bounds=yes --check-bounds=no \ $(JLL_NAMES) STDLIBS_EXT = Pkg Statistics LibCURL DelimitedFiles Downloads ArgTools Tar NetworkOptions SuiteSparse SparseArrays SHA