WIP: fix #17997, don't load packages in Main

[ci skip]

Author: JeffBezanson

base/loading.jl

@@ -288,6 +288,7 @@ function reload(name::AbstractString)
         error("use `include` instead of `reload` to load source files")
     else
         # reload("Package") is ok
+        unreference_module(Symbol(name))
         require(Symbol(name))
     end
 end
@@ -315,25 +316,84 @@ all platforms, including those with case-insensitive filesystems like macOS and
 Windows.
 """
 function require(mod::Symbol)
-    _require(mod)
+    existed = root_module_exists(mod)
+    M = _require(mod)
     # After successfully loading, notify downstream consumers
     if toplevel_load[] && myid() == 1 && nprocs() > 1
         # broadcast top-level import/using from node 1 (only)
         @sync for p in procs()
             p == 1 && continue
             @async remotecall_wait(p) do
-                if !isbindingresolved(Main, mod) || !isdefined(Main, mod)
-                    _require(mod)
-                end
+                _require(mod)
+                nothing
             end
         end
     end
-    for callback in package_callbacks
-        invokelatest(callback, mod)
+    if !existed
+        for callback in package_callbacks
+            invokelatest(callback, mod)
+        end
+    end
+    return M
+end
+
+const loaded_modules = ObjectIdDict()
+const module_keys = ObjectIdDict()
+
+function register_root_module(key, m::Module)
+    if haskey(loaded_modules, key)
+        oldm = loaded_modules[key]
+        if oldm !== m
+            name = module_name(oldm)
+            warn("replacing module $name.")
+        end
+    end
+    loaded_modules[key] = m
+    module_keys[m] = key
+    nothing
+end
+
+register_root_module(:Core, Core)
+register_root_module(:Base, Base)
+register_root_module(:Main, Main)
+
+is_root_module(m::Module) = haskey(module_keys, m)
+
+root_module_key(m::Module) = module_keys[m]
+
+# This is used as the current module when loading top-level modules.
+# It has the special behavior that modules evaluated in it get added
+# to the loaded_modules table instead of getting bindings.
+baremodule __toplevel__
+using Base
+end
+
+# get a top-level Module from the given key
+# for now keys can only be Symbols, but that will change
+root_module(key::Symbol) = loaded_modules[key]
+
+root_module_exists(key::Symbol) = haskey(loaded_modules, key)
+
+loaded_modules_array() = collect(values(loaded_modules))
+
+function unreference_module(key)
+    if haskey(loaded_modules, key)
+        m = pop!(loaded_modules, key)
+        # need to ensure all modules are GC rooted; will still be referenced
+        # in module_keys
+    end
+end
+
+function register_all(a)
+    for m in a
+        register_root_module(module_name(m), m)
     end
 end
 
 function _require(mod::Symbol)
+    if root_module_exists(mod)
+        return root_module(mod)
+    end
     # dependency-tracking is only used for one top-level include(path),
     # and is not applied recursively to imported modules:
     old_track_dependencies = _track_dependencies[]
@@ -345,7 +405,7 @@ function _require(mod::Symbol)
     if loading !== false
         # load already in progress for this module
         wait(loading)
-        return
+        return root_module(mod)
     end
     package_locks[mod] = Condition()
 
@@ -364,7 +424,8 @@ function _require(mod::Symbol)
         if JLOptions().use_compilecache != 0
             doneprecompile = _require_search_from_serialized(mod, path)
             if !isa(doneprecompile, Bool)
-                return # success
+                register_all(doneprecompile)
+                return root_module(mod) # success
             end
         end
 
@@ -391,14 +452,17 @@ function _require(mod::Symbol)
                 warn(m, prefix="WARNING: ")
                 # fall-through, TODO: disable __precompile__(true) error so that the normal include will succeed
             else
-                return # success
+                register_all(m)
+                return root_module(mod) # success
             end
         end
 
         # just load the file normally via include
         # for unknown dependencies
+        local M
         try
-            Base.include_relative(Main, path)
+            Base.include_relative(__toplevel__, path)
+            return root_module(mod)
         catch ex
             if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true)
                 rethrow() # rethrow non-precompilable=true errors
@@ -411,6 +475,8 @@ function _require(mod::Symbol)
                 # TODO: disable __precompile__(true) error and do normal include instead of error
                 error("Module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.")
             end
+            register_all(m)
+            return root_module(mod)
         end
     finally
         toplevel_load[] = last
@@ -532,7 +598,7 @@ function create_expr_cache(input::String, output::String, concrete_deps::Vector{
                       task_local_storage()[:SOURCE_PATH] = $(source)
                       end)
         end
-        serialize(in, :(Base.include(Main, $(abspath(input)))))
+        serialize(in, :(Base.include(Base.__toplevel__, $(abspath(input)))))
         if source !== nothing
             serialize(in, :(delete!(task_local_storage(), :SOURCE_PATH)))
         end
@@ -570,15 +636,9 @@ function compilecache(name::String)
     cachefile::String = abspath(cachepath, name*".ji")
     # build up the list of modules that we want the precompile process to preserve
     concrete_deps = copy(_concrete_dependencies)
-    for existing in names(Main)
-        if isdefined(Main, existing)
-            mod = getfield(Main, existing)
-            if isa(mod, Module) && !(mod === Main || mod === Core || mod === Base)
-                mod = mod::Module
-                if module_parent(mod) === Main && module_name(mod) === existing
-                    push!(concrete_deps, (existing, module_uuid(mod)))
-                end
-            end
+    for (key,mod) in loaded_modules
+        if !(mod === Main || mod === Core || mod === Base)
+            push!(concrete_deps, (key, module_uuid(mod)))
         end
     end
     # run the expression and cache the result
@@ -675,7 +735,7 @@ function stale_cachefile(modpath::String, cachefile::String)
             if mod == :Main || mod == :Core || mod == :Base
                 continue
             # Module is already loaded
-            elseif isbindingresolved(Main, mod)
+            elseif root_module_exists(mod)
                 continue
             end
             name = string(mod)

base/serialize.jl

@@ -343,9 +343,10 @@ function serialize(s::AbstractSerializer, d::Dict)
 end
 
 function serialize_mod_names(s::AbstractSerializer, m::Module)
-    p = module_parent(m)
-    if m !== p
-        serialize_mod_names(s, p)
+    if Base.is_root_module(m)
+        serialize(s, Base.root_module_key(m))
+    else
+        serialize_mod_names(s, module_parent(m))
         serialize(s, module_name(m))
     end
 end
@@ -772,21 +773,25 @@ function deserialize_svec(s::AbstractSerializer)
 end
 
 function deserialize_module(s::AbstractSerializer)
-    path = deserialize(s)
-    m = Main
-    if isa(path,Tuple) && path !== ()
-        # old version
-        for mname in path
-            m = getfield(m,mname)::Module
+    mkey = deserialize(s)
+    if isa(mkey, Tuple)
+        # old version, TODO: remove
+        if mkey === ()
+            return Main
+        end
+        m = Base.root_module(mkey[1])
+        for i = 2:length(mkey)
+            m = getfield(m, mkey[i])::Module
         end
     else
-        mname = path
+        m = Base.root_module(mkey)
+        mname = deserialize(s)
         while mname !== ()
-            m = getfield(m,mname)::Module
+            m = getfield(m, mname)::Module
             mname = deserialize(s)
         end
     end
-    m
+    return m
 end
 
 function deserialize(s::AbstractSerializer, ::Type{Method})

base/show.jl

@@ -380,8 +380,8 @@ function show(io::IO, p::Pair)
 end
 
 function show(io::IO, m::Module)
-    if m === Main
-        print(io, "Main")
+    if is_root_module(m)
+        print(io, module_name(m))
     else
         print(io, join(fullname(m),"."))
     end