loading: install at packages/$name/$slug instead of no names (#26057)

Save .ji files at libs/v#.#/$name/$slug.ji so all the caches for a
given package name are in the same directory of that name. These
can be a mix of .ji files for different packages with the same name
but I think that's perfectly acceptable.

Also base the slug code on CRC32 instead of modular arithmetic.

Author: StefanKarpinski

base/loading.jl

@@ -91,12 +91,9 @@ struct SHA1
     end
 end
 SHA1(s::Union{String,SubString{String}}) = SHA1(hex2bytes(s))
+string(hash::SHA1) = bytes2hex(hash.bytes)
 
-convert(::Type{String}, hash::SHA1) = bytes2hex(convert(Vector{UInt8}, hash))
-convert(::Type{Vector{UInt8}}, hash::SHA1) = hash.bytes
-
-string(hash::SHA1) = String(hash)
-show(io::IO, hash::SHA1) = print(io, "SHA1(", convert(String, hash), ")")
+show(io::IO, hash::SHA1) = print(io, "SHA1(", string(hash), ")")
 isless(a::SHA1, b::SHA1) = lexless(a.bytes, b.bytes)
 hash(a::SHA1, h::UInt) = hash((SHA1, a.bytes), h)
 ==(a::SHA1, b::SHA1) = a.bytes == b.bytes
@@ -123,36 +120,28 @@ dummy_uuid(project_file::String) = isfile_casesensitive(project_file) ?
 
 ## package path slugs: turning UUID + SHA1 into a pair of 4-byte "slugs" ##
 
-const SlugInt = UInt32 # max p = 4
-const chars = String(['A':'Z'; 'a':'z'; '0':'9'])
-const nchars = SlugInt(length(chars))
-const max_p = floor(Int, log(nchars, typemax(SlugInt) >>> 8))
+const slug_chars = String(['A':'Z'; 'a':'z'; '0':'9'])
 
-function slug(x::SlugInt, p::Int)
-    1 ≤ p ≤ max_p || # otherwise previous steps are wrong
-        error("invalid slug size: $p (need 1 ≤ p ≤ $max_p)")
-    return sprint() do io
+function slug(x::UInt32, p::Int)
+    sprint(sizehint=p) do io
+        n = length(slug_chars)
         for i = 1:p
-            x, d = divrem(x, nchars)
-            write(io, chars[1+d])
+            x, d = divrem(x, n)
+            write(io, slug_chars[1+d])
         end
     end
 end
-slug(x::Integer, p::Int) = slug(SlugInt(x), p)
 
-function slug(bytes::Vector{UInt8}, p::Int)
-    n = nchars^p
-    x = zero(SlugInt)
-    for (i, b) in enumerate(bytes)
-        x = (x + b*powermod(2, 8(i-1), n)) % n
-    end
-    slug(x, p)
+function package_slug(uuid::UUID, p::Int=4)
+    crc = _crc32c(uuid)
+    return slug(crc, p)
 end
 
-slug(uuid::UUID, p::Int=4) = slug(uuid.value % nchars^p, p)
-slug(sha1::SHA1, p::Int=4) = slug(sha1.bytes, p)
-
-version_slug(uuid::UUID, sha1::SHA1) = joinpath(slug(uuid), slug(sha1))
+function version_slug(uuid::UUID, sha1::SHA1, p::Int=4)
+    crc = _crc32c(uuid)
+    crc = _crc32c(sha1.bytes, crc)
+    return slug(crc, p)
+end
 
 ## load path expansion: turn LOAD_PATH entries into concrete paths ##
 
@@ -585,7 +574,7 @@ function explicit_manifest_uuid_path(manifest_file::String, pkg::PkgId)::Union{N
             return entry_path(path, name)
         end
         hash == nothing && return nothing
-        slug = version_slug(uuid, hash)
+        slug = joinpath(name, version_slug(uuid, hash))
         for depot in DEPOT_PATH
             path = abspath(depot, "packages", slug)
             ispath(path) && return path
@@ -638,10 +627,13 @@ function find_source_file(path::AbstractString)
     return isfile(base_path) ? base_path : nothing
 end
 
+cache_file_entry(pkg::PkgId) =
+    pkg.uuid === nothing ? "$(pkg.name).ji" :
+        joinpath(pkg.name, "$(package_slug(pkg.uuid)).ji")
+
 function find_all_in_cache_path(pkg::PkgId)
     paths = String[]
-    suffix = "$(pkg.name).ji"
-    pkg.uuid !== nothing && (suffix = joinpath(slug(pkg.uuid), suffix))
+    suffix = cache_file_entry(pkg)
     for prefix in LOAD_CACHE_PATH
         path = joinpath(prefix, suffix)
         isfile_casesensitive(path) && push!(paths, path)
@@ -1179,12 +1171,9 @@ function compilecache(pkg::PkgId)
     path = locate_package(pkg)
     path === nothing && throw(ArgumentError("$name not found in path"))
     # decide where to put the resulting cache file
-    cachepath = abspath(LOAD_CACHE_PATH[1])
-    pkg.uuid !== nothing && (cachepath = joinpath(cachepath, slug(pkg.uuid)))
-    if !isdir(cachepath)
-        mkpath(cachepath)
-    end
-    cachefile::String = joinpath(cachepath, "$(pkg.name).ji")
+    cachefile = abspath(LOAD_CACHE_PATH[1], cache_file_entry(pkg))
+    cachepath = dirname(cachefile)
+    isdir(cachepath) || mkpath(cachepath)
     # build up the list of modules that we want the precompile process to preserve
     concrete_deps = copy(_concrete_dependencies)
     for (key, mod) in loaded_modules

base/sysimg.jl

@@ -426,9 +426,6 @@ using .StackTraces
 include("initdefs.jl")
 include("client.jl")
 
-# misc useful functions & macros
-include("util.jl")
-
 # statistics
 include("statistics.jl")
 
@@ -442,6 +439,9 @@ include("threadcall.jl")
 include("uuid.jl")
 include("loading.jl")
 
+# misc useful functions & macros
+include("util.jl")
+
 # set up depot & load paths to be able to find stdlib packages
 let BINDIR = ccall(:jl_get_julia_bindir, Any, ())
     init_depot_path(BINDIR)

base/util.jl

@@ -561,7 +561,8 @@ function _crc32c(io::IO, nb::Integer, crc::UInt32=0x00000000)
 end
 _crc32c(io::IO, crc::UInt32=0x00000000) = _crc32c(io, typemax(Int64), crc)
 _crc32c(io::IOStream, crc::UInt32=0x00000000) = _crc32c(io, filesize(io)-position(io), crc)
-
+_crc32c(uuid::UUID, crc::UInt32=0x00000000) =
+    ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt128}, Csize_t), crc, uuid.value, 16)
 
 """
     @kwdef typedef

test/depot/packages/9HkB/TCSb/src/Baz.jl test/depot/packages/Baz/81oL/src/Baz.jl

@@ -94,11 +94,11 @@ push!(empty!(DEPOT_PATH), "depot")
 @testset "project & manifest identify_package & locate_package" begin
     local path
     for (names, uuid, path) in [
-        ("Foo",     "767738be-2f1f-45a9-b806-0234f3164144", "project/deps/Foo1/src/Foo.jl"       ),
-        ("Bar.Foo", "6f418443-bd2e-4783-b551-cdbac608adf2", "project/deps/Foo2.jl/src/Foo.jl"    ),
-        ("Bar",     "2a550a13-6bab-4a91-a4ee-dff34d6b99d0", "project/deps/Bar/src/Bar.jl"        ),
-        ("Foo.Baz", "6801f525-dc68-44e8-a4e8-cabd286279e7", "depot/packages/9HkB/TCSb/src/Baz.jl"),
-        ("Foo.Qux", "b5ec9b9c-e354-47fd-b367-a348bdc8f909", "project/deps/Qux.jl"                ),
+        ("Foo",     "767738be-2f1f-45a9-b806-0234f3164144", "project/deps/Foo1/src/Foo.jl"      ),
+        ("Bar.Foo", "6f418443-bd2e-4783-b551-cdbac608adf2", "project/deps/Foo2.jl/src/Foo.jl"   ),
+        ("Bar",     "2a550a13-6bab-4a91-a4ee-dff34d6b99d0", "project/deps/Bar/src/Bar.jl"       ),
+        ("Foo.Baz", "6801f525-dc68-44e8-a4e8-cabd286279e7", "depot/packages/Baz/81oL/src/Baz.jl"),
+        ("Foo.Qux", "b5ec9b9c-e354-47fd-b367-a348bdc8f909", "project/deps/Qux.jl"               ),
     ]
         n = map(String, split(names, '.'))
         pkg = identify_package(n...)