Make REPL not slow-down if we load a valid cachefile (#51565)

Fixes #51532.

We have funny series of interactions. With the REPL and its dependencies
being removed from the system image,
we observed latency regressions when users create their own
precompilation cache of REPL.

During the precompilation of REPL we launch a subordinate process that
we send statements too.
Now we do want that process to use the existing cache of the REPL
dependencies so we launch it with
`--compiled-modules=existing`. Otherwise precompilation of REPL is even
slower than it is now.

When the user triggers recompilation of REPL due to the use of `-O3` the
subordinate process sees
a valid cache file for REPL itself. Thus no (or very few) precompilation
statements are being
generated. Leading to the cache file compiled with `-O3` to have a
significant latency regression.

In this PR I work around this by replaying the precompilation statements
of REPL from the subordinate
process. A bit hacky, but should be more reliable than trying to set up
a "just right" depot,
or filtering the REPL cache-file out.

Author: vchuravy

src/gf.c

@@ -2328,11 +2328,24 @@ jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi, size_t world)
 }
 
 jl_mutex_t precomp_statement_out_lock;
+ios_t f_precompile;
+JL_STREAM* s_precompile = NULL;
+
+static void init_precompile_output(void)
+{
+    const char *t = jl_options.trace_compile;
+    if (!strncmp(t, "stderr", 6)) {
+        s_precompile = JL_STDERR;
+    }
+    else {
+        if (ios_file(&f_precompile, t, 1, 1, 1, 1) == NULL)
+            jl_errorf("cannot open precompile statement file \"%s\" for writing", t);
+        s_precompile = (JL_STREAM*) &f_precompile;
+    }
+}
 
 static void record_precompile_statement(jl_method_instance_t *mi)
 {
-    static ios_t f_precompile;
-    static JL_STREAM* s_precompile = NULL;
     jl_method_t *def = mi->def.method;
     if (jl_options.trace_compile == NULL)
         return;
@@ -2341,15 +2354,7 @@ static void record_precompile_statement(jl_method_instance_t *mi)
 
     JL_LOCK(&precomp_statement_out_lock);
     if (s_precompile == NULL) {
-        const char *t = jl_options.trace_compile;
-        if (!strncmp(t, "stderr", 6)) {
-            s_precompile = JL_STDERR;
-        }
-        else {
-            if (ios_file(&f_precompile, t, 1, 1, 1, 1) == NULL)
-                jl_errorf("cannot open precompile statement file \"%s\" for writing", t);
-            s_precompile = (JL_STREAM*) &f_precompile;
-        }
+        init_precompile_output();
     }
     if (!jl_has_free_typevars(mi->specTypes)) {
         jl_printf(s_precompile, "precompile(");
@@ -2361,6 +2366,20 @@ static void record_precompile_statement(jl_method_instance_t *mi)
     JL_UNLOCK(&precomp_statement_out_lock);
 }
 
+JL_DLLEXPORT void jl_write_precompile_statement(char* statement)
+{
+    if (jl_options.trace_compile == NULL)
+        return;
+    JL_LOCK(&precomp_statement_out_lock);
+    if (s_precompile == NULL) {
+        init_precompile_output();
+    }
+    jl_printf(s_precompile, "%s\n", statement);
+    if (s_precompile != JL_STDERR)
+        ios_flush(&f_precompile);
+    JL_UNLOCK(&precomp_statement_out_lock);
+}
+
 jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_PROPAGATES_ROOT);
 
 jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t world)

src/jl_exported_funcs.inc

@@ -363,6 +363,7 @@
     XX(jl_new_typevar) \
     XX(jl_next_from_addrinfo) \
     XX(jl_normalize_to_compilable_sig) \
+    XX(jl_write_precompile_statement) \
     XX(jl_no_exc_handler) \
     XX(jl_object_id) \
     XX(jl_object_id_) \

stdlib/REPL/src/REPL.jl

@@ -14,8 +14,21 @@ REPL.run_repl(repl)
 """
 module REPL
 
+const PRECOMPILE_STATEMENTS = Vector{String}()
+
 function __init__()
     Base.REPL_MODULE_REF[] = REPL
+    # We can encounter the situation where the sub-ordinate process used
+    # during precompilation of REPL, can load a valid cache-file.
+    # We need to replay the statements such that the parent process
+    # can also include those. See JuliaLang/julia#51532
+    if Base.JLOptions().trace_compile !== C_NULL && !isempty(PRECOMPILE_STATEMENTS)
+        for statement in PRECOMPILE_STATEMENTS
+            ccall(:jl_write_precompile_statement, Cvoid, (Cstring,), statement)
+        end
+    else
+        empty!(PRECOMPILE_STATEMENTS)
+    end
 end
 
 Base.Experimental.@optlevel 1

stdlib/REPL/src/precompile.jl

@@ -176,9 +176,9 @@ generate_precompile_statements() = try
             if !isexpr(ps, :call)
                 # these are typically comments
                 @debug "skipping statement because it does not parse as an expression" statement
-                delete!(statements, statement)
                 continue
             end
+            push!(REPL.PRECOMPILE_STATEMENTS, statement)
             popfirst!(ps.args) # precompile(...)
             ps.head = :tuple
             # println(ps)