Probe and dlopen() the correct libstdc++ (#46976)

* Probe if system libstdc++ is newer than ours

If the system libstdc++ is detected to be newer, load it.
Otherwise, load the one that we ship. This improves compatibility
with external shared libraries that the user might have on their
system.

Fixes #34276

Co-authored-by: Jameson Nash <[email protected]>
Co-authored-by: Elliot Saba <[email protected]>

* Addressed review comments.

* Change error handling in wrapper functions

Co-authored-by: Jameson Nash <[email protected]>

* Call write_wrapper three times instead of snprintf

Co-authored-by: Jameson Nash <[email protected]>

* Apply suggestions from code review

Co-authored-by: Jameson Nash <[email protected]>

* Update cli/loader_lib.c

Co-authored-by: Jameson Nash <[email protected]>

* Reordered reading and waiting to avoid a deadlock.

* Fixed obvious issues.

* Only load libstdc++ preemptively on linux.

* Update cli/loader_lib.c

Co-authored-by: Jameson Nash <[email protected]>

* Update cli/loader_lib.c

Co-authored-by: Jameson Nash <[email protected]>

* Specified path to bundled libstdc++ on the command line.

* Removed whitespace.

* Update cli/Makefile

Co-authored-by: Jameson Nash <[email protected]>

* Handled make install stringreplace.

* Correctly quoted stringreplace.

* Added -Wl,--enable-new-dtags to prevent DT_RPATH for transitive dependencies

* Updated news entry.

* Added comment about environment variable.

* patched rpath for libgfortran and libLLVM.

* Added explaination to Make.inc

* Removed trailing space

* Removed patchelf for libgfortran, now that BB has been fixed.

* Fixed typos and comments

Co-authored-by: Max Horn <[email protected]>

Co-authored-by: Mosè Giordano <[email protected]>
Co-authored-by: Jameson Nash <[email protected]>
Co-authored-by: Elliot Saba <[email protected]>
Co-authored-by: Max Horn <[email protected]>
(cherry picked from commit eb708d6)

Author: apaz-cli

Make.inc

@@ -1147,6 +1147,29 @@ BB_TRIPLET := $(subst $(SPACE),-,$(filter-out cxx%,$(filter-out libgfortran%,$(s
 
 LIBGFORTRAN_VERSION := $(subst libgfortran,,$(filter libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN))))
 
+# CSL_NEXT_GLIBCXX_VERSION is a triple of the symbols representing support for whatever
+# the next libstdc++ version would be. This is used for two things.
+# 1. Whether the system libraries are new enough, if we need to use the libs bundled with CSL
+# 2. To know which libstdc++ to load at runtime
+# We want whichever libstdc++ library is newer, because if we don't it can cause problems.
+# While what CSL bundles is quite bleeding-edge compared to what most distros ship, if someone
+# tries to build an older branch of Julia, the version of CSL that ships with it may be
+# relatively old. This is not a problem for code that is built in BB, but when we build Julia
+# with the system compiler, that compiler uses the version of `libstdc++` that it is bundled
+# with, and we can get linker errors when trying to run that `julia` executable with the
+# `libstdc++` that comes from the (now old) BB-built CSL.
+# To fix this, we take note when the system `libstdc++.so` is newer than whatever we
+# would get from CSL (by searching for a `GLIBCXX_X.Y.Z` symbol that does not exist
+# in our CSL, but would in a newer one), and default to `USE_BINARYBUILDER_CSL=0` in
+# this case. This ensures that we link against a version with the symbols required.
+# We also check the system libstdc++ at runtime in the cli loader library, and
+# load it if it contains the version symbol that indicates that it is newer than the one
+# shipped with CSL. Although we do not depend on any of the symbols, it is entirely
+# possible that a user might choose to install a library which depends on symbols provided
+# by a newer libstdc++. Without runtime detection, those libraries would break.
+CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.31|GLIBCXX_3\.5\.|GLIBCXX_4\.
+
+
 # This is the set of projects that BinaryBuilder dependencies are hooked up for.
 # Note: we explicitly _do not_ define `CSL` here, since it requires some more
 # advanced techniques to decide whether it should be installed from a BB source
@@ -1203,18 +1226,16 @@ ifneq (,$(filter $(OS),WINNT emscripten))
   RPATH :=
   RPATH_ORIGIN :=
   RPATH_ESCAPED_ORIGIN :=
-  RPATH_LIB :=
 else ifeq ($(OS), Darwin)
   RPATH := -Wl,-rpath,'@executable_path/$(build_libdir_rel)'
   RPATH_ORIGIN := -Wl,-rpath,'@loader_path/'
   RPATH_ESCAPED_ORIGIN := $(RPATH_ORIGIN)
-  RPATH_LIB := -Wl,-rpath,'@loader_path/'
 else
-  RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin
-  RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin
-  RPATH_ESCAPED_ORIGIN := -Wl,-rpath,'\$$\$$ORIGIN' -Wl,-z,origin -Wl,-rpath-link,$(build_shlibdir)
-  RPATH_LIB := -Wl,-rpath,'$$ORIGIN/' -Wl,-z,origin
+  RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin -Wl,--enable-new-dtags
+  RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin -Wl,--enable-new-dtags
+  RPATH_ESCAPED_ORIGIN := -Wl,-rpath,'\$$\$$ORIGIN' -Wl,-z,origin -Wl,-rpath-link,$(build_shlibdir) -Wl,--enable-new-dtags
 endif
+RPATH_LIB := $(RPATH_ORIGIN)
 
 # --whole-archive
 ifeq ($(OS), Darwin)

Makefile

@@ -225,7 +225,7 @@ endif
 # Note that we disable MSYS2's path munging here, as otherwise
 # it replaces our `:`-separated list as a `;`-separated one.
 define stringreplace
-	MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - $1 | grep $2 | awk '{print $$1;}') $3 255 "$(call cygpath_w,$1)"
+	MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - '$1' | grep "$2" | awk '{print $$1;}') "$3" 255 "$(call cygpath_w,$1)"
 endef
 
 
@@ -374,6 +374,16 @@ ifeq ($(BUNDLE_DEBUG_LIBS),1)
 endif
 endif
 
+	# Fix rpaths for dependencies. This should be fixed in BinaryBuilder later.
+ifeq ($(OS), Linux)
+	-$(PATCHELF) --set-rpath '$$ORIGIN' $(DESTDIR)$(private_shlibdir)/libLLVM.$(SHLIB_EXT)
+endif
+
+	# Replace libstdc++ path, which is also moving from `lib` to `../lib/julia`.
+ifeq ($(OS),Linux)
+	$(call stringreplace,$(DESTDIR)$(shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT),\*libstdc++\.so\.6$$,*$(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libstdc++.so.6))
+endif
+
 
 ifneq ($(LOADER_BUILD_DEP_LIBS),$(LOADER_INSTALL_DEP_LIBS))
 	# Next, overwrite relative path to libjulia-internal in our loader if $$(LOADER_BUILD_DEP_LIBS) != $$(LOADER_INSTALL_DEP_LIBS)

NEWS.md

@@ -244,7 +244,8 @@ Deprecated or removed
 
 External dependencies
 ---------------------
-
+* On Linux, now autodetects the system libstdc++ version, and automatically loads the system library if it is newer. The old behavior of loading the bundled libstdc++ regardless of the system version obtained by setting the environment variable `JULIA_PROBE_LIBSTDCXX=0`.
+* Removed `RPATH` from the julia binary. On Linux this may break libraries that have failed to set `RUNPATH`.
 
 Tooling Improvements
 ---------------------

cli/Makefile

@@ -13,6 +13,8 @@ LOADER_LDFLAGS = $(JLDFLAGS) -ffreestanding -L$(build_shlibdir) -L$(build_libdir
 
 ifeq ($(OS),WINNT)
 LOADER_CFLAGS += -municode -mconsole -nostdlib -fno-stack-check -fno-stack-protector -mno-stack-arg-probe
+else ifeq ($(OS),Linux)
+LOADER_CFLAGS += -DGLIBCXX_LEAST_VERSION_SYMBOL=\"$(shell echo "$(CSL_NEXT_GLIBCXX_VERSION)" | cut -d'|' -f1 | sed 's/\\//g')\"
 endif
 
 ifeq ($(OS),WINNT)
@@ -111,7 +113,7 @@ endif
 
 $(build_shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT): $(LIB_OBJS) $(SRCDIR)/list_strip_symbols.h | $(build_shlibdir) $(build_libdir)
 	@$(call PRINT_LINK, $(CC) $(call IMPLIB_FLAGS,$@.tmp) $(LOADER_CFLAGS) -DLIBRARY_EXPORTS -shared $(SHIPFLAGS) $(LIB_OBJS) -o $@ \
-		$(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(RPATH_LIB) $(call SONAME_FLAGS,libjulia.$(JL_MAJOR_SHLIB_EXT)))
+		$(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(call SONAME_FLAGS,libjulia.$(JL_MAJOR_SHLIB_EXT)))
 	@$(INSTALL_NAME_CMD)libjulia.$(SHLIB_EXT) $@
 ifeq ($(OS), WINNT)
 	@# Note that if the objcopy command starts getting too long, we can use `@file` to read
@@ -121,7 +123,7 @@ endif
 
 $(build_shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT): $(LIB_DOBJS) $(SRCDIR)/list_strip_symbols.h | $(build_shlibdir) $(build_libdir)
 	@$(call PRINT_LINK, $(CC) $(call IMPLIB_FLAGS,$@.tmp) $(LOADER_CFLAGS) -DLIBRARY_EXPORTS -shared $(DEBUGFLAGS) $(LIB_DOBJS) -o $@ \
-		$(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(RPATH_LIB) $(call SONAME_FLAGS,libjulia-debug.$(JL_MAJOR_SHLIB_EXT)))
+		$(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(call SONAME_FLAGS,libjulia-debug.$(JL_MAJOR_SHLIB_EXT)))
 	@$(INSTALL_NAME_CMD)libjulia-debug.$(SHLIB_EXT) $@
 ifeq ($(OS), WINNT)
 	@$(call PRINT_ANALYZE, $(OBJCOPY) $(build_libdir)/$(notdir $@).tmp.a $(STRIP_EXPORTED_FUNCS) $(build_libdir)/$(notdir $@).a && rm $(build_libdir)/$(notdir $@).tmp.a)

cli/loader_lib.c

@@ -16,7 +16,7 @@ extern "C" {
 #endif
 
 // Save DEP_LIBS to a variable that is explicitly sized for expansion
-static char dep_libs[1024] = DEP_LIBS;
+static char dep_libs[1024] = "\0" DEP_LIBS;
 
 JL_DLLEXPORT void jl_loader_print_stderr(const char * msg)
 {
@@ -33,7 +33,6 @@ void jl_loader_print_stderr3(const char * msg1, const char * msg2, const char *
 /* Wrapper around dlopen(), with extra relative pathing thrown in*/
 static void * load_library(const char * rel_path, const char * src_dir, int err) {
     void * handle = NULL;
-
     // See if a handle is already open to the basename
     const char *basename = rel_path + strlen(rel_path);
     while (basename-- > rel_path)
@@ -147,6 +146,174 @@ JL_DLLEXPORT const char * jl_get_libdir()
     return lib_dir;
 }
 
+// On Linux, it can happen that the system has a newer libstdc++ than the one we ship,
+// which can break loading of some system libraries: <https://github.com/JuliaLang/julia/issues/34276>.
+// As a fix, on linux we probe the system libstdc++ to see if it is newer, and then load it if it is.
+// Otherwise, we load the bundled one. This improves compatibility with third party dynamic libs that
+// may depend on symbols exported by the system libstdxc++.
+#ifdef _OS_LINUX_
+#ifndef GLIBCXX_LEAST_VERSION_SYMBOL
+#warning GLIBCXX_LEAST_VERSION_SYMBOL should always be defined in the makefile.
+#define GLIBCXX_LEAST_VERSION_SYMBOL "GLIBCXX_a.b.c" /* Appease the linter */
+#endif
+
+#include <link.h>
+#include <sys/wait.h>
+
+// write(), but handle errors and avoid EINTR
+static void write_wrapper(int fd, const char *str, size_t len)
+{
+    size_t written_sofar = 0;
+    while (len) {
+        ssize_t bytes_written = write(fd, str + written_sofar, len);
+        if (bytes_written == -1 && errno == EINTR) continue;
+        if (bytes_written == -1 && errno != EINTR) {
+            perror("(julia) child libstdcxxprobe write");
+            _exit(1);
+        }
+        len -= bytes_written;
+        written_sofar += bytes_written;
+    }
+}
+
+// read(), but handle errors and avoid EINTR
+static void read_wrapper(int fd, char **ret, size_t *ret_len)
+{
+    // Allocate an initial buffer
+    size_t len = JL_PATH_MAX;
+    char *buf = (char *)malloc(len + 1);
+    if (!buf) {
+        perror("(julia) malloc");
+        exit(1);
+    }
+
+    // Read into it, reallocating as necessary
+    size_t have_read = 0;
+    while (1) {
+        ssize_t n = read(fd, buf + have_read, len - have_read);
+        have_read += n;
+        if (n == 0) break;
+        if (n == -1 && errno != EINTR) {
+            perror("(julia) libstdcxxprobe read");
+            exit(1);
+        }
+        if (n == -1 && errno == EINTR) continue;
+        if (have_read == len) {
+            buf = (char *)realloc(buf, 1 + (len *= 2));
+            if (!buf) {
+                perror("(julia) realloc");
+                exit(1);
+            }
+        }
+    }
+
+    *ret = buf;
+    *ret_len = have_read;
+}
+
+// Return the path to the libstdcxx to load.
+// If the path is found, return it.
+// Otherwise, print the error and exit.
+// The path returned must be freed.
+static char *libstdcxxprobe(void)
+{
+    // Create the pipe and child process.
+    int fork_pipe[2];
+    int ret = pipe(fork_pipe);
+    if (ret == -1) {
+        perror("(julia) Error during libstdcxxprobe: pipe");
+        exit(1);
+    }
+    pid_t pid = fork();
+    if (pid == -1)  {
+        perror("Error during libstdcxxprobe:\nfork");
+        exit(1);
+    }
+    if (pid == (pid_t) 0) { // Child process.
+        close(fork_pipe[0]);
+
+        // Open the first available libstdc++.so.
+        // If it can't be found, report so by exiting zero.
+        // The star is there to prevent the compiler from merging constants
+        // with "\0*libstdc++.so.6", which we string replace inside the .so during
+        // make install.
+        void *handle = dlopen("libstdc++.so.6\0*", RTLD_LAZY);
+        if (!handle) {
+            _exit(0);
+        }
+
+        // See if the version is compatible
+        char *dlerr = dlerror(); // clear out dlerror
+        void *sym = dlsym(handle, GLIBCXX_LEAST_VERSION_SYMBOL);
+        dlerr = dlerror();
+        if (dlerr) {
+            // We can't use the library that was found, so don't write anything.
+            // The main process will see that nothing was written,
+            // then exit the function and return null.
+            _exit(0);
+        }
+
+        // No error means the symbol was found, we can use this library.
+        // Get the path to it, and write it to the parent process.
+        struct link_map *lm;
+        ret = dlinfo(handle, RTLD_DI_LINKMAP, &lm);
+        if (ret == -1) {
+            char *errbuf = dlerror();
+            char *errdesc = (char*)"Error during libstdcxxprobe in child process:\ndlinfo: ";
+            write_wrapper(STDERR_FILENO, errdesc, strlen(errdesc));
+            write_wrapper(STDERR_FILENO, errbuf, strlen(errbuf));
+            write_wrapper(STDERR_FILENO, "\n", 1);
+            _exit(1);
+        }
+        char *libpath = lm->l_name;
+        write_wrapper(fork_pipe[1], libpath, strlen(libpath));
+        _exit(0);
+    }
+    else { // Parent process.
+        close(fork_pipe[1]);
+
+        // Read the absolute path to the lib from the child process.
+        char *path;
+        size_t pathlen;
+        read_wrapper(fork_pipe[0], &path, &pathlen);
+
+        // Close the read end of the pipe
+        close(fork_pipe[0]);
+
+        // Wait for the child to complete.
+        while (1) {
+            int wstatus;
+            pid_t npid = waitpid(pid, &wstatus, 0);
+            if (npid == -1) {
+                if (errno == EINTR) continue;
+                if (errno != EINTR) {
+                    perror("Error during libstdcxxprobe in parent process:\nwaitpid");
+                    exit(1);
+                }
+            }
+            else if (!WIFEXITED(wstatus)) {
+                const char *err_str = "Error during libstdcxxprobe in parent process:\n"
+                                      "The child process did not exit normally.\n";
+                size_t err_strlen = strlen(err_str);
+                write_wrapper(STDERR_FILENO, err_str, err_strlen);
+                exit(1);
+            }
+            else if (WEXITSTATUS(wstatus)) {
+                // The child has printed an error and exited, so the parent should exit too.
+                exit(1);
+            }
+            break;
+        }
+
+        if (!pathlen) {
+            free(path);
+            return NULL;
+        }
+        return path;
+    }
+}
+#endif
+
 void * libjulia_internal = NULL;
 __attribute__((constructor)) void jl_load_libjulia_internal(void) {
     // Only initialize this once
@@ -155,11 +322,43 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) {
     }
 
     // Introspect to find our own path
-    const char * lib_dir = jl_get_libdir();
+    const char *lib_dir = jl_get_libdir();
 
     // Pre-load libraries that libjulia-internal needs.
-    int deps_len = strlen(dep_libs);
-    char * curr_dep = &dep_libs[0];
+    int deps_len = strlen(&dep_libs[1]);
+    char *curr_dep = &dep_libs[1];
+
+    void *cxx_handle;
+
+#if defined(_OS_LINUX_)
+    int do_probe = 1;
+    int done_probe = 0;
+    char *probevar = getenv("JULIA_PROBE_LIBSTDCXX");
+    if (probevar) {
+        if (strcmp(probevar, "1") == 0 || strcmp(probevar, "yes") == 0)
+            do_probe = 1;
+        else if (strcmp(probevar, "0") == 0 || strcmp(probevar, "no") == 0)
+            do_probe = 0;
+    }
+    if (do_probe) {
+        char *cxxpath = libstdcxxprobe();
+        if (cxxpath) {
+            cxx_handle = dlopen(cxxpath, RTLD_LAZY);
+            char *dlr = dlerror();
+            if (dlr) {
+                jl_loader_print_stderr("ERROR: Unable to dlopen(cxxpath) in parent!\n");
+                jl_loader_print_stderr3("Message: ", dlr, "\n");
+                exit(1);
+            }
+            free(cxxpath);
+            done_probe = 1;
+        }
+    }
+    if (!done_probe) {
+        const static char bundled_path[256] = "\0*libstdc++.so.6";
+        load_library(&bundled_path[2], lib_dir, 1);
+    }
+#endif
 
     // We keep track of "special" libraries names (ones whose name is prefixed with `@`)
     // which are libraries that we want to load in some special, custom way, such as
@@ -183,7 +382,8 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) {
             }
             special_library_names[special_idx] = curr_dep + 1;
             special_idx += 1;
-        } else {
+        }
+        else {
             load_library(curr_dep, lib_dir, 1);
         }
 
@@ -272,7 +472,7 @@ JL_DLLEXPORT int jl_load_repl(int argc, char * argv[]) {
 }
 
 #ifdef _OS_WINDOWS_
-int __stdcall DllMainCRTStartup(void* instance, unsigned reason, void* reserved) {
+int __stdcall DllMainCRTStartup(void *instance, unsigned reason, void *reserved) {
     setup_stdio();
 
     // Because we override DllMainCRTStartup, we have to manually call our constructor methods

deps/csl.mk

@@ -12,17 +12,16 @@ endef
 
 # CSL bundles lots of system compiler libraries, and while it is quite bleeding-edge
 # as compared to what most distros ship, if someone tries to build an older branch,
-# the version of CSL that ships with that branch may become relatively old.  This is
-# not a problem for code that is built in BB, but when we build Julia with the system
+# the version of CSL that ships with that branch may be relatively old. This is not
+# a problem for code that is built in BB, but when we build Julia with the system
 # compiler, that compiler uses the version of `libstdc++` that it is bundled with,
-# and we can get linker errors when trying to run that `julia` executable with the
+# and we can get linker errors when trying to run that 	`julia` executable with the
 # `libstdc++` that comes from the (now old) BB-built CSL.
 #
 # To fix this, we take note when the system `libstdc++.so` is newer than whatever we
 # would get from CSL (by searching for a `GLIBCXX_3.4.X` symbol that does not exist
 # in our CSL, but would in a newer one), and default to `USE_BINARYBUILDER_CSL=0` in
 # this case.
-CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.31|GLIBCXX_3\.5\.|GLIBCXX_4\.
 
 # First, check to see if BB is disabled on a global setting
 ifeq ($(USE_BINARYBUILDER),0)