YJIT: Fix shared/static library symbol leaks

Rust 1.58.0 unfortunately doesn't provide facilities to control symbol
visibility/presence, but we care about controlling the list of
symbols exported from libruby-static.a and libruby.so.

This commit uses `ld -r` to make a single object out of rustc's
staticlib output, libyjit.a. This moves libyjit.a out of MAINLIBS and adds
libyjit.o into COMMONOBJS, which obviates the code for merging libyjit.a
into libruby-static.a. The odd appearance of libyjit.a in SOLIBS is also
gone.

To filter out symbols we do not want to export on ELF platforms, we use
objcopy after the partial link. On darwin, we supply a symbol list to
the linker which takes care of hiding unprefixed symbols.

[Bug #19255]

Co-authored-by: Nobuyoshi Nakada <nobu@ruby-lang.org>

6 files changed, 49 insertions, 15 deletions

diff --git a/common.mk b/common.mk
index 158ca86b8c..fcd0923a4d 100644
--- a/common.mk
+++ b/common.mk
@@ -151,6 +151,7 @@ COMMONOBJS    = array.$(OBJEXT) \
 		vm_sync.$(OBJEXT) \
 		vm_trace.$(OBJEXT) \
 		$(YJIT_OBJ) \
+		$(YJIT_LIBOBJ) \
 		$(COROUTINE_OBJ) \
 		$(DTRACE_OBJ) \
 		$(BUILTIN_ENCOBJS) \
diff --git a/template/Makefile.in b/template/Makefile.in
index c4d05ef243..bae3980440 100644
--- a/template/Makefile.in
+++ b/template/Makefile.in
@@ -114,6 +114,7 @@ MJIT_TABS=@MJIT_TABS@
 YJIT_SUPPORT=@YJIT_SUPPORT@
 YJIT_LIBS=@YJIT_LIBS@
 YJIT_OBJ=@YJIT_OBJ@
+YJIT_LIBOBJ = $(YJIT_LIBS:.a=.@OBJEXT@)
 CARGO_TARGET_DIR=@abs_top_builddir@/yjit/target
 CARGO_BUILD_ARGS=@CARGO_BUILD_ARGS@
 LDFLAGS = @STATIC@ $(CFLAGS) @LDFLAGS@
@@ -130,7 +131,7 @@ XDLDFLAGS = @DLDFLAGS@
 DLDFLAGS = @LIBRUBY_DLDFLAGS@ $(XLDFLAGS) $(ARCH_FLAG)
 SOLIBS = @SOLIBS@
 ENABLE_DEBUG_ENV = @ENABLE_DEBUG_ENV@
-MAINLIBS = $(YJIT_LIBS) @MAINLIBS@
+MAINLIBS = @MAINLIBS@
 ARCHMINIOBJS = @MINIOBJS@
 DLNOBJ = @DLNOBJ@
 ENCOBJS = @ENCOBJS@
@@ -319,16 +320,6 @@ $(LIBRUBY_A):
 		@-[ -z "$(EXTSTATIC)" ] || $(PRE_LIBRUBY_UPDATE)
 		$(ECHO) linking static-library $@
 		$(Q) $(AR) $(ARFLAGS) $@ $(LIBRUBY_A_OBJS) $(INITOBJS)
-		$(Q) if [ 'no' != '$(YJIT_SUPPORT)' ]; then \
-		  set -eu$(V0:1=x) && \
-		  $(ECHO0) 'merging $(YJIT_LIBS) into $@' && \
-		  $(RMALL)    libyjit/ && \
-		  $(MAKEDIRS) libyjit/ && \
-		  trap "$(RMALL) libyjit/" 0 && \
-		  (cd libyjit/ && $(AR) -x ../$(YJIT_LIBS)) && \
-		  :  "$(AR) $(ARFLAGS) $@ libyjit/*.$(OBJEXT)" && \
-		  find libyjit/ -name '*.$(OBJEXT)' -exec $(AR) $(ARFLAGS) $@ '{}' '+' ; \
-		fi
 		@-$(RANLIB) $@ 2> /dev/null || true
 
 verify-static-library: $(LIBRUBY_A)
diff --git a/tool/leaked-globals b/tool/leaked-globals
index 083f658fb1..88b8ae9e72 100755
--- a/tool/leaked-globals
+++ b/tool/leaked-globals
@@ -43,9 +43,16 @@ ARGV.reject! do |n|
     true
   end
 end
+# darwin's ld64 seems to require exception handling personality functions to be
+# extern, so we allow the Rust one.
+REPLACE.push("rust_eh_personality") if RUBY_PLATFORM.include?("darwin")
+# nm errors with Rust's LLVM bitcode when Rust uses a newer LLVM version than nm.
+# In case we're working with llvm-nm, tell it to not worry about the bitcode.
+no_llvm = "--no-llvm-bc" if `#{NM} --version` =~ /llvm/i
+
 print "Checking leaked global symbols..."
 STDOUT.flush
-IO.foreach("|#{NM} #{ARGV.join(' ')}") do |line|
+IO.foreach("|#{NM} #{no_llvm} #{ARGV.join(' ')}") do |line|
   n, t, = line.split
   next unless /[A-TV-Z]/ =~ t
   next unless n.sub!(/^#{SYMBOL_PREFIX}/o, "")
diff --git a/tool/update-deps b/tool/update-deps
index 445f9a00e3..fcd7373dbd 100755
--- a/tool/update-deps
+++ b/tool/update-deps
@@ -301,6 +301,7 @@ def read_make_deps(cwd)
       deps = deps.scan(%r{[/0-9a-zA-Z._-]+})
       deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp
       next if /\.o\z/ !~ target.to_s
+      next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source)
       next if /\.bundle\// =~ target.to_s
       next if /\A\./ =~ target.to_s # skip rules such as ".c.o"
       #p [curdir, target, deps]
diff --git a/yjit/not_gmake.mk b/yjit/not_gmake.mk
index cbc60c09f1..8bb01d65be 100644
--- a/yjit/not_gmake.mk
+++ b/yjit/not_gmake.mk
@@ -11,4 +11,8 @@ yjit-static-lib:
 	$(ECHO) 'building Rust YJIT (release mode)'
 	$(Q) $(RUSTC) $(YJIT_RUSTC_ARGS)
 
-miniruby$(EXEEXT): $(YJIT_LIBS)
+# Assume GNU flavor LD and OBJCOPY. Works on FreeBSD 13, at least.
+$(YJIT_LIBOBJ): $(YJIT_LIBS)
+	$(ECHO) 'partial linking $(YJIT_LIBS) into $@'
+	$(Q) $(LD) -r -o $@ --whole-archive $(YJIT_LIBS)
+	-$(Q) $(OBJCOPY) --wildcard --keep-global-symbol='$(SYMBOL_PREFIX)rb_*' $(@)
diff --git a/yjit/yjit.mk b/yjit/yjit.mk
index bcfe902be7..6cd12c4bf3 100644
--- a/yjit/yjit.mk
+++ b/yjit/yjit.mk
@@ -40,8 +40,38 @@ $(YJIT_LIBS): $(YJIT_SRC_FILES)
 else
 endif
 
-# Put this here instead of in common.mk to avoid breaking nmake builds
-miniruby$(EXEEXT): $(YJIT_LIBS)
+yjit-libobj: $(YJIT_LIBOBJ)
+
+# Note, BSD handling is in yjit/not_gmake.mk
+YJIT_LIB_SYMBOLS = $(YJIT_LIBS:.a=).symbols
+$(YJIT_LIBOBJ): $(YJIT_LIBS)
+	$(ECHO) 'partial linking $(YJIT_LIBS) into $@'
+ifneq ($(findstring linux,$(target_os)),)
+	$(Q) $(LD) -r -o $@ --whole-archive $(YJIT_LIBS)
+	-$(Q) $(OBJCOPY) --wildcard --keep-global-symbol='$(SYMBOL_PREFIX)rb_*' $(@)
+else ifneq ($(findstring darwin,$(target_os)),)
+	$(Q) $(CC) -nodefaultlibs -r -o $@ -exported_symbols_list $(YJIT_LIB_SYMBOLS) $(YJIT_LIBS)
+else
+	false
+endif
+
+# For Darwin only: a list of symbols that we want the glommed Rust static lib to export.
+# Unfortunately, using wildcard like '_rb_*' with -exported-symbol does not work, at least
+# not on version 820.1. Assume llvm-nm, so XCode 8.0 (from 2016) or newer.
+#
+# The -exported_symbols_list pulls out the right archive members. Symbols not listed
+# in the list are made private extern, which are in turn made local as we're using `ld -r`.
+# Note, section about -keep_private_externs in ld's man page hints at this behavior on which
+# we rely.
+ifneq ($(findstring darwin,$(target_os)),)
+$(YJIT_LIB_SYMBOLS): $(YJIT_LIBS)
+	$(Q) $(NM) --no-llvm-bc --defined-only --extern-only $(YJIT_LIBS) | \
+	sed -n -e 's/.* //' -e '/^$(SYMBOL_PREFIX)rb_/p' \
+	-e '/^$(SYMBOL_PREFIX)rust_eh_personality/p' \
+	> $@
+
+$(YJIT_LIBOBJ): $(YJIT_LIB_SYMBOLS)
+endif
 
 # By using YJIT_BENCH_OPTS instead of RUN_OPTS, you can skip passing the options to `make install`
 YJIT_BENCH_OPTS = $(RUN_OPTS) --enable-gems