[dylink] Fix rpath calculation in nested dependencies (#24234)

This fixes a bug in rpath calculation in nested dependencies.

Basically, it fixes a case when a relative path is passed to the
`loadDynamicLibrary`.

Think of the following scenario:

0. libhello.wasm depends on libhello_dep.so depends on
libhello_nested_dep.so
1. [main.c] calls dlopen('/usr/lib/libhello.wasm')
2. [dynlink.c] calls `loadDynamicLibrary('/usr/lib/libhello.wasm')`
(_dlopen_js ==> dlopenInternal ==> loadDynamicLibrary)
3. [libdylink.js] inside `loadDynamicLibrary('/usr/lib/libhello.wasm')`,
it loads the dependency `libhello_dep.so`. When locating
`libhello_dep.so`, it uses `parentLibPath: /usr/lib/libhello.wasm` it
locates the library correctly from `/usr/lib/libhello_dep.so`.
5. [libdylink.js] Now we call `loadDynamicLibrary('libhello_dep.so')`,
and it loads its dependency 'libhello_nested_dep.so' first. In this case
it uses `parentLibPath: libhello_dep.so` so it fails to locate the
library.

Author: ryanking13

src/lib/libdylink.js

@@ -977,13 +977,19 @@ var LibraryDylink = {
     // now load needed libraries and the module itself.
     if (flags.loadAsync) {
       return metadata.neededDynlibs
-        .reduce((chain, dynNeeded) => chain.then(() =>
-          loadDynamicLibrary(dynNeeded, flags, localScope)
-        ), Promise.resolve())
+        .reduce((chain, needed) => chain.then(() => {
+#if FILESYSTEM
+          needed = findLibraryFS(needed, flags.rpath) ?? needed;
+#endif
+          return loadDynamicLibrary(needed, flags, localScope);
+        }), Promise.resolve())
         .then(loadModule);
     }
 
     for (var needed of metadata.neededDynlibs) {
+#if FILESYSTEM
+      needed = findLibraryFS(needed, flags.rpath) ?? needed;
+#endif
       loadDynamicLibrary(needed, flags, localScope)
     }
     return loadModule();

test/codesize/test_codesize_hello_dylink.json

@@ -1,10 +1,10 @@
 {
-  "a.out.js": 26672,
-  "a.out.js.gz": 11377,
+  "a.out.js": 26712,
+  "a.out.js.gz": 11400,
   "a.out.nodebug.wasm": 17751,
   "a.out.nodebug.wasm.gz": 8984,
-  "total": 44423,
-  "total_gz": 20361,
+  "total": 44463,
+  "total_gz": 20384,
   "sent": [
     "__syscall_stat64",
     "emscripten_resize_heap",

test/codesize/test_codesize_hello_dylink_all.json

@@ -1,7 +1,7 @@
 {
-  "a.out.js": 245679,
+  "a.out.js": 245720,
   "a.out.nodebug.wasm": 574007,
-  "total": 819686,
+  "total": 819727,
   "sent": [
     "IMG_Init",
     "IMG_Load",

test/test_other.py

@@ -6773,11 +6773,22 @@ def test_ld_library_path(self, args):
 
   @also_with_wasmfs
   def test_dlopen_rpath(self):
+    create_file('hello_nested_dep.c', r'''
+    #include <stdio.h>
+
+    void hello_nested_dep() {
+      printf("Hello_nested_dep\n");
+      return;
+    }
+    ''')
     create_file('hello_dep.c', r'''
     #include <stdio.h>
 
+    void hello_nested_dep();
+
     void hello_dep() {
       printf("Hello_dep\n");
+      hello_nested_dep();
       return;
     }
     ''')
@@ -6804,7 +6815,7 @@ def test_dlopen_rpath(self):
       void (*f)();
       double (*f2)(double);
 
-      h = dlopen("/usr/lib/libhello.wasm", RTLD_NOW);
+      h = dlopen("/usr/lib/libhello.so", RTLD_NOW);
       assert(h);
       f = dlsym(h, "hello");
       assert(f);
@@ -6819,20 +6830,22 @@ def test_dlopen_rpath(self):
     os.mkdir('subdir')
 
     def _build(rpath_flag, expected, **kwds):
-      self.run_process([EMCC, '-o', 'subdir/libhello_dep.so', 'hello_dep.c', '-sSIDE_MODULE'])
-      self.run_process([EMCC, '-o', 'hello.wasm', 'hello.c', '-sSIDE_MODULE', 'subdir/libhello_dep.so'] + rpath_flag)
+      self.run_process([EMCC, '-o', 'subdir/libhello_nested_dep.so', 'hello_nested_dep.c', '-sSIDE_MODULE'])
+      self.run_process([EMCC, '-o', 'subdir/libhello_dep.so', 'hello_dep.c', '-sSIDE_MODULE', 'subdir/libhello_nested_dep.so'] + rpath_flag)
+      self.run_process([EMCC, '-o', 'hello.so', 'hello.c', '-sSIDE_MODULE', 'subdir/libhello_dep.so'] + rpath_flag)
       args = ['--profiling-funcs', '-sMAIN_MODULE=2', '-sINITIAL_MEMORY=32Mb',
-                        '--embed-file', 'hello.wasm@/usr/lib/libhello.wasm',
+                        '--embed-file', 'hello.so@/usr/lib/libhello.so',
                         '--embed-file', 'subdir/libhello_dep.so@/usr/lib/subdir/libhello_dep.so',
-                        'hello.wasm', '-sNO_AUTOLOAD_DYLIBS',
-                        '-L./subdir', '-lhello_dep']
+                        '--embed-file', 'subdir/libhello_nested_dep.so@/usr/lib/subdir/libhello_nested_dep.so',
+                        'hello.so', '-sNO_AUTOLOAD_DYLIBS',
+                        '-L./subdir', '-lhello_dep', '-lhello_nested_dep']
       self.do_runf('main.c', expected, cflags=args, **kwds)
 
     # case 1) without rpath: fail to locate the library
     _build([], r"no such file or directory, open '.*libhello_dep\.so'", regex=True, assert_returncode=NON_ZERO)
 
     # case 2) with rpath: success
-    _build(['-Wl,-rpath,$ORIGIN/subdir'], "Hello\nHello_dep\nOk\n")
+    _build(['-Wl,-rpath,$ORIGIN/subdir,-rpath,$ORIGIN'], "Hello\nHello_dep\nHello_nested_dep\nOk\n")
 
   def test_dlopen_bad_flags(self):
     create_file('main.c', r'''