Add Engine/Component::deserialize_raw (#10321)

* Add Engine/Component::deserialize_raw

For targets without virtual memory support, the only mechanism for
load code previously was to copy bytes from a provided slice into
a separate, owned, allocated buffer containing a copy of the serialized
module contents.  This is expensive for constrained targets and
prohibits embedded runtimes like doing things like running code
directly out of memory-mapped devices such as NOR flash.

The tradeoff for directly using the externally owned memory is that
the caller must ensure that the code memory will not be written
to for the lifetime of the CodeMemory.

Loading code from externally owned memory is supported for all
configurations but is expected to fail to deserialize on platforms
that suport virtual memory and are attempting to load modules
or components that require that the memory be made executable
(native code rather than pulley).

#10245

* Fix tests for s390x

Previously, there was a bad assumption that pulley32/64 were the
only targets but there's also the "be" variants.  Use the
`pulley_host` helper to better get the right pulley target
for the host.

Author: posborne

crates/wasmtime/src/engine.rs

@@ -7,6 +7,7 @@ use crate::runtime::type_registry::TypeRegistry;
 use crate::runtime::vm::GcRuntime;
 use crate::Config;
 use alloc::sync::Arc;
+use core::ptr::NonNull;
 #[cfg(target_has_atomic = "64")]
 use core::sync::atomic::{AtomicU64, Ordering};
 #[cfg(any(feature = "cranelift", feature = "winch"))]
@@ -783,6 +784,29 @@ impl Engine {
         )
     }
 
+    /// Loads a `CodeMemory` from the specified memory region without copying
+    ///
+    /// The `expected` marker here is whether the bytes are expected to be
+    /// a precompiled module or a component.  The `memory` provided is expected
+    /// to be a serialized module (.cwasm) generated by `[Module::serialize]`
+    /// or [`Engine::precompile_module] or their `Component` counterparts
+    /// [`Component::serialize`] or `[Engine::precompile_component]`.
+    ///
+    /// The memory provided is guaranteed to only be immutably by the runtime.
+    ///
+    /// # Safety
+    ///
+    /// As there is no copy here, the runtime will be making direct readonly use
+    /// of the provided memory. As such, outside writes to this memory region
+    /// will result in undefined and likely very undesirable behavior.
+    pub(crate) unsafe fn load_code_raw(
+        &self,
+        memory: NonNull<[u8]>,
+        expected: ObjectKind,
+    ) -> Result<Arc<crate::CodeMemory>> {
+        self.load_code(crate::runtime::vm::MmapVec::from_raw(memory)?, expected)
+    }
+
     /// Like `load_code_bytes`, but creates a mmap from a file on disk.
     #[cfg(feature = "std")]
     pub(crate) fn load_code_file(

crates/wasmtime/src/runtime/code_memory.rs

@@ -324,11 +324,17 @@ impl CodeMemory {
             // we aren't able to make it readonly, but this is just a
             // defense-in-depth measure and isn't required for correctness.
             #[cfg(has_virtual_memory)]
-            self.mmap.make_readonly(0..self.mmap.len())?;
+            if self.mmap.supports_virtual_memory() {
+                self.mmap.make_readonly(0..self.mmap.len())?;
+            }
 
             // Switch the executable portion from readonly to read/execute.
             if self.needs_executable {
                 if !self.custom_publish()? {
+                    if !self.mmap.supports_virtual_memory() {
+                        bail!("this target requires virtual memory to be enabled");
+                    }
+
                     #[cfg(has_virtual_memory)]
                     {
                         let text = self.text();
@@ -349,8 +355,6 @@ impl CodeMemory {
                         // Flush any in-flight instructions from the pipeline
                         icache_coherence::pipeline_flush_mt().expect("Failed pipeline flush");
                     }
-                    #[cfg(not(has_virtual_memory))]
-                    bail!("this target requires virtual memory to be enabled");
                 }
             }
 
@@ -395,6 +399,10 @@ impl CodeMemory {
             return Ok(());
         }
 
+        if self.mmap.is_always_readonly() {
+            bail!("Unable to apply relocations to readonly MmapVec");
+        }
+
         for (offset, libcall) in self.relocations.iter() {
             let offset = self.text.start + offset;
             let libcall = match libcall {
@@ -413,6 +421,7 @@ impl CodeMemory {
                 #[cfg(not(target_arch = "x86_64"))]
                 obj::LibCall::X86Pshufb => unreachable!(),
             };
+
             self.mmap
                 .as_mut_slice()
                 .as_mut_ptr()

crates/wasmtime/src/runtime/component/component.rs

@@ -214,6 +214,23 @@ impl Component {
         Component::from_parts(engine, code, None)
     }
 
+    /// Same as [`Module::deserialize_raw`], but for components.
+    ///
+    /// See [`Component::deserialize`] for additional information; this method
+    /// works identically except that it will not create a copy of the provided
+    /// memory but will use it directly.
+    ///
+    /// # Unsafety
+    ///
+    /// All of the safety notes from [`Component::deserialize`] apply here as well
+    /// with the additional constraint that the code memory provide by `memory`
+    /// lives for as long as the module and is nevery externally modified for
+    /// the lifetime of the deserialized module.
+    pub unsafe fn deserialize_raw(engine: &Engine, memory: NonNull<[u8]>) -> Result<Component> {
+        let code = engine.load_code_raw(memory, ObjectKind::Component)?;
+        Component::from_parts(engine, code, None)
+    }
+
     /// Same as [`Module::deserialize_file`], but for components.
     ///
     /// Note that the file referenced here must contain contents previously

crates/wasmtime/src/runtime/module.rs

@@ -402,6 +402,24 @@ impl Module {
         Module::from_parts(engine, code, None)
     }
 
+    /// In-place deserialization of an in-memory compiled module previously
+    /// created with [`Module::serialize`] or [`Engine::precompile_module`].
+    ///
+    /// See [`Self::deserialize`] for additional information; this method
+    /// works identically except that it will not create a copy of the provided
+    /// memory but will use it directly.
+    ///
+    /// # Unsafety
+    ///
+    /// All of the safety notes from [`Self::deserialize`] apply here as well
+    /// with the additional constraint that the code memory provide by `memory`
+    /// lives for as long as the module and is nevery externally modified for
+    /// the lifetime of the deserialized module.
+    pub unsafe fn deserialize_raw(engine: &Engine, memory: NonNull<[u8]>) -> Result<Module> {
+        let code = engine.load_code_raw(memory, ObjectKind::Module)?;
+        Module::from_parts(engine, code, None)
+    }
+
     /// Same as [`deserialize`], except that the contents of `path` are read to
     /// deserialize into a [`Module`].
     ///

crates/wasmtime/src/runtime/vm/mmap_vec.rs

@@ -1,13 +1,11 @@
 use crate::prelude::*;
-#[cfg(not(has_virtual_memory))]
 use crate::runtime::vm::send_sync_ptr::SendSyncPtr;
 #[cfg(has_virtual_memory)]
 use crate::runtime::vm::{mmap::UnalignedLength, Mmap};
 #[cfg(not(has_virtual_memory))]
 use alloc::alloc::Layout;
 use alloc::sync::Arc;
 use core::ops::{Deref, Range};
-#[cfg(not(has_virtual_memory))]
 use core::ptr::NonNull;
 #[cfg(feature = "std")]
 use std::fs::File;
@@ -40,6 +38,8 @@ pub enum MmapVec {
         layout: Layout,
     },
     #[doc(hidden)]
+    ExternallyOwned { memory: SendSyncPtr<[u8]> },
+    #[doc(hidden)]
     #[cfg(has_virtual_memory)]
     Mmap {
         mmap: Mmap<UnalignedLength>,
@@ -74,6 +74,11 @@ impl MmapVec {
         MmapVec::Alloc { base, layout }
     }
 
+    fn new_externally_owned(memory: NonNull<[u8]>) -> MmapVec {
+        let memory = SendSyncPtr::new(memory);
+        MmapVec::ExternallyOwned { memory }
+    }
+
     /// Creates a new zero-initialized `MmapVec` with the given `size`
     /// and `alignment`.
     ///
@@ -101,6 +106,25 @@ impl MmapVec {
         MmapVec::from_slice_with_alignment(slice, 1)
     }
 
+    /// Creates a new `MmapVec` from an existing memory region
+    ///
+    /// This method avoids the copy performed by [`Self::from_slice`] by
+    /// directly using the memory region provided. This must be done with
+    /// extreme care, however, as any concurrent modification of the provided
+    /// memory will cause undefined and likely very, very bad things to
+    /// happen.
+    ///
+    /// The memory provided is guaranteed to not be mutated by the runtime.
+    ///
+    /// # Safety
+    ///
+    /// As there is no copy here, the runtime will be making direct readonly use
+    /// of the provided memory. As such, outside writes to this memory region
+    /// will result in undefined and likely very undesirable behavior.
+    pub unsafe fn from_raw(memory: NonNull<[u8]>) -> Result<MmapVec> {
+        Ok(MmapVec::new_externally_owned(memory))
+    }
+
     /// Creates a new `MmapVec` from the contents of an existing
     /// `slice`, with a minimum alignment.
     ///
@@ -110,7 +134,7 @@ impl MmapVec {
     ///
     /// A new `MmapVec` is allocated to hold the contents of `slice` and then
     /// `slice` is copied into the new mmap. It's recommended to avoid this
-    /// method if possible to avoid the need to copy data around.  pub
+    /// method if possible to avoid the need to copy data around.
     pub fn from_slice_with_alignment(slice: &[u8], align: usize) -> Result<MmapVec> {
         let mut result = MmapVec::with_capacity_and_alignment(slice.len(), align)?;
         // SAFETY: The mmap hasn't been made readonly yet so this should be
@@ -121,6 +145,37 @@ impl MmapVec {
         Ok(result)
     }
 
+    /// Return `true` if the `MmapVec` suport virtual memory operations
+    ///
+    /// In some cases, such as when using externally owned memory, the underlying
+    /// platform may support virtual memory but it still may not be legal
+    /// to perform virtual memory operations on this memory.
+    pub fn supports_virtual_memory(&self) -> bool {
+        match self {
+            #[cfg(has_virtual_memory)]
+            MmapVec::Mmap { .. } => true,
+            MmapVec::ExternallyOwned { .. } => false,
+            #[cfg(not(has_virtual_memory))]
+            MmapVec::Alloc { .. } => false,
+        }
+    }
+
+    /// Return true if this `MmapVec` is always readonly
+    ///
+    /// Attempting to get access to mutate readonly memory via
+    /// [`MmapVec::as_mut`] will result in a panic.  Note that this method
+    /// does not change with runtime changes to portions of the code memory
+    /// via `MmapVec::make_readonly` for platforms with virtual memory.
+    pub fn is_always_readonly(&self) -> bool {
+        match self {
+            #[cfg(has_virtual_memory)]
+            MmapVec::Mmap { .. } => false,
+            MmapVec::ExternallyOwned { .. } => true,
+            #[cfg(not(has_virtual_memory))]
+            MmapVec::Alloc { .. } => false,
+        }
+    }
+
     /// Creates a new `MmapVec` which is the given `File` mmap'd into memory.
     ///
     /// This function will determine the file's size and map the full contents
@@ -148,6 +203,9 @@ impl MmapVec {
     ) -> Result<()> {
         let (mmap, len) = match self {
             MmapVec::Mmap { mmap, len } => (mmap, *len),
+            MmapVec::ExternallyOwned { .. } => {
+                bail!("Unable to make externally owned memory executable");
+            }
         };
         assert!(range.start <= range.end);
         assert!(range.end <= len);
@@ -159,6 +217,9 @@ impl MmapVec {
     pub unsafe fn make_readonly(&self, range: Range<usize>) -> Result<()> {
         let (mmap, len) = match self {
             MmapVec::Mmap { mmap, len } => (mmap, *len),
+            MmapVec::ExternallyOwned { .. } => {
+                bail!("Unable to make externally owned memory readonly");
+            }
         };
         assert!(range.start <= range.end);
         assert!(range.end <= len);
@@ -171,6 +232,7 @@ impl MmapVec {
         match self {
             #[cfg(not(has_virtual_memory))]
             MmapVec::Alloc { .. } => None,
+            MmapVec::ExternallyOwned { .. } => None,
             #[cfg(has_virtual_memory)]
             MmapVec::Mmap { mmap, .. } => mmap.original_file(),
         }
@@ -189,13 +251,21 @@ impl MmapVec {
     /// # Unsafety
     ///
     /// This method is only safe if `make_readonly` hasn't been called yet to
-    /// ensure that the memory is indeed writable
+    /// ensure that the memory is indeed writable.  For a MmapVec created from
+    /// a raw pointer using this memory as mutable is only safe if there are
+    /// no outside reads or writes to the memory region.
+    ///
+    /// Externally owned code is implicitly considered to be readonly and this
+    /// code will panic if called on externally owned memory.
     pub unsafe fn as_mut_slice(&mut self) -> &mut [u8] {
         match self {
             #[cfg(not(has_virtual_memory))]
             MmapVec::Alloc { base, layout } => {
                 core::slice::from_raw_parts_mut(base.as_mut(), layout.size())
             }
+            MmapVec::ExternallyOwned { .. } => {
+                panic!("Mutating externally owned memory is prohibited");
+            }
             #[cfg(has_virtual_memory)]
             MmapVec::Mmap { mmap, len } => mmap.slice_mut(0..*len),
         }
@@ -212,6 +282,7 @@ impl Deref for MmapVec {
             MmapVec::Alloc { base, layout } => unsafe {
                 core::slice::from_raw_parts(base.as_ptr(), layout.size())
             },
+            MmapVec::ExternallyOwned { memory } => unsafe { memory.as_ref() },
             #[cfg(has_virtual_memory)]
             MmapVec::Mmap { mmap, len } => {
                 // SAFETY: all bytes for this mmap, which is owned by
@@ -229,6 +300,9 @@ impl Drop for MmapVec {
             MmapVec::Alloc { base, layout, .. } => unsafe {
                 alloc::alloc::dealloc(base.as_mut(), layout.clone());
             },
+            MmapVec::ExternallyOwned { .. } => {
+                // Memory is allocated externally, nothing to do
+            }
             #[cfg(has_virtual_memory)]
             MmapVec::Mmap { .. } => {
                 // Drop impl on the `mmap` takes care of this case.

examples/min-platform/embedding/src/lib.rs

@@ -5,6 +5,7 @@ extern crate alloc;
 
 use alloc::string::ToString;
 use anyhow::Result;
+use core::ptr;
 use wasmtime::{Engine, Instance, Linker, Module, Store};
 
 mod allocator;
@@ -94,7 +95,11 @@ fn simple_host_fn(module: &[u8]) -> Result<()> {
 }
 
 fn deserialize(engine: &Engine, module: &[u8]) -> Result<Option<Module>> {
-    match unsafe { Module::deserialize(engine, module) } {
+    // NOTE: deserialize_raw avoids creating a copy of the module code.  See the
+    // safety notes before using in your embedding.
+    let memory_ptr = ptr::slice_from_raw_parts(module.as_ptr(), module.len());
+    let module_memory = ptr::NonNull::new(memory_ptr.cast_mut()).unwrap();
+    match unsafe { Module::deserialize_raw(engine, module_memory) } {
         Ok(module) => Ok(Some(module)),
         Err(e) => {
             // Currently if custom signals/virtual memory are disabled then this