From 9f67122e3b84dde5742dbfed2fe1d45ccd7b9980 Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 25 Sep 2024 12:45:05 +0200 Subject: [PATCH 1/7] adds library bundling to prelink --- src/lib.rs | 189 +++++++++++-------------------------------------- src/prelink.rs | 134 +++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 148 deletions(-) create mode 100644 src/prelink.rs diff --git a/src/lib.rs b/src/lib.rs index b1eb179..302cb7d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,15 +1,7 @@ #![deny(warnings)] use { - anyhow::{anyhow, bail, ensure, Context, Error, Result}, - async_trait::async_trait, - bytes::Bytes, - component_init::Invoker, - futures::future::FutureExt, - heck::ToSnakeCase, - indexmap::{IndexMap, IndexSet}, - serde::Deserialize, - std::{ + anyhow::{anyhow, bail, ensure, Context, Error, Result}, async_trait::async_trait, bytes::Bytes, component_init::Invoker, futures::future::FutureExt, heck::ToSnakeCase, indexmap::{IndexMap, IndexSet}, prelink::{embedded_helper_utils, embedded_python_standard_library}, serde::Deserialize, std::{ collections::{HashMap, HashSet}, env, fs, io::Cursor, @@ -17,25 +9,16 @@ use { ops::Deref, path::{Path, PathBuf}, str, - }, - summary::{Escape, Locations, Summary}, - tar::Archive, - wasm_convert::IntoValType, - wasm_encoder::{ + }, summary::{Escape, Locations, Summary}, wasm_convert::IntoValType, wasm_encoder::{ CodeSection, ExportKind, ExportSection, Function, FunctionSection, Instruction, Module, TypeSection, - }, - wasmparser::{FuncType, Parser, Payload, TypeRef}, - wasmtime::{ + }, wasmparser::{FuncType, Parser, Payload, TypeRef}, wasmtime::{ component::{Component, Instance, Linker, ResourceTable, ResourceType}, Config, Engine, Store, - }, - wasmtime_wasi::{ + }, wasmtime_wasi::{ pipe::{MemoryInputPipe, MemoryOutputPipe}, DirPerms, FilePerms, WasiCtx, WasiCtxBuilder, WasiView, - }, - wit_parser::{Resolve, TypeDefKind, UnresolvedPackageGroup, WorldId, WorldItem, WorldKey}, - zstd::Decoder, + }, wit_parser::{Resolve, TypeDefKind, UnresolvedPackageGroup, WorldId, WorldItem, WorldKey} }; mod abi; @@ -48,6 +31,7 @@ mod summary; #[cfg(test)] mod test; mod util; +mod prelink; static NATIVE_EXTENSION_SUFFIX: &str = ".cpython-312-wasm32-wasi.so"; @@ -62,6 +46,12 @@ pub struct Ctx { table: ResourceTable, } +pub struct Library { + name: String, + module: Vec, + dl_openable: bool, +} + impl WasiView for Ctx { fn ctx(&mut self) -> &mut WasiCtx { &mut self.wasi @@ -104,7 +94,7 @@ impl TryFrom<(&Path, RawComponentizePyConfig)> for ComponentizePyConfig { } #[derive(Debug)] -struct ConfigContext { +pub struct ConfigContext { module: String, root: PathBuf, path: PathBuf, @@ -207,29 +197,20 @@ pub async fn componentize( .filter_map(|&s| Path::new(s).exists().then_some(s)) .collect::>(); - // Untar the embedded copy of the Python standard library into a temporary directory - let stdlib = tempfile::tempdir()?; - - Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/python-lib.tar.zst" - ))))?) - .unpack(stdlib.path())?; + let embedded_python_standard_lib = embedded_python_standard_library(); + let embedded_helper_utils = embedded_helper_utils(); - // Untar the embedded copy of helper utilities into a temporary directory - let bundled = tempfile::tempdir()?; - - Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/bundled.tar.zst" - ))))?) - .unpack(bundled.path())?; + // Remove non-existent elements from `python_path` so we don't choke on them later + let python_path = &python_path + .iter() + .filter_map(|&s| Path::new(s).exists().then_some(s)) + .collect::>(); // Search `python_path` for native extension libraries and/or componentize-py.toml files. Packages containing // the latter may contain their own WIT files defining their own worlds (in addition to what the caller // specified as paramters), which we'll try to match up with `module_worlds` in the next step. - let mut raw_configs = Vec::new(); - let mut library_path = Vec::with_capacity(python_path.len()); + let mut raw_configs: Vec> = Vec::new(); + let mut library_path: Vec<(&str, Vec)> = Vec::with_capacity(python_path.len()); for path in python_path { let mut libraries = Vec::new(); search_directory( @@ -238,10 +219,12 @@ pub async fn componentize( &mut libraries, &mut raw_configs, &mut HashSet::new(), - )?; + ).unwrap(); library_path.push((*path, libraries)); } + let mut libraries = prelink::bundle_libraries(library_path); + // Validate the paths parsed from any componentize-py.toml files discovered above and match them up with // `module_worlds` entries. Note that we use an `IndexMap` to preserve the order specified in `module_worlds`, // which is required to be topologically sorted with respect to package dependencies. @@ -341,108 +324,11 @@ pub async fn componentize( let summary = Summary::try_new(&resolve, &worlds)?; - struct Library { - name: String, - module: Vec, - dl_openable: bool, - } - - let mut libraries = vec![ - Library { - name: "libcomponentize_py_runtime.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libcomponentize_py_runtime.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libpython3.12.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libpython3.12.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libc.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libc.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libwasi-emulated-mman.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libwasi-emulated-mman.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libwasi-emulated-process-clocks.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libwasi-emulated-process-clocks.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libwasi-emulated-getpid.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libwasi-emulated-getpid.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libwasi-emulated-signal.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libwasi-emulated-signal.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libc++.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libc++.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libc++abi.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libc++abi.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libcomponentize_py_bindings.so".into(), - module: bindings::make_bindings(&resolve, &worlds, &summary)?, - dl_openable: false, - }, - ]; - - for (index, (path, libs)) in library_path.iter().enumerate() { - for library in libs { - let path = library - .strip_prefix(path) - .unwrap() - .to_str() - .context("non-UTF-8 path")? - .replace('\\', "/"); - - libraries.push(Library { - name: format!("/{index}/{path}"), - module: fs::read(library)?, - dl_openable: true, - }); - } - } + libraries.push(Library { + name: "libcomponentize_py_bindings.so".into(), + module: bindings::make_bindings(&resolve, &worlds, &summary)?, + dl_openable: false, + }); // Link all the libraries (including any native extensions) into a single component. let mut linker = wit_component::Linker::default().validate(true); @@ -524,7 +410,7 @@ pub async fn componentize( // application's first and only chance to load any standard or third-party modules since we do not yet include // a virtual filesystem in the component to make those modules available at runtime. - let stdout = MemoryOutputPipe::new(10000); + let stdout: MemoryOutputPipe = MemoryOutputPipe::new(10000); let stderr = MemoryOutputPipe::new(10000); let mut wasi = WasiCtxBuilder::new(); @@ -534,8 +420,8 @@ pub async fn componentize( .env("PYTHONUNBUFFERED", "1") .env("COMPONENTIZE_PY_APP_NAME", app_name) .env("PYTHONHOME", "/python") - .preopened_dir(stdlib.path(), "python", DirPerms::all(), FilePerms::all())? - .preopened_dir(bundled.path(), "bundled", DirPerms::all(), FilePerms::all())?; + .preopened_dir(embedded_python_standard_lib.path(), "python", DirPerms::all(), FilePerms::all())? + .preopened_dir(embedded_helper_utils.path(), "bundled", DirPerms::all(), FilePerms::all())?; // Generate guest mounts for each host directory in `python_path`. for (index, path) in python_path.iter().enumerate() { @@ -628,7 +514,7 @@ pub async fn componentize( Ok(()) } - replace(bundled.path(), "proxy", &module)?; + replace(embedded_helper_utils.path(), "proxy", &module)?; }; for (mounts, world_dir) in world_dir_mounts.iter() { @@ -663,6 +549,10 @@ pub async fn componentize( let engine = Engine::new(&config)?; let mut linker = Linker::new(&engine); + + //whenever the guest adds to fulfil x or y, + //add to linker is giving access to wasi so it can access the host file system + let added_to_linker = if let Some(add_to_linker) = add_to_linker { add_to_linker(&mut linker)?; true @@ -672,6 +562,9 @@ pub async fn componentize( let mut store = Store::new(&engine, Ctx { wasi, table }); + //can stub out wasi, if does try to access file system then trap it + //there are no env variables etc + let app_name = app_name.to_owned(); let component = component_init::initialize_staged( &component, diff --git a/src/prelink.rs b/src/prelink.rs new file mode 100644 index 0000000..307360b --- /dev/null +++ b/src/prelink.rs @@ -0,0 +1,134 @@ +#![deny(warnings)] + +use std::{fs, io::Cursor}; + +use anyhow::Context; +use tar::Archive; +use tempfile::TempDir; +use zstd::Decoder; + +use crate::Library; + +pub fn embedded_python_standard_library() -> TempDir { + // Untar the embedded copy of the Python standard library into a temporary directory + let stdlib = tempfile::tempdir().expect("could not create temp dirfor python stnadard lib"); + + Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/python-lib.tar.zst" + )))).unwrap()) + .unpack(stdlib.path()).unwrap(); + + return stdlib; +} + +pub fn embedded_helper_utils() -> TempDir { + // Untar the embedded copy of helper utilities into a temporary directory + let bundled = tempfile::tempdir().expect("could not create tempdir for embedded helper utils"); + + Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/bundled.tar.zst" + )))).unwrap()) + .unpack(bundled.path()).unwrap(); + + return bundled; +} + +pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Vec { + + let mut libraries = vec![ + Library { + name: "libcomponentize_py_runtime.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libcomponentize_py_runtime.so.zst" + )))).unwrap(), + dl_openable: false, + }, + Library { + name: "libpython3.12.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libpython3.12.so.zst" + )))).unwrap(), + dl_openable: false, + }, + Library { + name: "libc.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libc.so.zst" + )))).unwrap(), + dl_openable: false, + }, + Library { + name: "libwasi-emulated-mman.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libwasi-emulated-mman.so.zst" + )))).unwrap(), + dl_openable: false, + }, + Library { + name: "libwasi-emulated-process-clocks.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libwasi-emulated-process-clocks.so.zst" + )))).unwrap(), + dl_openable: false, + }, + Library { + name: "libwasi-emulated-getpid.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libwasi-emulated-getpid.so.zst" + )))).unwrap(), + dl_openable: false, + }, + Library { + name: "libwasi-emulated-signal.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libwasi-emulated-signal.so.zst" + )))).unwrap(), + dl_openable: false, + }, + Library { + name: "libc++.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libc++.so.zst" + )))).unwrap(), + dl_openable: false, + }, + Library { + name: "libc++abi.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libc++abi.so.zst" + )))).unwrap(), + dl_openable: false, + } + ]; + + for (index, (path, libs)) in library_path.iter().enumerate() { + for library in libs { + let path = library + .strip_prefix(path) + .unwrap() + .to_str() + .context("non-UTF-8 path") + .unwrap() + .replace('\\', "/"); + + libraries.push(Library { + name: format!("/{index}/{path}"), + module: fs::read(library).unwrap(), + dl_openable: true, + }); + } + } + + return libraries; +} From 0913c816dc77653a84402ca878393f3c041aa43d Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 25 Sep 2024 12:48:05 +0200 Subject: [PATCH 2/7] removes comments --- src/lib.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 302cb7d..0f2fe33 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -410,7 +410,7 @@ pub async fn componentize( // application's first and only chance to load any standard or third-party modules since we do not yet include // a virtual filesystem in the component to make those modules available at runtime. - let stdout: MemoryOutputPipe = MemoryOutputPipe::new(10000); + let stdout = MemoryOutputPipe::new(10000); let stderr = MemoryOutputPipe::new(10000); let mut wasi = WasiCtxBuilder::new(); @@ -550,9 +550,6 @@ pub async fn componentize( let mut linker = Linker::new(&engine); - //whenever the guest adds to fulfil x or y, - //add to linker is giving access to wasi so it can access the host file system - let added_to_linker = if let Some(add_to_linker) = add_to_linker { add_to_linker(&mut linker)?; true @@ -562,9 +559,6 @@ pub async fn componentize( let mut store = Store::new(&engine, Ctx { wasi, table }); - //can stub out wasi, if does try to access file system then trap it - //there are no env variables etc - let app_name = app_name.to_owned(); let component = component_init::initialize_staged( &component, From af0680587ec4a10de76e9156330b3e9e8adff563 Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 25 Sep 2024 12:49:39 +0200 Subject: [PATCH 3/7] removes dupe python path check --- src/lib.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0f2fe33..86479ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,12 +200,6 @@ pub async fn componentize( let embedded_python_standard_lib = embedded_python_standard_library(); let embedded_helper_utils = embedded_helper_utils(); - // Remove non-existent elements from `python_path` so we don't choke on them later - let python_path = &python_path - .iter() - .filter_map(|&s| Path::new(s).exists().then_some(s)) - .collect::>(); - // Search `python_path` for native extension libraries and/or componentize-py.toml files. Packages containing // the latter may contain their own WIT files defining their own worlds (in addition to what the caller // specified as paramters), which we'll try to match up with `module_worlds` in the next step. @@ -549,7 +543,6 @@ pub async fn componentize( let engine = Engine::new(&config)?; let mut linker = Linker::new(&engine); - let added_to_linker = if let Some(add_to_linker) = add_to_linker { add_to_linker(&mut linker)?; true From 5b0f0f0dde74e11d7e4deb41d29346a9422bfe7a Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 25 Sep 2024 12:55:25 +0200 Subject: [PATCH 4/7] refactoring --- src/lib.rs | 8 ++++---- src/prelink.rs | 36 ++++++++++++++++++------------------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 86479ad..aaef867 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -197,8 +197,8 @@ pub async fn componentize( .filter_map(|&s| Path::new(s).exists().then_some(s)) .collect::>(); - let embedded_python_standard_lib = embedded_python_standard_library(); - let embedded_helper_utils = embedded_helper_utils(); + let embedded_python_standard_lib = embedded_python_standard_library().unwrap(); + let embedded_helper_utils = embedded_helper_utils().unwrap(); // Search `python_path` for native extension libraries and/or componentize-py.toml files. Packages containing // the latter may contain their own WIT files defining their own worlds (in addition to what the caller @@ -213,11 +213,11 @@ pub async fn componentize( &mut libraries, &mut raw_configs, &mut HashSet::new(), - ).unwrap(); + )?; library_path.push((*path, libraries)); } - let mut libraries = prelink::bundle_libraries(library_path); + let mut libraries = prelink::bundle_libraries(library_path).unwrap(); // Validate the paths parsed from any componentize-py.toml files discovered above and match them up with // `module_worlds` entries. Note that we use an `IndexMap` to preserve the order specified in `module_worlds`, diff --git a/src/prelink.rs b/src/prelink.rs index 307360b..c7d3b0e 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -1,6 +1,6 @@ #![deny(warnings)] -use std::{fs, io::Cursor}; +use std::{fs, io::{self, Cursor}}; use anyhow::Context; use tar::Archive; @@ -9,33 +9,33 @@ use zstd::Decoder; use crate::Library; -pub fn embedded_python_standard_library() -> TempDir { +pub fn embedded_python_standard_library() -> Result { // Untar the embedded copy of the Python standard library into a temporary directory let stdlib = tempfile::tempdir().expect("could not create temp dirfor python stnadard lib"); Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/python-lib.tar.zst" - )))).unwrap()) + ))))?) .unpack(stdlib.path()).unwrap(); - return stdlib; + return Ok(stdlib); } -pub fn embedded_helper_utils() -> TempDir { +pub fn embedded_helper_utils() -> Result { // Untar the embedded copy of helper utilities into a temporary directory let bundled = tempfile::tempdir().expect("could not create tempdir for embedded helper utils"); Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/bundled.tar.zst" - )))).unwrap()) + ))))?) .unpack(bundled.path()).unwrap(); - return bundled; + return Ok(bundled); } -pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Vec { +pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Result, io::Error> { let mut libraries = vec![ Library { @@ -43,7 +43,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libcomponentize_py_runtime.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, }, Library { @@ -51,7 +51,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libpython3.12.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, }, Library { @@ -59,7 +59,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libc.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, }, Library { @@ -67,7 +67,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libwasi-emulated-mman.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, }, Library { @@ -75,7 +75,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libwasi-emulated-process-clocks.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, }, Library { @@ -83,7 +83,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libwasi-emulated-getpid.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, }, Library { @@ -91,7 +91,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libwasi-emulated-signal.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, }, Library { @@ -99,7 +99,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libc++.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, }, Library { @@ -107,7 +107,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/libc++abi.so.zst" - )))).unwrap(), + ))))?, dl_openable: false, } ]; @@ -130,5 +130,5 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> V } } - return libraries; + return Ok(libraries); } From 0250b344c6d9b530dcb801659f7cb17df0104c91 Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 25 Sep 2024 13:59:40 +0200 Subject: [PATCH 5/7] formatting --- src/lib.rs | 45 ++++++++++++++++++++++++++++++++++++--------- src/prelink.rs | 32 +++++++++++++++++++------------- 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index aaef867..a8c003f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,16 @@ #![deny(warnings)] use { - anyhow::{anyhow, bail, ensure, Context, Error, Result}, async_trait::async_trait, bytes::Bytes, component_init::Invoker, futures::future::FutureExt, heck::ToSnakeCase, indexmap::{IndexMap, IndexSet}, prelink::{embedded_helper_utils, embedded_python_standard_library}, serde::Deserialize, std::{ + anyhow::{anyhow, bail, ensure, Context, Error, Result}, + async_trait::async_trait, + bytes::Bytes, + component_init::Invoker, + futures::future::FutureExt, + heck::ToSnakeCase, + indexmap::{IndexMap, IndexSet}, + prelink::{embedded_helper_utils, embedded_python_standard_library}, + serde::Deserialize, + std::{ collections::{HashMap, HashSet}, env, fs, io::Cursor, @@ -9,29 +18,36 @@ use { ops::Deref, path::{Path, PathBuf}, str, - }, summary::{Escape, Locations, Summary}, wasm_convert::IntoValType, wasm_encoder::{ + }, + summary::{Escape, Locations, Summary}, + wasm_convert::IntoValType, + wasm_encoder::{ CodeSection, ExportKind, ExportSection, Function, FunctionSection, Instruction, Module, TypeSection, - }, wasmparser::{FuncType, Parser, Payload, TypeRef}, wasmtime::{ + }, + wasmparser::{FuncType, Parser, Payload, TypeRef}, + wasmtime::{ component::{Component, Instance, Linker, ResourceTable, ResourceType}, Config, Engine, Store, - }, wasmtime_wasi::{ + }, + wasmtime_wasi::{ pipe::{MemoryInputPipe, MemoryOutputPipe}, DirPerms, FilePerms, WasiCtx, WasiCtxBuilder, WasiView, - }, wit_parser::{Resolve, TypeDefKind, UnresolvedPackageGroup, WorldId, WorldItem, WorldKey} + }, + wit_parser::{Resolve, TypeDefKind, UnresolvedPackageGroup, WorldId, WorldItem, WorldKey}, }; mod abi; mod bindgen; mod bindings; pub mod command; +mod prelink; #[cfg(feature = "pyo3")] mod python; mod summary; #[cfg(test)] mod test; mod util; -mod prelink; static NATIVE_EXTENSION_SUFFIX: &str = ".cpython-312-wasm32-wasi.so"; @@ -204,7 +220,8 @@ pub async fn componentize( // the latter may contain their own WIT files defining their own worlds (in addition to what the caller // specified as paramters), which we'll try to match up with `module_worlds` in the next step. let mut raw_configs: Vec> = Vec::new(); - let mut library_path: Vec<(&str, Vec)> = Vec::with_capacity(python_path.len()); + let mut library_path: Vec<(&str, Vec)> = + Vec::with_capacity(python_path.len()); for path in python_path { let mut libraries = Vec::new(); search_directory( @@ -414,8 +431,18 @@ pub async fn componentize( .env("PYTHONUNBUFFERED", "1") .env("COMPONENTIZE_PY_APP_NAME", app_name) .env("PYTHONHOME", "/python") - .preopened_dir(embedded_python_standard_lib.path(), "python", DirPerms::all(), FilePerms::all())? - .preopened_dir(embedded_helper_utils.path(), "bundled", DirPerms::all(), FilePerms::all())?; + .preopened_dir( + embedded_python_standard_lib.path(), + "python", + DirPerms::all(), + FilePerms::all(), + )? + .preopened_dir( + embedded_helper_utils.path(), + "bundled", + DirPerms::all(), + FilePerms::all(), + )?; // Generate guest mounts for each host directory in `python_path`. for (index, path) in python_path.iter().enumerate() { diff --git a/src/prelink.rs b/src/prelink.rs index c7d3b0e..0df393b 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -1,6 +1,9 @@ #![deny(warnings)] -use std::{fs, io::{self, Cursor}}; +use std::{ + fs, + io::{self, Cursor}, +}; use anyhow::Context; use tar::Archive; @@ -10,16 +13,17 @@ use zstd::Decoder; use crate::Library; pub fn embedded_python_standard_library() -> Result { - // Untar the embedded copy of the Python standard library into a temporary directory - let stdlib = tempfile::tempdir().expect("could not create temp dirfor python stnadard lib"); + // Untar the embedded copy of the Python standard library into a temporary directory + let stdlib = tempfile::tempdir().expect("could not create temp dirfor python stnadard lib"); - Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/python-lib.tar.zst" - ))))?) - .unpack(stdlib.path()).unwrap(); + Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/python-lib.tar.zst" + ))))?) + .unpack(stdlib.path()) + .unwrap(); - return Ok(stdlib); + return Ok(stdlib); } pub fn embedded_helper_utils() -> Result { @@ -30,13 +34,15 @@ pub fn embedded_helper_utils() -> Result { env!("OUT_DIR"), "/bundled.tar.zst" ))))?) - .unpack(bundled.path()).unwrap(); + .unpack(bundled.path()) + .unwrap(); return Ok(bundled); } -pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Result, io::Error> { - +pub fn bundle_libraries( + library_path: Vec<(&str, Vec)>, +) -> Result, io::Error> { let mut libraries = vec![ Library { name: "libcomponentize_py_runtime.so".into(), @@ -109,7 +115,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> R "/libc++abi.so.zst" ))))?, dl_openable: false, - } + }, ]; for (index, (path, libs)) in library_path.iter().enumerate() { From f9a1803fadae7189e3850bf1127b300f610241d1 Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 25 Sep 2024 14:19:01 +0200 Subject: [PATCH 6/7] remove unnecessary return --- src/prelink.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/prelink.rs b/src/prelink.rs index 0df393b..3e94445 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -23,7 +23,7 @@ pub fn embedded_python_standard_library() -> Result { .unpack(stdlib.path()) .unwrap(); - return Ok(stdlib); + Ok(stdlib); } pub fn embedded_helper_utils() -> Result { @@ -37,7 +37,7 @@ pub fn embedded_helper_utils() -> Result { .unpack(bundled.path()) .unwrap(); - return Ok(bundled); + Ok(bundled); } pub fn bundle_libraries( @@ -136,5 +136,5 @@ pub fn bundle_libraries( } } - return Ok(libraries); + Ok(libraries); } From b1e3f2f2bbb213625fe0682dfbc45c0f3d8caed9 Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 25 Sep 2024 14:21:17 +0200 Subject: [PATCH 7/7] whoops --- src/prelink.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/prelink.rs b/src/prelink.rs index 3e94445..debee07 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -23,7 +23,7 @@ pub fn embedded_python_standard_library() -> Result { .unpack(stdlib.path()) .unwrap(); - Ok(stdlib); + Ok(stdlib) } pub fn embedded_helper_utils() -> Result { @@ -37,7 +37,7 @@ pub fn embedded_helper_utils() -> Result { .unpack(bundled.path()) .unwrap(); - Ok(bundled); + Ok(bundled) } pub fn bundle_libraries( @@ -136,5 +136,5 @@ pub fn bundle_libraries( } } - Ok(libraries); + Ok(libraries) }