Skip to content

Commit f1878a8

Browse files
author
Gentle
committed
add zlib, bzip2, libxz and zstd
1 parent 1571725 commit f1878a8

File tree

2 files changed

+172
-10
lines changed

2 files changed

+172
-10
lines changed

‎Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,17 @@ tempfile = "3.13.0"
6565

6666
[build-dependencies]
6767
anyhow = "1.0.89"
68+
reqwest = { version = "0.12.15", features = [
69+
"blocking",
70+
"brotli",
71+
"deflate",
72+
"gzip",
73+
"zstd",
74+
] }
6875
tar = "0.4.41"
6976
zstd = "0.13.2"
7077
test-generator = { path = "test-generator" }
78+
flate2 = "1.1.1"
7179

7280
[workspace]
7381
members = ["runtime", "shared", "test-generator"]

‎build.rs‎

Lines changed: 164 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -261,22 +261,20 @@ fn add(builder: &mut Builder<impl Write>, root: &Path, path: &Path) -> Result<()
261261
}
262262

263263
fn find_python_binary(cpython_dir: &Path) -> Option<&'static str> {
264-
for name in ["python", "python.exe"] {
265-
if cpython_dir.join(name).is_file() {
266-
return Some(name);
267-
}
268-
}
269-
None
264+
["python", "python.exe"]
265+
.into_iter()
266+
.find(|name| cpython_dir.join(name).is_file())
267+
.map(|v| v as _)
270268
}
271269

272270
fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> {
273271
let cpython_wasi_dir = repo_dir.join("cpython/builddir/wasi");
274272
if !cpython_wasi_dir.join("libpython3.14.so").exists() {
273+
fs::create_dir_all(&cpython_wasi_dir)?;
275274
if !cpython_wasi_dir.join("libpython3.14.a").exists() {
276275
let cpython_native_dir = repo_dir.join("cpython/builddir/build");
277-
if let None = find_python_binary(&cpython_native_dir) {
276+
if find_python_binary(&cpython_native_dir).is_none() {
278277
fs::create_dir_all(&cpython_native_dir)?;
279-
fs::create_dir_all(&cpython_wasi_dir)?;
280278

281279
run(Command::new("../../configure")
282280
.current_dir(&cpython_native_dir)
@@ -288,9 +286,15 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> {
288286
run(Command::new("make").current_dir(&cpython_native_dir))?;
289287
}
290288
let Some(python_executable) = find_python_binary(&cpython_native_dir) else {
291-
bail!("python binary not found");
289+
anyhow::bail!("python binary not found");
292290
};
293291

292+
let lib_install_dir = cpython_wasi_dir.join("deps");
293+
build_zlib(wasi_sdk, &lib_install_dir)?;
294+
build_bzip2(wasi_sdk, &lib_install_dir)?;
295+
build_xz(wasi_sdk, &lib_install_dir)?;
296+
build_zstd(wasi_sdk, &lib_install_dir)?;
297+
294298
let config_guess =
295299
run(Command::new("../../config.guess").current_dir(&cpython_wasi_dir))?;
296300

@@ -299,7 +303,14 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> {
299303
"CONFIG_SITE",
300304
"../../Tools/wasm/wasi/config.site-wasm32-wasi",
301305
)
302-
.env("CFLAGS", "-fPIC")
306+
.env(
307+
"CFLAGS",
308+
format!("-fPIC -I{}/deps/include", cpython_wasi_dir.display()),
309+
)
310+
.env(
311+
"LDFLAGS",
312+
format!("-L{}/deps/lib", cpython_wasi_dir.display()),
313+
)
303314
.current_dir(&cpython_wasi_dir)
304315
.args([
305316
"../../configure",
@@ -336,6 +347,10 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> {
336347
.arg(cpython_wasi_dir.join("Modules/_hacl/libHacl_Hash_SHA3.a"))
337348
.arg(cpython_wasi_dir.join("Modules/_decimal/libmpdec/libmpdec.a"))
338349
.arg(cpython_wasi_dir.join("Modules/expat/libexpat.a"))
350+
.arg(cpython_wasi_dir.join("deps/lib/libz.a"))
351+
.arg(cpython_wasi_dir.join("deps/lib/libbz2.a"))
352+
.arg(cpython_wasi_dir.join("deps/lib/liblzma.a"))
353+
.arg(cpython_wasi_dir.join("deps/lib/libzstd.a"))
339354
.arg("-lwasi-emulated-signal")
340355
.arg("-lwasi-emulated-getpid")
341356
.arg("-lwasi-emulated-process-clocks")
@@ -395,3 +410,142 @@ fn make_pyo3_config(repo_dir: &Path) -> Result<()> {
395410

396411
Ok(())
397412
}
413+
414+
fn fetch_extract(url: &str, out_dir: &Path) -> Result<()> {
415+
let response = reqwest::blocking::get(url)?;
416+
let decoder = flate2::read::GzDecoder::new(response);
417+
let mut archive = tar::Archive::new(decoder);
418+
archive.unpack(out_dir)?;
419+
Ok(())
420+
}
421+
422+
fn add_compile_envs(wasi_sdk: &Path, command: &mut Command) {
423+
let sysroot = wasi_sdk.join("share/wasi-sysroot");
424+
let sysroot = sysroot.to_string_lossy();
425+
command
426+
.env("AR", wasi_sdk.join("bin/ar"))
427+
.env("CC", wasi_sdk.join("bin/clang"))
428+
.env("RANLIB", wasi_sdk.join("bin/ranlib"))
429+
.env(
430+
"CFLAGS",
431+
format!("--target=wasm32-wasi --sysroot={sysroot} -I{sysroot}/include/wasm32-wasip1 -D_WASI_EMULATED_SIGNAL -fPIC"),
432+
)
433+
.env(
434+
"LDFLAGS",
435+
format!("--target=wasm32-wasip2 --sysroot={sysroot} -L{sysroot}/lib -lwasi-emulated-signal")
436+
);
437+
}
438+
439+
fn copy_headers(from: &Path, to: &Path) -> Result<()> {
440+
for entry in fs::read_dir(from)? {
441+
let entry = entry?;
442+
let path = entry.path();
443+
444+
if let Some(ext) = path.extension() {
445+
if ext == "h" {
446+
let filename = path.file_name().unwrap();
447+
fs::copy(&path, to.join(filename))?;
448+
}
449+
}
450+
}
451+
Ok(())
452+
}
453+
454+
fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> {
455+
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
456+
fetch_extract(
457+
"https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz",
458+
&out_dir,
459+
)?;
460+
let src_dir = out_dir.join("zlib-1.3.1");
461+
let mut configure = Command::new("./configure");
462+
configure
463+
.current_dir(&src_dir)
464+
.arg("--static")
465+
.arg(format!("--prefix={}", install_dir.display()));
466+
add_compile_envs(wasi_sdk, &mut configure);
467+
run(&mut configure)?;
468+
let mut make = Command::new("make");
469+
add_compile_envs(wasi_sdk, &mut make);
470+
make.current_dir(src_dir)
471+
.arg(format!("AR={}", wasi_sdk.join("bin/ar").display()))
472+
.arg("ARFLAGS=rcs")
473+
.arg(format!("CC={}", wasi_sdk.join("bin/clang").display()))
474+
.arg("static")
475+
.arg("install");
476+
run(&mut make)?;
477+
Ok(())
478+
}
479+
480+
fn build_bzip2(wasi_sdk: &Path, install_dir: &Path) -> Result<()> {
481+
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
482+
fetch_extract(
483+
"https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz",
484+
&out_dir,
485+
)?;
486+
let src_dir = out_dir.join("bzip2-1.0.8");
487+
let mut command = Command::new("make");
488+
command
489+
.current_dir(&src_dir)
490+
.arg(format!("CC={}", wasi_sdk.join("bin/clang").display()))
491+
.arg(format!("AR={}", wasi_sdk.join("bin/ar").display()))
492+
.arg(format!("RANLIB={}", wasi_sdk.join("bin/ranlib").display()))
493+
.arg("CFLAGS=\"-fPIC\"")
494+
.arg("libbz2.a");
495+
add_compile_envs(wasi_sdk, &mut command);
496+
run(&mut command)?;
497+
copy_headers(&src_dir, &install_dir.join("include"))?;
498+
fs::copy(src_dir.join("libbz2.a"), install_dir.join("lib/libbz2.a"))?;
499+
Ok(())
500+
}
501+
502+
fn build_xz(wasi_sdk: &Path, install_dir: &Path) -> Result<()> {
503+
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
504+
fetch_extract(
505+
"https://github.com/tukaani-project/xz/releases/download/v5.8.1/xz-5.8.1.tar.gz",
506+
&out_dir,
507+
)?;
508+
let src_dir = out_dir.join("xz-5.8.1");
509+
let mut command = Command::new("./autogen.sh");
510+
command.current_dir(&src_dir);
511+
add_compile_envs(wasi_sdk, &mut command);
512+
run(&mut command)?;
513+
let mut command = Command::new("./configure");
514+
command
515+
.current_dir(&src_dir)
516+
.arg(format!("--prefix={}", install_dir.display()))
517+
.arg("--host=wasm32-unknown-wasi")
518+
.arg("--enable-threads=no")
519+
.arg("--disable-xz")
520+
.arg("--disable-xzdec")
521+
.arg("--disable-lzmadec")
522+
.arg("--disable-lzmainfo")
523+
.arg("--disable-lzma-links")
524+
.arg("--disable-scripts")
525+
.arg("--disable-doc")
526+
.arg("--disable-shared")
527+
.arg("--enable-static");
528+
add_compile_envs(wasi_sdk, &mut command);
529+
run(&mut command)?;
530+
let mut command = Command::new("make");
531+
add_compile_envs(wasi_sdk, &mut command);
532+
command.current_dir(&src_dir).arg("all").arg("install");
533+
run(&mut command)?;
534+
Ok(())
535+
}
536+
537+
fn build_zstd(wasi_sdk: &Path, install_dir: &Path) -> Result<()> {
538+
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
539+
fetch_extract(
540+
"https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz",
541+
&out_dir,
542+
)?;
543+
let src_dir = out_dir.join("zstd-1.5.7/lib");
544+
let mut command = Command::new("make");
545+
add_compile_envs(wasi_sdk, &mut command);
546+
command.current_dir(&src_dir).arg("libzstd.a");
547+
run(&mut command)?;
548+
copy_headers(&src_dir, &install_dir.join("include"))?;
549+
fs::copy(src_dir.join("libzstd.a"), install_dir.join("lib/libzstd.a"))?;
550+
Ok(())
551+
}

0 commit comments

Comments
 (0)