From 3c3900e7d3369b803153607d30931c3e21094719 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Wed, 6 Nov 2024 10:48:46 +0100 Subject: [PATCH 01/21] Migrate cli test to integration test --- .github/workflows/test_examples.sh | 4 -- Cargo.lock | 73 ++++++++++++++++++++++++++++++ Cargo.toml | 3 +- tests/integration.rs | 31 +++++++++++++ 4 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 tests/integration.rs diff --git a/.github/workflows/test_examples.sh b/.github/workflows/test_examples.sh index dddd905..1fa5146 100755 --- a/.github/workflows/test_examples.sh +++ b/.github/workflows/test_examples.sh @@ -8,10 +8,6 @@ export WASMTIME_BACKTRACE_DETAILS=1 cargo build --release -# CLI -(cd examples/cli \ - && ../../target/release/componentize-py -d ../../wit -w wasi:cli/command@0.2.0 componentize app -o cli.wasm \ - && wasmtime run cli.wasm) # HTTP # Just compiling for now diff --git a/Cargo.lock b/Cargo.lock index 9f450e0..a91b32a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,6 +126,22 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +[[package]] +name = "assert_cmd" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -212,6 +228,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bstr" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -394,6 +421,7 @@ name = "componentize-py" version = "0.15.2" dependencies = [ "anyhow", + "assert_cmd", "async-trait", "bincode", "bytes", @@ -643,6 +671,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.10.7" @@ -694,6 +728,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "either" version = "1.13.0" @@ -1394,6 +1434,33 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97" +dependencies = [ + "anstyle", + "difflib", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" + +[[package]] +name = "predicates-tree" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "pretty_env_logger" version = "0.5.0" @@ -1919,6 +1986,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "test-generator" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 8f1e12c..ef60f25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,8 +49,9 @@ toml = "0.8.19" semver = "1.0.23" [dev-dependencies] -proptest = "1.5.0" +assert_cmd = "2.0.16" hex = "0.4.3" +proptest = "1.5.0" tempfile = "3.13.0" [build-dependencies] diff --git a/tests/integration.rs b/tests/integration.rs new file mode 100644 index 0000000..84a786e --- /dev/null +++ b/tests/integration.rs @@ -0,0 +1,31 @@ +use assert_cmd::Command; + +#[test] +fn cli_example() -> anyhow::Result<()> { + let dir = "./examples/cli"; + + Command::cargo_bin("componentize-py")? + .current_dir(dir) + .args([ + "-d", + "../../wit", + "-w", + "wasi:cli/command@0.2.0", + "componentize", + "app", + "-o", + "cli.wasm", + ]) + .assert() + .success() + .stdout("Component built successfully\n"); + + Command::new("wasmtime") + .current_dir(dir) + .args(["run", "cli.wasm"]) + .assert() + .success() + .stdout("Hello, world!\n"); + + Ok(()) +} From 0cd01097001fea0be669024f5453e2d51f4aec3a Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Wed, 6 Nov 2024 11:05:13 +0100 Subject: [PATCH 02/21] Move test files to temp dirs --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + tests/integration.rs | 16 +++++++++++----- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a91b32a..5b7224f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -429,6 +429,7 @@ dependencies = [ "clap", "component-init", "componentize-py-shared", + "fs_extra", "futures", "heck 0.5.0", "hex", @@ -857,6 +858,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" diff --git a/Cargo.toml b/Cargo.toml index ef60f25..29f8884 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ semver = "1.0.23" [dev-dependencies] assert_cmd = "2.0.16" +fs_extra = "1.3.0" hex = "0.4.3" proptest = "1.5.0" tempfile = "3.13.0" diff --git a/tests/integration.rs b/tests/integration.rs index 84a786e..19004ec 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,18 +1,24 @@ use assert_cmd::Command; +use fs_extra::dir::CopyOptions; #[test] fn cli_example() -> anyhow::Result<()> { - let dir = "./examples/cli"; + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/cli", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; Command::cargo_bin("componentize-py")? - .current_dir(dir) + .current_dir(dir.path()) .args([ "-d", - "../../wit", + "wit", "-w", "wasi:cli/command@0.2.0", "componentize", - "app", + "cli.app", "-o", "cli.wasm", ]) @@ -21,7 +27,7 @@ fn cli_example() -> anyhow::Result<()> { .stdout("Component built successfully\n"); Command::new("wasmtime") - .current_dir(dir) + .current_dir(dir.path()) .args(["run", "cli.wasm"]) .assert() .success() From 919d8af6cf5b527365b51ae1919a82699435ef6a Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Wed, 6 Nov 2024 12:14:02 +0100 Subject: [PATCH 03/21] Migrate sandbox test --- .github/workflows/test_examples.sh | 5 --- tests/integration.rs | 55 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_examples.sh b/.github/workflows/test_examples.sh index 1fa5146..6931050 100755 --- a/.github/workflows/test_examples.sh +++ b/.github/workflows/test_examples.sh @@ -21,11 +21,6 @@ cargo build --release && ../../target/release/componentize-py -d ../../wit -w matrix-math componentize app -o matrix-math.wasm \ && wasmtime run matrix-math.wasm '[[1, 2], [4, 5], [6, 7]]' '[[1, 2, 3], [4, 5, 6]]') -# Sandbox -(cd examples/sandbox \ - && ../../target/release/componentize-py -d sandbox.wit componentize --stub-wasi guest -o sandbox.wasm \ - && python -m wasmtime.bindgen sandbox.wasm --out-dir sandbox \ - && python host.py "2 + 2") # TCP # Just compiling for now diff --git a/tests/integration.rs b/tests/integration.rs index 19004ec..a635253 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -35,3 +35,58 @@ fn cli_example() -> anyhow::Result<()> { Ok(()) } + +#[test] +fn sandbox_example() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items(&["./examples/sandbox"], dir.path(), &CopyOptions::new())?; + let path = dir.path().join("sandbox"); + + Command::cargo_bin("componentize-py")? + .current_dir(&path) + .args([ + "-d", + "sandbox.wit", + "componentize", + "--stub-wasi", + "guest", + "-o", + "sandbox.wasm", + ]) + .assert() + .success() + .stdout("Component built successfully\n"); + + Command::new("python3") + .current_dir(dir.path()) + .args(["-m", "venv", ".venv"]) + .assert() + .success(); + + Command::new("./.venv/bin/pip") + .current_dir(dir.path()) + .args(["install", "wasmtime"]) + .assert() + .success(); + + Command::new("../.venv/bin/python") + .current_dir(&path) + .args([ + "-m", + "wasmtime.bindgen", + "sandbox.wasm", + "--out-dir", + "sandbox", + ]) + .assert() + .success(); + + Command::new("../.venv/bin/python") + .current_dir(&path) + .args(["host.py", "2 + 2"]) + .assert() + .success() + .stdout("result: 4\n"); + + Ok(()) +} From 62b2c1467c28e78629edbc591401159268ac5848 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Wed, 6 Nov 2024 12:31:59 +0100 Subject: [PATCH 04/21] Migrate matrix-math test --- .github/workflows/test_examples.sh | 8 ---- tests/integration.rs | 65 ++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test_examples.sh b/.github/workflows/test_examples.sh index 6931050..83fd46d 100755 --- a/.github/workflows/test_examples.sh +++ b/.github/workflows/test_examples.sh @@ -14,14 +14,6 @@ cargo build --release (cd examples/http \ && ../../target/release/componentize-py -d ../../wit -w wasi:http/proxy@0.2.0 componentize app -o http.wasm) -# Matrix Math -(cd examples/matrix-math \ - && curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz \ - && tar xf numpy-wasi.tar.gz \ - && ../../target/release/componentize-py -d ../../wit -w matrix-math componentize app -o matrix-math.wasm \ - && wasmtime run matrix-math.wasm '[[1, 2], [4, 5], [6, 7]]' '[[1, 2, 3], [4, 5, 6]]') - - # TCP # Just compiling for now (cd examples/tcp \ diff --git a/tests/integration.rs b/tests/integration.rs index a635253..282a51c 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -9,16 +9,17 @@ fn cli_example() -> anyhow::Result<()> { dir.path(), &CopyOptions::new(), )?; + let path = dir.path().join("cli"); Command::cargo_bin("componentize-py")? - .current_dir(dir.path()) + .current_dir(&path) .args([ "-d", - "wit", + "../wit", "-w", "wasi:cli/command@0.2.0", "componentize", - "cli.app", + "app", "-o", "cli.wasm", ]) @@ -27,7 +28,7 @@ fn cli_example() -> anyhow::Result<()> { .stdout("Component built successfully\n"); Command::new("wasmtime") - .current_dir(dir.path()) + .current_dir(&path) .args(["run", "cli.wasm"]) .assert() .success() @@ -36,6 +37,62 @@ fn cli_example() -> anyhow::Result<()> { Ok(()) } +#[test] +fn matrix_math_example() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/matrix-math", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; + let path = dir.path().join("matrix-math"); + + Command::new("curl") + .current_dir(&path) + .args([ + "-OL", + "https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz", + ]) + .assert() + .success(); + + Command::new("tar") + .current_dir(&path) + .args(["xf", "numpy-wasi.tar.gz"]) + .assert() + .success(); + + Command::cargo_bin("componentize-py")? + .current_dir(&path) + .args([ + "-d", + "../wit", + "-w", + "matrix-math", + "componentize", + "app", + "-o", + "matrix-math.wasm", + ]) + .assert() + .success() + .stdout("Component built successfully\n"); + + Command::new("wasmtime") + .current_dir(&path) + .args([ + "run", + "matrix-math.wasm", + "[[1, 2], [4, 5], [6, 7]]", + "[[1, 2, 3], [4, 5, 6]]", + ]) + .assert() + .success() + .stdout("matrix_multiply received arguments [[1, 2], [4, 5], [6, 7]] and [[1, 2, 3], [4, 5, 6]]\n[[9, 12, 15], [24, 33, 42], [34, 47, 60]]\n"); + + Ok(()) +} + #[test] fn sandbox_example() -> anyhow::Result<()> { let dir = tempfile::tempdir()?; From ad42e335d2f4b5dfd0cc47e80ec3244420fcdf70 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Wed, 6 Nov 2024 14:26:33 +0100 Subject: [PATCH 05/21] Migrate http example test and test output --- .github/workflows/test_examples.sh | 6 --- Cargo.lock | 19 ++++++++++ Cargo.toml | 1 + tests/integration.rs | 61 ++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_examples.sh b/.github/workflows/test_examples.sh index 83fd46d..d9372e7 100755 --- a/.github/workflows/test_examples.sh +++ b/.github/workflows/test_examples.sh @@ -8,12 +8,6 @@ export WASMTIME_BACKTRACE_DETAILS=1 cargo build --release - -# HTTP -# Just compiling for now -(cd examples/http \ - && ../../target/release/componentize-py -d ../../wit -w wasi:http/proxy@0.2.0 componentize app -o http.wasm) - # TCP # Just compiling for now (cd examples/tcp \ diff --git a/Cargo.lock b/Cargo.lock index 5b7224f..2b2f30d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -436,6 +436,7 @@ dependencies = [ "im-rc", "indexmap", "once_cell", + "predicates", "pretty_env_logger", "proptest", "pyo3", @@ -826,6 +827,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1337,6 +1347,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "num-bigint" version = "0.4.6" @@ -1449,7 +1465,10 @@ checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97" dependencies = [ "anstyle", "difflib", + "float-cmp", + "normalize-line-endings", "predicates-core", + "regex", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 29f8884..a0ed2f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ semver = "1.0.23" assert_cmd = "2.0.16" fs_extra = "1.3.0" hex = "0.4.3" +predicates = "3.1.2" proptest = "1.5.0" tempfile = "3.13.0" diff --git a/tests/integration.rs b/tests/integration.rs index 282a51c..e32d5c7 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,5 +1,6 @@ use assert_cmd::Command; use fs_extra::dir::CopyOptions; +use predicates::prelude::predicate; #[test] fn cli_example() -> anyhow::Result<()> { @@ -37,6 +38,66 @@ fn cli_example() -> anyhow::Result<()> { Ok(()) } +#[test] +fn http_example() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/http", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; + let path = dir.path().join("http"); + + Command::cargo_bin("componentize-py")? + .current_dir(&path) + .args([ + "-d", + "../wit", + "-w", + "wasi:http/proxy@0.2.0", + "componentize", + "app", + "-o", + "http.wasm", + ]) + .assert() + .success() + .stdout("Component built successfully\n"); + + let mut handle = std::process::Command::new("wasmtime") + .current_dir(&path) + .args(["serve", "--wasi", "common", "http.wasm"]) + .spawn()?; + + let content = "’Twas brillig, and the slithy toves + Did gyre and gimble in the wabe: +All mimsy were the borogoves, + And the mome raths outgrabe. +"; + + Command::new("curl") + .current_dir(&path) + .args([ + "-i", + "-H", + "content-type: text/plain", + "--retry-connrefused", + "--retry", + "5", + "--data-binary", + "@-", + "http://127.0.0.1:8080/echo", + ]) + .write_stdin(content) + .assert() + .success() + .stdout(predicate::str::ends_with(content)); + + handle.kill()?; + + Ok(()) +} + #[test] fn matrix_math_example() -> anyhow::Result<()> { let dir = tempfile::tempdir()?; From e6bc0ecf45b7bef1b39a323f1e808d4887f39f11 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Wed, 6 Nov 2024 15:14:17 +0100 Subject: [PATCH 06/21] Check hash-all endpoint, and fix poll loop calls --- bundled/poll_loop.py | 222 ++++++++++++++++++++++++++++--------------- tests/integration.rs | 23 +++++ 2 files changed, 170 insertions(+), 75 deletions(-) diff --git a/bundled/poll_loop.py b/bundled/poll_loop.py index 4a08f4a..d6d8911 100644 --- a/bundled/poll_loop.py +++ b/bundled/poll_loop.py @@ -13,7 +13,12 @@ from proxy.types import Ok, Err from proxy.imports import types, streams, poll, outgoing_handler -from proxy.imports.types import IncomingBody, OutgoingBody, OutgoingRequest, IncomingResponse +from proxy.imports.types import ( + IncomingBody, + OutgoingBody, + OutgoingRequest, + IncomingResponse, +) from proxy.imports.streams import StreamError_Closed, InputStream from proxy.imports.poll import Pollable from typing import Optional, cast @@ -21,9 +26,10 @@ # Maximum number of bytes to read at a time READ_SIZE: int = 16 * 1024 + async def send(request: OutgoingRequest) -> IncomingResponse: """Send the specified request and wait asynchronously for the response.""" - + future = outgoing_handler.handle(request, None) while True: @@ -39,8 +45,10 @@ async def send(request: OutgoingRequest) -> IncomingResponse: else: raise response + class Stream: """Reader abstraction over `wasi:http/types#incoming-body`.""" + def __init__(self, body: IncomingBody): self.body: Optional[IncomingBody] = body self.stream: Optional[InputStream] = body.stream() @@ -57,13 +65,16 @@ async def next(self) -> Optional[bytes]: else: buffer = self.stream.read(READ_SIZE) if len(buffer) == 0: - await register(cast(PollLoop, asyncio.get_event_loop()), self.stream.subscribe()) + await register( + cast(PollLoop, asyncio.get_event_loop()), + self.stream.subscribe(), + ) else: return buffer except Err as e: if isinstance(e.value, StreamError_Closed): if self.stream is not None: - self.stream.__exit__() + self.stream.__exit__(None, None, None) self.stream = None if self.body is not None: IncomingBody.finish(self.body) @@ -71,8 +82,10 @@ async def next(self) -> Optional[bytes]: else: raise e + class Sink: """Writer abstraction over `wasi:http/types#outgoing-body`.""" + def __init__(self, body: OutgoingBody): self.body = body self.stream = body.write() @@ -87,7 +100,9 @@ async def send(self, chunk: bytes): while True: count = self.stream.check_write() if count == 0: - await register(cast(PollLoop, asyncio.get_event_loop()), self.stream.subscribe()) + await register( + cast(PollLoop, asyncio.get_event_loop()), self.stream.subscribe() + ) elif offset == len(chunk): if flushing: return @@ -96,20 +111,21 @@ async def send(self, chunk: bytes): flushing = True else: count = min(count, len(chunk) - offset) - self.stream.write(chunk[offset:offset+count]) + self.stream.write(chunk[offset : offset + count]) offset += count def close(self): """Close the stream, indicating no further data will be written.""" - self.stream.__exit__() + self.stream.__exit__(None, None, None) self.stream = None OutgoingBody.finish(self.body, None) self.body = None - + + class PollLoop(asyncio.AbstractEventLoop): """Custom `asyncio` event loop backed by `wasi:io/poll#poll`.""" - + def __init__(self): self.wakers = [] self.running = False @@ -130,18 +146,18 @@ def run_until_complete(self, future): for handle in handles: if not handle._cancelled: handle._run() - + if self.wakers: [pollables, wakers] = list(map(list, zip(*self.wakers))) - + new_wakers = [] ready = [False] * len(pollables) for index in poll.poll(pollables): ready[index] = True - + for (ready, pollable), waker in zip(zip(ready, pollables), wakers): if ready: - pollable.__exit__() + pollable.__exit__(None, None, None) waker.set_result(None) else: new_wakers.append((pollable, waker)) @@ -150,7 +166,7 @@ def run_until_complete(self, future): if self.exception is not None: raise self.exception - + return future.result() def is_running(self): @@ -169,7 +185,7 @@ def shutdown_asyncgens(self): pass def call_exception_handler(self, context): - self.exception = context.get('exception', None) + self.exception = context.get("exception", None) def call_soon(self, callback, *args, context=None): handle = asyncio.Handle(callback, args, self, context) @@ -211,72 +227,119 @@ def run_in_executor(self, executor, func, *args): def set_default_executor(self, executor): raise NotImplementedError - async def getaddrinfo(self, host, port, *, - family=0, type=0, proto=0, flags=0): + async def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0): raise NotImplementedError async def getnameinfo(self, sockaddr, flags=0): raise NotImplementedError async def create_connection( - self, protocol_factory, host=None, port=None, - *, ssl=None, family=0, proto=0, - flags=0, sock=None, local_addr=None, - server_hostname=None, - ssl_handshake_timeout=None, - ssl_shutdown_timeout=None, - happy_eyeballs_delay=None, interleave=None): + self, + protocol_factory, + host=None, + port=None, + *, + ssl=None, + family=0, + proto=0, + flags=0, + sock=None, + local_addr=None, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + happy_eyeballs_delay=None, + interleave=None, + ): raise NotImplementedError async def create_server( - self, protocol_factory, host=None, port=None, - *, family=socket.AF_UNSPEC, - flags=socket.AI_PASSIVE, sock=None, backlog=100, - ssl=None, reuse_address=None, reuse_port=None, - ssl_handshake_timeout=None, - ssl_shutdown_timeout=None, - start_serving=True): - raise NotImplementedError - - async def sendfile(self, transport, file, offset=0, count=None, - *, fallback=True): - raise NotImplementedError - - async def start_tls(self, transport, protocol, sslcontext, *, - server_side=False, - server_hostname=None, - ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + self, + protocol_factory, + host=None, + port=None, + *, + family=socket.AF_UNSPEC, + flags=socket.AI_PASSIVE, + sock=None, + backlog=100, + ssl=None, + reuse_address=None, + reuse_port=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + start_serving=True, + ): + raise NotImplementedError + + async def sendfile(self, transport, file, offset=0, count=None, *, fallback=True): + raise NotImplementedError + + async def start_tls( + self, + transport, + protocol, + sslcontext, + *, + server_side=False, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + ): raise NotImplementedError async def create_unix_connection( - self, protocol_factory, path=None, *, - ssl=None, sock=None, - server_hostname=None, - ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + self, + protocol_factory, + path=None, + *, + ssl=None, + sock=None, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + ): raise NotImplementedError async def create_unix_server( - self, protocol_factory, path=None, *, - sock=None, backlog=100, ssl=None, - ssl_handshake_timeout=None, - ssl_shutdown_timeout=None, - start_serving=True): + self, + protocol_factory, + path=None, + *, + sock=None, + backlog=100, + ssl=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + start_serving=True, + ): raise NotImplementedError async def connect_accepted_socket( - self, protocol_factory, sock, - *, ssl=None, - ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): - raise NotImplementedError - - async def create_datagram_endpoint(self, protocol_factory, - local_addr=None, remote_addr=None, *, - family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, - allow_broadcast=None, sock=None): + self, + protocol_factory, + sock, + *, + ssl=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + ): + raise NotImplementedError + + async def create_datagram_endpoint( + self, + protocol_factory, + local_addr=None, + remote_addr=None, + *, + family=0, + proto=0, + flags=0, + reuse_address=None, + reuse_port=None, + allow_broadcast=None, + sock=None, + ): raise NotImplementedError async def connect_read_pipe(self, protocol_factory, pipe): @@ -285,18 +348,27 @@ async def connect_read_pipe(self, protocol_factory, pipe): async def connect_write_pipe(self, protocol_factory, pipe): raise NotImplementedError - async def subprocess_shell(self, protocol_factory, cmd, *, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - **kwargs): + async def subprocess_shell( + self, + protocol_factory, + cmd, + *, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs, + ): raise NotImplementedError - async def subprocess_exec(self, protocol_factory, *args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - **kwargs): + async def subprocess_exec( + self, + protocol_factory, + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs, + ): raise NotImplementedError def add_reader(self, fd, callback, *args): @@ -335,8 +407,7 @@ async def sock_connect(self, sock, address): async def sock_accept(self, sock): raise NotImplementedError - async def sock_sendfile(self, sock, file, offset=0, count=None, - *, fallback=None): + async def sock_sendfile(self, sock, file, offset=0, count=None, *, fallback=None): raise NotImplementedError def add_signal_handler(self, sig, callback, *args): @@ -363,6 +434,7 @@ def default_exception_handler(self, context): def set_debug(self, enabled): raise NotImplementedError + async def register(loop: PollLoop, pollable: Pollable): waker = loop.create_future() loop.wakers.append((pollable, waker)) diff --git a/tests/integration.rs b/tests/integration.rs index e32d5c7..791d992 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -93,6 +93,29 @@ All mimsy were the borogoves, .success() .stdout(predicate::str::ends_with(content)); + Command::new("curl") + .current_dir(&path) + .args([ + "-i", + "-H", + "url: https://webassembly.github.io/spec/core/", + "-H", + "url: https://www.w3.org/groups/wg/wasm/", + "-H", + "url: https://bytecodealliance.org/", + "--retry-connrefused", + "--retry", + "5", + "http://127.0.0.1:8080/hash-all", + ]) + .assert() + .success() + .stdout(predicate::str::ends_with(" +https://webassembly.github.io/spec/core/: d08394b6dfa544179f7e438c54b690ee1ac120572267175d986f128025b92792 +https://bytecodealliance.org/: c79c3ab89afdc9d8027626ca87457ccba7700aa963df8d3a5ce8c5daa92a830b +https://www.w3.org/groups/wg/wasm/: b9cd0d52238845d0e90120828ec66ac2de67ed008975696f31bd1b49af21200d +")); + handle.kill()?; Ok(()) From e40b7589b1d23b6e8cd9d51ad695d4cc76bc88b1 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Wed, 6 Nov 2024 15:50:38 +0100 Subject: [PATCH 07/21] Migrate TCP example --- .github/workflows/test.yaml | 5 +- .github/workflows/test_examples.sh | 14 ------ tests/{integration.rs => componentize.rs} | 61 +++++++++++++++++++++++ 3 files changed, 62 insertions(+), 18 deletions(-) delete mode 100755 .github/workflows/test_examples.sh rename tests/{integration.rs => componentize.rs} (80%) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1cdb4b0..7b07cc3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -89,10 +89,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install wasmtime mypy - - name: Test examples - shell: bash - run: bash .github/workflows/test_examples.sh + - run: pip install mypy - name: Lint examples shell: bash run: bash .github/workflows/lint_examples.sh diff --git a/.github/workflows/test_examples.sh b/.github/workflows/test_examples.sh deleted file mode 100755 index d9372e7..0000000 --- a/.github/workflows/test_examples.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -export COMPONENTIZE_PY_TEST_COUNT=0 -export COMPONENTIZE_PY_TEST_SEED=bc6ad1950594f1fe477144ef5b3669dd5962e49de4f3b666e5cbf9072507749a -export WASMTIME_BACKTRACE_DETAILS=1 - -cargo build --release - -# TCP -# Just compiling for now -(cd examples/tcp \ - && ../../target/release/componentize-py -d ../../wit -w wasi:cli/command@0.2.0 componentize app -o tcp.wasm) diff --git a/tests/integration.rs b/tests/componentize.rs similarity index 80% rename from tests/integration.rs rename to tests/componentize.rs index 791d992..eb44496 100644 --- a/tests/integration.rs +++ b/tests/componentize.rs @@ -1,3 +1,5 @@ +use std::{io::Write, process::Stdio}; + use assert_cmd::Command; use fs_extra::dir::CopyOptions; use predicates::prelude::predicate; @@ -231,3 +233,62 @@ fn sandbox_example() -> anyhow::Result<()> { Ok(()) } + +#[test] +fn tcp_example() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/tcp", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; + let path = dir.path().join("tcp"); + + Command::cargo_bin("componentize-py")? + .current_dir(&path) + .args([ + "-d", + "../wit", + "-w", + "wasi:cli/command@0.2.0", + "componentize", + "app", + "-o", + "tcp.wasm", + ]) + .assert() + .success() + .stdout("Component built successfully\n"); + + let mut nc_handle = std::process::Command::new("nc") + .current_dir(&path) + .args(["-l", "127.0.0.1", "3456"]) + .stdin(Stdio::piped()) + .spawn()?; + + let tcp_handle = std::process::Command::new("wasmtime") + .current_dir(&path) + .args([ + "run", + "--wasi", + "inherit-network", + "tcp.wasm", + "127.0.0.1:3456", + ]) + .stdout(Stdio::piped()) + .spawn()?; + + let mut nc_std_in = nc_handle.stdin.take().unwrap(); + + nc_std_in.write_all(b"hello")?; + + let output = tcp_handle.wait_with_output()?; + nc_handle.kill()?; + + assert_eq!( + String::from_utf8_lossy(&output.stdout), + "received: b'hello'\n" + ); + + Ok(()) +} From 3b91e84dc14c8cf1e6740d9a63e06a1cde4dc331 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 09:11:31 +0100 Subject: [PATCH 08/21] ci: move wasmtime cli installation earlier --- .github/workflows/test.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7b07cc3..85a45d2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -79,17 +79,20 @@ jobs: shell: bash run: bash .github/workflows/lint.sh - - name: Test - shell: bash - run: COMPONENTIZE_PY_TEST_COUNT=20 PROPTEST_MAX_SHRINK_ITERS=0 cargo test --release - - uses: taiki-e/install-action@v2 with: tool: wasmtime-cli + - uses: actions/setup-python@v5 with: python-version: "3.12" + - run: pip install mypy + + - name: Test + shell: bash + run: COMPONENTIZE_PY_TEST_COUNT=20 PROPTEST_MAX_SHRINK_ITERS=0 cargo test --release + - name: Lint examples shell: bash run: bash .github/workflows/lint_examples.sh From 041e2c780b460ba2f498ac8da47cf9e4ca02a40f Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 10:07:34 +0100 Subject: [PATCH 09/21] Looser match for hash return value --- tests/componentize.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/componentize.rs b/tests/componentize.rs index eb44496..007c97f 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -112,11 +112,13 @@ All mimsy were the borogoves, ]) .assert() .success() - .stdout(predicate::str::ends_with(" -https://webassembly.github.io/spec/core/: d08394b6dfa544179f7e438c54b690ee1ac120572267175d986f128025b92792 -https://bytecodealliance.org/: c79c3ab89afdc9d8027626ca87457ccba7700aa963df8d3a5ce8c5daa92a830b -https://www.w3.org/groups/wg/wasm/: b9cd0d52238845d0e90120828ec66ac2de67ed008975696f31bd1b49af21200d -")); + .stdout(predicate::str::is_match( + " +https://webassembly.github.io/spec/core/: .+ +https://bytecodealliance.org/: .+ +https://www.w3.org/groups/wg/wasm/: .+ +", + )?); handle.kill()?; From ad0af409dd01620abe07b4f4463d283db73e9641 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 10:28:06 +0100 Subject: [PATCH 10/21] Move cli lint to rust test --- .github/workflows/lint_examples.sh | 5 --- tests/bindings.rs | 53 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 tests/bindings.rs diff --git a/.github/workflows/lint_examples.sh b/.github/workflows/lint_examples.sh index 7b85c40..5df4a60 100755 --- a/.github/workflows/lint_examples.sh +++ b/.github/workflows/lint_examples.sh @@ -8,11 +8,6 @@ export WASMTIME_BACKTRACE_DETAILS=1 cargo build --release -# CLI -(cd examples/cli \ - && rm -rf command || true \ - && ../../target/release/componentize-py -d ../../wit -w wasi:cli/command@0.2.0 bindings . \ - && mypy --strict .) # HTTP # poll_loop.py has many errors that might not be worth adjusting at the moment, so ignore for now diff --git a/tests/bindings.rs b/tests/bindings.rs new file mode 100644 index 0000000..c5414bc --- /dev/null +++ b/tests/bindings.rs @@ -0,0 +1,53 @@ +use std::path::Path; + +use assert_cmd::{assert::Assert, Command}; +use fs_extra::dir::CopyOptions; +use predicates::{prelude::predicate, Predicate}; + +#[test] +fn lint_cli_bindings() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/cli", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; + let path = dir.path().join("cli"); + + generate_bindings(&path, "wasi:cli/command@0.2.0")?; + + assert!(predicate::path::is_dir().eval(&path.join("command"))); + + mypy_command(dir.path()) + .current_dir(&path) + .args(["--strict", "."]) + .assert() + .success() + .stdout("Success: no issues found in 33 source files\n"); + + Ok(()) +} + +fn generate_bindings(path: &Path, world: &str) -> Result { + Ok(Command::cargo_bin("componentize-py")? + .current_dir(path) + .args(["-d", "../wit", "-w", world, "bindings", "."]) + .assert() + .success()) +} + +fn mypy_command(temp_dir: &Path) -> Command { + Command::new("python3") + .current_dir(temp_dir) + .args(["-m", "venv", ".venv"]) + .assert() + .success(); + + Command::new("./.venv/bin/pip") + .current_dir(temp_dir) + .args(["install", "mypy"]) + .assert() + .success(); + + Command::new("../.venv/bin/mypy") +} From cc9c0d8f2cdb5bbf279605b94d5d2413914726c4 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 10:35:57 +0100 Subject: [PATCH 11/21] Migrate linting of http bindings --- .github/workflows/lint_examples.sh | 8 ----- tests/bindings.rs | 58 ++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/.github/workflows/lint_examples.sh b/.github/workflows/lint_examples.sh index 5df4a60..4b58f4d 100755 --- a/.github/workflows/lint_examples.sh +++ b/.github/workflows/lint_examples.sh @@ -8,14 +8,6 @@ export WASMTIME_BACKTRACE_DETAILS=1 cargo build --release - -# HTTP -# poll_loop.py has many errors that might not be worth adjusting at the moment, so ignore for now -(cd examples/http \ - && rm -rf proxy || true \ - && ../../target/release/componentize-py -d ../../wit -w wasi:http/proxy@0.2.0 bindings . \ - && mypy --strict --ignore-missing-imports -m app -p proxy) - # # Matrix Math (cd examples/matrix-math \ && rm -rf matrix_math || true \ diff --git a/tests/bindings.rs b/tests/bindings.rs index c5414bc..68e35d2 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{ffi::OsStr, path::Path}; use assert_cmd::{assert::Assert, Command}; use fs_extra::dir::CopyOptions; @@ -18,12 +18,37 @@ fn lint_cli_bindings() -> anyhow::Result<()> { assert!(predicate::path::is_dir().eval(&path.join("command"))); - mypy_command(dir.path()) - .current_dir(&path) - .args(["--strict", "."]) - .assert() - .success() - .stdout("Success: no issues found in 33 source files\n"); + mypy_check(&path, ["--strict", "."]); + + Ok(()) +} + +#[test] +fn lint_http_bindings() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/http", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; + let path = dir.path().join("http"); + + generate_bindings(&path, "wasi:http/proxy@0.2.0")?; + + assert!(predicate::path::is_dir().eval(&path.join("proxy"))); + + mypy_check( + &path, + [ + "--strict", + // poll_loop.py has many errors that might not be worth adjusting at the moment, so ignore for now + "--ignore-missing-imports", + "-m", + "app", + "-p", + "proxy", + ], + ); Ok(()) } @@ -36,18 +61,29 @@ fn generate_bindings(path: &Path, world: &str) -> Result .success()) } -fn mypy_command(temp_dir: &Path) -> Command { +fn mypy_check(path: &Path, args: I) -> Assert +where + I: IntoIterator, + S: AsRef, +{ Command::new("python3") - .current_dir(temp_dir) + .current_dir(path) .args(["-m", "venv", ".venv"]) .assert() .success(); Command::new("./.venv/bin/pip") - .current_dir(temp_dir) + .current_dir(path) .args(["install", "mypy"]) .assert() .success(); - Command::new("../.venv/bin/mypy") + Command::new("./.venv/bin/mypy") + .current_dir(path) + .args(args) + .assert() + .success() + .stdout( + predicate::str::is_match("^Success: no issues found in \\d+ source files\n$").unwrap(), + ) } From e8caeda27e38adbf5cc6d50d060df9a00cfae43f Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 10:41:19 +0100 Subject: [PATCH 12/21] Migrate tcp linting --- .github/workflows/lint_examples.sh | 6 ------ tests/bindings.rs | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint_examples.sh b/.github/workflows/lint_examples.sh index 4b58f4d..913e88f 100755 --- a/.github/workflows/lint_examples.sh +++ b/.github/workflows/lint_examples.sh @@ -21,9 +21,3 @@ cargo build --release && rm -rf sandbox || true \ && ../../target/release/componentize-py -d sandbox.wit bindings . \ && mypy --strict -m guest -p sandbox) - -# TCP -(cd examples/tcp \ - && rm -rf command || true \ - && ../../target/release/componentize-py -d ../../wit -w wasi:cli/command@0.2.0 bindings . \ - && mypy --strict .) diff --git a/tests/bindings.rs b/tests/bindings.rs index 68e35d2..a70b015 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -53,6 +53,25 @@ fn lint_http_bindings() -> anyhow::Result<()> { Ok(()) } +#[test] +fn lint_tcp_bindings() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/tcp", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; + let path = dir.path().join("tcp"); + + generate_bindings(&path, "wasi:cli/command@0.2.0")?; + + assert!(predicate::path::is_dir().eval(&path.join("command"))); + + mypy_check(&path, ["--strict", "."]); + + Ok(()) +} + fn generate_bindings(path: &Path, world: &str) -> Result { Ok(Command::cargo_bin("componentize-py")? .current_dir(path) From 1720b447beee83f4474decd1d4ac9596f127ef98 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 10:43:35 +0100 Subject: [PATCH 13/21] Move sandbox linting --- .github/workflows/lint_examples.sh | 6 ------ tests/bindings.rs | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint_examples.sh b/.github/workflows/lint_examples.sh index 913e88f..103c55c 100755 --- a/.github/workflows/lint_examples.sh +++ b/.github/workflows/lint_examples.sh @@ -15,9 +15,3 @@ cargo build --release && tar xf numpy-wasi.tar.gz \ && ../../target/release/componentize-py -d ../../wit -w matrix-math bindings . \ && mypy --strict --follow-imports silent -m app -p matrix_math) - -# Sandbox -(cd examples/sandbox \ - && rm -rf sandbox || true \ - && ../../target/release/componentize-py -d sandbox.wit bindings . \ - && mypy --strict -m guest -p sandbox) diff --git a/tests/bindings.rs b/tests/bindings.rs index a70b015..406ead9 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -53,6 +53,25 @@ fn lint_http_bindings() -> anyhow::Result<()> { Ok(()) } +#[test] +fn lint_sandbox_bindings() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items(&["./examples/sandbox"], dir.path(), &CopyOptions::new())?; + let path = dir.path().join("sandbox"); + + Command::cargo_bin("componentize-py")? + .current_dir(&path) + .args(["-d", "sandbox.wit", "bindings", "."]) + .assert() + .success(); + + assert!(predicate::path::is_dir().eval(&path.join("sandbox"))); + + mypy_check(&path, ["--strict", "-m", "guest", "-p", "sandbox"]); + + Ok(()) +} + #[test] fn lint_tcp_bindings() -> anyhow::Result<()> { let dir = tempfile::tempdir()?; From 6d59a8bb7bb806d5f26c77274997fbc8f587923b Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 10:48:24 +0100 Subject: [PATCH 14/21] Move matrix math linting --- .github/workflows/lint_examples.sh | 17 ---------- .github/workflows/test.yaml | 6 ---- tests/bindings.rs | 50 ++++++++++++++++++++++++++++++ tests/componentize.rs | 34 +++++++++++--------- 4 files changed, 69 insertions(+), 38 deletions(-) delete mode 100755 .github/workflows/lint_examples.sh diff --git a/.github/workflows/lint_examples.sh b/.github/workflows/lint_examples.sh deleted file mode 100755 index 103c55c..0000000 --- a/.github/workflows/lint_examples.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -export COMPONENTIZE_PY_TEST_COUNT=0 -export COMPONENTIZE_PY_TEST_SEED=bc6ad1950594f1fe477144ef5b3669dd5962e49de4f3b666e5cbf9072507749a -export WASMTIME_BACKTRACE_DETAILS=1 - -cargo build --release - -# # Matrix Math -(cd examples/matrix-math \ - && rm -rf matrix_math || true \ - && curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz \ - && tar xf numpy-wasi.tar.gz \ - && ../../target/release/componentize-py -d ../../wit -w matrix-math bindings . \ - && mypy --strict --follow-imports silent -m app -p matrix_math) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 85a45d2..8cba0b0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -87,12 +87,6 @@ jobs: with: python-version: "3.12" - - run: pip install mypy - - name: Test shell: bash run: COMPONENTIZE_PY_TEST_COUNT=20 PROPTEST_MAX_SHRINK_ITERS=0 cargo test --release - - - name: Lint examples - shell: bash - run: bash .github/workflows/lint_examples.sh diff --git a/tests/bindings.rs b/tests/bindings.rs index 406ead9..ca2f7cf 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -53,6 +53,39 @@ fn lint_http_bindings() -> anyhow::Result<()> { Ok(()) } +#[test] +fn lint_matrix_math_bindings() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/matrix-math", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; + let path = dir.path().join("matrix-math"); + + install_numpy(&path); + + generate_bindings(&path, "matrix-math")?; + + assert!(predicate::path::is_dir().eval(&path.join("matrix_math"))); + + mypy_check( + &path, + [ + "--strict", + // numpy doesn't pass + "--follow-imports", + "silent", + "-m", + "app", + "-p", + "matrix_math", + ], + ); + + Ok(()) +} + #[test] fn lint_sandbox_bindings() -> anyhow::Result<()> { let dir = tempfile::tempdir()?; @@ -125,3 +158,20 @@ where predicate::str::is_match("^Success: no issues found in \\d+ source files\n$").unwrap(), ) } + +fn install_numpy(path: &Path) { + Command::new("curl") + .current_dir(path) + .args([ + "-OL", + "https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz", + ]) + .assert() + .success(); + + Command::new("tar") + .current_dir(path) + .args(["xf", "numpy-wasi.tar.gz"]) + .assert() + .success(); +} diff --git a/tests/componentize.rs b/tests/componentize.rs index 007c97f..7f04208 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -1,4 +1,4 @@ -use std::{io::Write, process::Stdio}; +use std::{io::Write, path::Path, process::Stdio}; use assert_cmd::Command; use fs_extra::dir::CopyOptions; @@ -135,20 +135,7 @@ fn matrix_math_example() -> anyhow::Result<()> { )?; let path = dir.path().join("matrix-math"); - Command::new("curl") - .current_dir(&path) - .args([ - "-OL", - "https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz", - ]) - .assert() - .success(); - - Command::new("tar") - .current_dir(&path) - .args(["xf", "numpy-wasi.tar.gz"]) - .assert() - .success(); + install_numpy(&path); Command::cargo_bin("componentize-py")? .current_dir(&path) @@ -294,3 +281,20 @@ fn tcp_example() -> anyhow::Result<()> { Ok(()) } + +fn install_numpy(path: &Path) { + Command::new("curl") + .current_dir(path) + .args([ + "-OL", + "https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz", + ]) + .assert() + .success(); + + Command::new("tar") + .current_dir(path) + .args(["xf", "numpy-wasi.tar.gz"]) + .assert() + .success(); +} From 85382654b4aae2b887e5f619f9a10b667d359567 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 11:20:15 +0100 Subject: [PATCH 15/21] Fix venv invocation on windows --- tests/bindings.rs | 11 +++++++++-- tests/componentize.rs | 17 ++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/bindings.rs b/tests/bindings.rs index ca2f7cf..207f9d0 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -143,13 +143,13 @@ where .assert() .success(); - Command::new("./.venv/bin/pip") + Command::new(venv_executable_path("pip")) .current_dir(path) .args(["install", "mypy"]) .assert() .success(); - Command::new("./.venv/bin/mypy") + Command::new(venv_executable_path("mypy")) .current_dir(path) .args(args) .assert() @@ -159,6 +159,13 @@ where ) } +fn venv_executable_path(executable: &str) -> String { + format!( + "./.venv/bin/{executable}{}", + if cfg!(windows) { ".exe" } else { "" } + ) +} + fn install_numpy(path: &Path) { Command::new("curl") .current_dir(path) diff --git a/tests/componentize.rs b/tests/componentize.rs index 7f04208..8a139ef 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -190,18 +190,18 @@ fn sandbox_example() -> anyhow::Result<()> { .stdout("Component built successfully\n"); Command::new("python3") - .current_dir(dir.path()) + .current_dir(&path) .args(["-m", "venv", ".venv"]) .assert() .success(); - Command::new("./.venv/bin/pip") - .current_dir(dir.path()) + Command::new(venv_executable_path("pip")) + .current_dir(&path) .args(["install", "wasmtime"]) .assert() .success(); - Command::new("../.venv/bin/python") + Command::new(venv_executable_path("python")) .current_dir(&path) .args([ "-m", @@ -213,7 +213,7 @@ fn sandbox_example() -> anyhow::Result<()> { .assert() .success(); - Command::new("../.venv/bin/python") + Command::new(venv_executable_path("python")) .current_dir(&path) .args(["host.py", "2 + 2"]) .assert() @@ -298,3 +298,10 @@ fn install_numpy(path: &Path) { .assert() .success(); } + +fn venv_executable_path(executable: &str) -> String { + format!( + "./.venv/bin/{executable}{}", + if cfg!(windows) { ".exe" } else { "" } + ) +} From c7e3d8c03b582bc45396b5c095f123c382bce53c Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 11:32:57 +0100 Subject: [PATCH 16/21] Make http hash test more resilient --- tests/componentize.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/componentize.rs b/tests/componentize.rs index 8a139ef..6d5dd1d 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -2,7 +2,7 @@ use std::{io::Write, path::Path, process::Stdio}; use assert_cmd::Command; use fs_extra::dir::CopyOptions; -use predicates::prelude::predicate; +use predicates::prelude::{predicate, PredicateBooleanExt}; #[test] fn cli_example() -> anyhow::Result<()> { @@ -112,13 +112,13 @@ All mimsy were the borogoves, ]) .assert() .success() - .stdout(predicate::str::is_match( - " -https://webassembly.github.io/spec/core/: .+ -https://bytecodealliance.org/: .+ -https://www.w3.org/groups/wg/wasm/: .+ -", - )?); + .stdout( + predicate::str::contains("https://webassembly.github.io/spec/core/:").and( + predicate::str::contains("https://bytecodealliance.org/:").and( + predicate::str::contains("https://www.w3.org/groups/wg/wasm/:"), + ), + ), + ); handle.kill()?; From 4b07bf1b778689ba6ac1b5048914a2c73f408418 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 11:49:39 +0100 Subject: [PATCH 17/21] Try less relative paths for running venv executables --- tests/bindings.rs | 11 ++--------- tests/componentize.rs | 13 +++---------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/tests/bindings.rs b/tests/bindings.rs index 207f9d0..4522f55 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -143,13 +143,13 @@ where .assert() .success(); - Command::new(venv_executable_path("pip")) + Command::new(path.join(".venv/bin/pip")) .current_dir(path) .args(["install", "mypy"]) .assert() .success(); - Command::new(venv_executable_path("mypy")) + Command::new(path.join(".venv/bin/mypy")) .current_dir(path) .args(args) .assert() @@ -159,13 +159,6 @@ where ) } -fn venv_executable_path(executable: &str) -> String { - format!( - "./.venv/bin/{executable}{}", - if cfg!(windows) { ".exe" } else { "" } - ) -} - fn install_numpy(path: &Path) { Command::new("curl") .current_dir(path) diff --git a/tests/componentize.rs b/tests/componentize.rs index 6d5dd1d..fded0aa 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -195,13 +195,13 @@ fn sandbox_example() -> anyhow::Result<()> { .assert() .success(); - Command::new(venv_executable_path("pip")) + Command::new(path.join(".venv/bin/pip")) .current_dir(&path) .args(["install", "wasmtime"]) .assert() .success(); - Command::new(venv_executable_path("python")) + Command::new(path.join(".venv/bin/python")) .current_dir(&path) .args([ "-m", @@ -213,7 +213,7 @@ fn sandbox_example() -> anyhow::Result<()> { .assert() .success(); - Command::new(venv_executable_path("python")) + Command::new(path.join(".venv/bin/python")) .current_dir(&path) .args(["host.py", "2 + 2"]) .assert() @@ -298,10 +298,3 @@ fn install_numpy(path: &Path) { .assert() .success(); } - -fn venv_executable_path(executable: &str) -> String { - format!( - "./.venv/bin/{executable}{}", - if cfg!(windows) { ".exe" } else { "" } - ) -} From b2637f0c5506c8732e77896b7f958971f0793803 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 12:28:49 +0100 Subject: [PATCH 18/21] Manually join all path dirs --- tests/bindings.rs | 4 ++-- tests/componentize.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/bindings.rs b/tests/bindings.rs index 4522f55..6f4cd6b 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -143,13 +143,13 @@ where .assert() .success(); - Command::new(path.join(".venv/bin/pip")) + Command::new(path.join(".venv").join("bin").join("pip")) .current_dir(path) .args(["install", "mypy"]) .assert() .success(); - Command::new(path.join(".venv/bin/mypy")) + Command::new(path.join(".venv").join("bin").join("mypy")) .current_dir(path) .args(args) .assert() diff --git a/tests/componentize.rs b/tests/componentize.rs index fded0aa..7b56ccc 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -195,13 +195,13 @@ fn sandbox_example() -> anyhow::Result<()> { .assert() .success(); - Command::new(path.join(".venv/bin/pip")) + Command::new(path.join(".venv").join("bin").join("pip")) .current_dir(&path) .args(["install", "wasmtime"]) .assert() .success(); - Command::new(path.join(".venv/bin/python")) + Command::new(path.join(".venv").join("bin").join("python")) .current_dir(&path) .args([ "-m", @@ -213,7 +213,7 @@ fn sandbox_example() -> anyhow::Result<()> { .assert() .success(); - Command::new(path.join(".venv/bin/python")) + Command::new(path.join(".venv").join("bin").join("python")) .current_dir(&path) .args(["host.py", "2 + 2"]) .assert() From 32d053676c3abc92b070f26c21fb6780483e907b Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 12:53:48 +0100 Subject: [PATCH 19/21] Try conditionally looking which directory these live in --- tests/bindings.rs | 14 +++++++++++--- tests/componentize.rs | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/bindings.rs b/tests/bindings.rs index 6f4cd6b..c5948b2 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -1,4 +1,7 @@ -use std::{ffi::OsStr, path::Path}; +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, +}; use assert_cmd::{assert::Assert, Command}; use fs_extra::dir::CopyOptions; @@ -143,13 +146,13 @@ where .assert() .success(); - Command::new(path.join(".venv").join("bin").join("pip")) + Command::new(venv_path(path).join("pip")) .current_dir(path) .args(["install", "mypy"]) .assert() .success(); - Command::new(path.join(".venv").join("bin").join("mypy")) + Command::new(venv_path(path).join("mypy")) .current_dir(path) .args(args) .assert() @@ -159,6 +162,11 @@ where ) } +fn venv_path(path: &Path) -> PathBuf { + path.join(".venv") + .join(if cfg!(windows) { "Scripts" } else { "bin" }) +} + fn install_numpy(path: &Path) { Command::new("curl") .current_dir(path) diff --git a/tests/componentize.rs b/tests/componentize.rs index 7b56ccc..098c169 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -1,4 +1,8 @@ -use std::{io::Write, path::Path, process::Stdio}; +use std::{ + io::Write, + path::{Path, PathBuf}, + process::Stdio, +}; use assert_cmd::Command; use fs_extra::dir::CopyOptions; @@ -195,13 +199,13 @@ fn sandbox_example() -> anyhow::Result<()> { .assert() .success(); - Command::new(path.join(".venv").join("bin").join("pip")) + Command::new(venv_path(&path).join("pip")) .current_dir(&path) .args(["install", "wasmtime"]) .assert() .success(); - Command::new(path.join(".venv").join("bin").join("python")) + Command::new(venv_path(&path).join("python")) .current_dir(&path) .args([ "-m", @@ -213,7 +217,7 @@ fn sandbox_example() -> anyhow::Result<()> { .assert() .success(); - Command::new(path.join(".venv").join("bin").join("python")) + Command::new(venv_path(&path).join("python")) .current_dir(&path) .args(["host.py", "2 + 2"]) .assert() @@ -282,6 +286,11 @@ fn tcp_example() -> anyhow::Result<()> { Ok(()) } +fn venv_path(path: &Path) -> PathBuf { + path.join(".venv") + .join(if cfg!(windows) { "Scripts" } else { "bin" }) +} + fn install_numpy(path: &Path) { Command::new("curl") .current_dir(path) From 2e1731d47f6d8250c9a90bd373afe70eaad944f2 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 13:13:09 +0100 Subject: [PATCH 20/21] Handle different line endings in output --- tests/bindings.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/bindings.rs b/tests/bindings.rs index c5948b2..01f591f 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -157,9 +157,7 @@ where .args(args) .assert() .success() - .stdout( - predicate::str::is_match("^Success: no issues found in \\d+ source files\n$").unwrap(), - ) + .stdout(predicate::str::is_match("Success: no issues found in \\d+ source files").unwrap()) } fn venv_path(path: &Path) -> PathBuf { From 11f28618605a6ac33a19b767b616946f665c6fe6 Mon Sep 17 00:00:00 2001 From: Ben Brandt Date: Thu, 7 Nov 2024 13:34:24 +0100 Subject: [PATCH 21/21] Fix line ending issue and only run tcp example on unix systems --- tests/componentize.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/componentize.rs b/tests/componentize.rs index 098c169..8f854c9 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -222,11 +222,12 @@ fn sandbox_example() -> anyhow::Result<()> { .args(["host.py", "2 + 2"]) .assert() .success() - .stdout("result: 4\n"); + .stdout(predicate::str::contains("result: 4")); Ok(()) } +#[cfg(unix)] #[test] fn tcp_example() -> anyhow::Result<()> { let dir = tempfile::tempdir()?;