From d5fb0967db9c7039a448badb201d5beabf9db7ed Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Mon, 10 Nov 2025 10:18:22 +0100 Subject: [PATCH 1/2] Add spacetime_worker_sender_errors_total metric Currently we have a metric for reducer panics called `spacetime_worker_wasm_instance_errors_total`. This commit adds a metric for tracking errors returned from the module, like for example an Err result in Rust, or throwing a SenderError in TypeScript --- .../src/host/wasm_common/module_host_actor.rs | 30 +++++++++++++++++-- crates/core/src/worker_metrics/mod.rs | 5 ++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 30d5819a303..e36face3560 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -4,6 +4,7 @@ use spacetimedb_datastore::locking_tx_datastore::FuncCallType; use spacetimedb_datastore::locking_tx_datastore::ViewCallInfo; use spacetimedb_lib::db::raw_def::v9::Lifecycle; use spacetimedb_lib::de::DeserializeSeed as _; +use spacetimedb_lib::Hash; use spacetimedb_primitives::ProcedureId; use spacetimedb_primitives::TableId; use spacetimedb_primitives::ViewFnPtr; @@ -694,7 +695,13 @@ impl InstanceCommon { ) } Err(ExecutionError::User(err)) => { - log_reducer_error(inst.replica_ctx(), timestamp, reducer_name, &err); + log_reducer_error( + inst.replica_ctx(), + timestamp, + reducer_name, + &err, + &self.info.module_hash, + ); EventStatus::Failed(err.into()) } // We haven't actually committed yet - `commit_and_broadcast_event` will commit @@ -712,7 +719,13 @@ impl InstanceCommon { Ok(()) => EventStatus::Committed(DatabaseUpdate::default()), Err(err) => { let err = err.to_string(); - log_reducer_error(inst.replica_ctx(), timestamp, reducer_name, &err); + log_reducer_error( + inst.replica_ctx(), + timestamp, + reducer_name, + &err, + &self.info.module_hash, + ); EventStatus::Failed(err) } } @@ -1118,9 +1131,20 @@ fn maybe_log_long_running_function(reducer_name: &str, total_duration: Duration) } /// Logs an error `message` for `reducer` at `timestamp` into `replica_ctx`. -fn log_reducer_error(replica_ctx: &ReplicaContext, timestamp: Timestamp, reducer: &str, message: &str) { +fn log_reducer_error( + replica_ctx: &ReplicaContext, + timestamp: Timestamp, + reducer: &str, + message: &str, + module_hash: &Hash, +) { use database_logger::Record; + WORKER_METRICS + .sender_errors + .with_label_values(&replica_ctx.database_identity, module_hash, reducer) + .inc(); + log::info!("reducer returned error: {message}"); let record = Record { diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index cf1ec661a23..306522931b4 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -246,6 +246,11 @@ metrics_group!( #[labels(caller_identity: Identity, module_hash: Hash, caller_connection_id: ConnectionId, reducer_symbol: str)] pub wasm_instance_errors: IntCounterVec, + #[name = spacetime_worker_sender_errors_total] + #[help = "The number of sender errors returned from reducers."] + #[labels(database_identity: Identity, module_hash: Hash, reducer_symbol: str)] + pub sender_errors: IntCounterVec, + #[name = spacetime_worker_wasm_memory_bytes] #[help = "The number of bytes of linear memory allocated by the database's WASM module instance"] #[labels(database_identity: Identity)] From ddae26686eeae0fd307e971af581ba48adeaf34c Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Mon, 24 Nov 2025 13:11:56 +0100 Subject: [PATCH 2/2] Fix compilation --- crates/core/src/host/wasm_common/module_host_actor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index ef97ea6e8ae..1c29647d24d 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -37,7 +37,7 @@ use spacetimedb_lib::buffer::DecodeError; use spacetimedb_lib::db::raw_def::v9::Lifecycle; use spacetimedb_lib::de::DeserializeSeed; use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::{bsatn, ConnectionId, RawModuleDef, Timestamp}; +use spacetimedb_lib::{bsatn, ConnectionId, Hash, RawModuleDef, Timestamp}; use spacetimedb_primitives::{ProcedureId, TableId, ViewFnPtr, ViewId}; use spacetimedb_schema::auto_migrate::{MigratePlan, MigrationPolicy, MigrationPolicyError}; use spacetimedb_schema::def::{ModuleDef, ViewDef};