Add spacetime_worker_reducer_returned_errors_total metric (#3613)

# Description of Changes

Currently, we have a metric for reducer panics called
`spacetime_worker_wasm_instance_errors_total`. This commit adds a metric
for tracking errors returned from the module, like for example an Err
result in Rust, or throwing a SenderError in TypeScript

This will be needed for the web database overview dashboard, but it
should be useful for tracking sender errors in general.

# Expected complexity level and risk

1

# Testing

- [x] I've tested the change locally verifying the counter increments
when an instance returns an error

---------

Co-authored-by: Tyler Cloutier <cloutiertyler@users.noreply.github.com>

Author: drogus

crates/core/src/host/wasm_common/module_host_actor.rs

@@ -37,7 +37,7 @@ use spacetimedb_lib::buffer::DecodeError;
 use spacetimedb_lib::db::raw_def::v9::Lifecycle;
 use spacetimedb_lib::de::DeserializeSeed;
 use spacetimedb_lib::identity::AuthCtx;
-use spacetimedb_lib::{bsatn, ConnectionId, RawModuleDef, Timestamp};
+use spacetimedb_lib::{bsatn, ConnectionId, Hash, RawModuleDef, Timestamp};
 use spacetimedb_primitives::{ProcedureId, TableId, ViewFnPtr, ViewId};
 use spacetimedb_schema::auto_migrate::{MigratePlan, MigrationPolicy, MigrationPolicyError};
 use spacetimedb_schema::def::{ModuleDef, ViewDef};
@@ -745,7 +745,13 @@ impl InstanceCommon {
                 self.handle_outer_error(&result.stats.energy, reducer_name)
             }
             Err(ExecutionError::User(err)) => {
-                log_reducer_error(inst.replica_ctx(), timestamp, reducer_name, &err);
+                log_reducer_error(
+                    inst.replica_ctx(),
+                    timestamp,
+                    reducer_name,
+                    &err,
+                    &self.info.module_hash,
+                );
                 EventStatus::Failed(err.into())
             }
             // We haven't actually committed yet - `commit_and_broadcast_event` will commit
@@ -763,7 +769,13 @@ impl InstanceCommon {
                     Ok(()) => EventStatus::Committed(DatabaseUpdate::default()),
                     Err(err) => {
                         let err = err.to_string();
-                        log_reducer_error(inst.replica_ctx(), timestamp, reducer_name, &err);
+                        log_reducer_error(
+                            inst.replica_ctx(),
+                            timestamp,
+                            reducer_name,
+                            &err,
+                            &self.info.module_hash,
+                        );
                         EventStatus::Failed(err)
                     }
                 }
@@ -1161,9 +1173,20 @@ fn maybe_log_long_running_function(reducer_name: &str, total_duration: Duration)
 }
 
 /// Logs an error `message` for `reducer` at `timestamp` into `replica_ctx`.
-fn log_reducer_error(replica_ctx: &ReplicaContext, timestamp: Timestamp, reducer: &str, message: &str) {
+fn log_reducer_error(
+    replica_ctx: &ReplicaContext,
+    timestamp: Timestamp,
+    reducer: &str,
+    message: &str,
+    module_hash: &Hash,
+) {
     use database_logger::Record;
 
+    WORKER_METRICS
+        .sender_errors
+        .with_label_values(&replica_ctx.database_identity, module_hash, reducer)
+        .inc();
+
     log::info!("reducer returned error: {message}");
 
     let record = Record {

crates/core/src/worker_metrics/mod.rs

@@ -246,6 +246,11 @@ metrics_group!(
         #[labels(database_identity: Identity, module_hash: Hash, reducer_symbol: str)]
         pub wasm_instance_errors: IntCounterVec,
 
+        #[name = spacetime_worker_sender_errors_total]
+        #[help = "The number of sender errors returned from reducers."]
+        #[labels(database_identity: Identity, module_hash: Hash, reducer_symbol: str)]
+        pub sender_errors: IntCounterVec,
+
         #[name = spacetime_worker_wasm_memory_bytes]
         #[help = "The number of bytes of linear memory allocated by the database's WASM module instance"]
         #[labels(database_identity: Identity)]