refactor(bench): stabilize benchmark throughput tool

Author: lalvarezt

.github/workflows/bench-command.yml

@@ -336,15 +336,15 @@ jobs:
           echo "Checking out $BASELINE_REF..."
           git checkout "$BASELINE_REF"
 
-          # Check if bench_throughput binary is defined in Cargo.toml
-          if ! grep -q 'name = "bench_throughput"' Cargo.toml 2>/dev/null; then
+          # Check if bench-throughput binary is defined in Cargo.toml
+          if ! grep -q 'name = "bench-throughput"' Cargo.toml 2>/dev/null; then
             echo "exists=false" >> $GITHUB_OUTPUT
-            echo "❌ Benchmark tool 'bench_throughput' not found in $BASELINE_REF"
+            echo "❌ Benchmark tool 'bench-throughput' not found in $BASELINE_REF"
             exit 0
           fi
 
           # Check if the source file exists
-          if ! grep -A 2 'name = "bench_throughput"' Cargo.toml | grep -q 'path.*='; then
+          if ! grep -A 2 'name = "bench-throughput"' Cargo.toml | grep -q 'path.*='; then
             echo "exists=false" >> $GITHUB_OUTPUT
             echo "❌ Benchmark tool source file not found in $BASELINE_REF"
             exit 0
@@ -362,15 +362,15 @@ jobs:
           echo "Checking out $CURRENT_REF..."
           git checkout "$CURRENT_REF"
 
-          # Check if bench_throughput binary is defined in Cargo.toml
-          if ! grep -q 'name = "bench_throughput"' Cargo.toml 2>/dev/null; then
+          # Check if bench-throughput binary is defined in Cargo.toml
+          if ! grep -q 'name = "bench-throughput"' Cargo.toml 2>/dev/null; then
             echo "exists=false" >> $GITHUB_OUTPUT
-            echo "❌ Benchmark tool 'bench_throughput' not found in $CURRENT_REF"
+            echo "❌ Benchmark tool 'bench-throughput' not found in $CURRENT_REF"
             exit 0
           fi
 
           # Check if the source file exists
-          if ! grep -A 2 'name = "bench_throughput"' Cargo.toml | grep -q 'path.*='; then
+          if ! grep -A 2 'name = "bench-throughput"' Cargo.toml | grep -q 'path.*='; then
             echo "exists=false" >> $GITHUB_OUTPUT
             echo "❌ Benchmark tool source file not found in $CURRENT_REF"
             exit 0
@@ -390,7 +390,7 @@ jobs:
             const current_exists = '${{ steps.check_current_tool.outputs.exists }}' === 'true';
 
             let message = '❌ **Benchmark comparison failed**\n\n';
-            message += '**Reason**: The benchmark tool (`bench_throughput`) does not exist in ';
+            message += '**Reason**: The benchmark tool (`bench-throughput`) does not exist in ';
 
             if (!baseline_exists && !current_exists) {
               message += `both refs:\n- \`${baseline_ref}\` (baseline/older)\n- \`${current_ref}\` (current/newer)`;
@@ -429,16 +429,15 @@ jobs:
           git checkout "$BASELINE_REF"
 
           echo "Building benchmark tool..."
-          if ! cargo build --release --bin bench_throughput 2>&1 | tee build_baseline.log; then
+          if ! cargo build --release --bin bench-throughput 2>&1 | tee build_baseline.log; then
             echo "❌ Failed to build benchmark tool for $BASELINE_REF"
             exit 1
           fi
 
           echo "Running benchmarks on baseline..."
-          ./target/release/bench_throughput \
+          ./target/release/bench-throughput \
             --sizes "$SIZES" \
             --iterations "$ITERATIONS" \
-            --format json \
             --output benchmark_baseline.json
 
       - name: Benchmark current (newer commit)
@@ -454,16 +453,15 @@ jobs:
 
           # Rebuild in case dependencies changed
           echo "Building benchmark tool..."
-          if ! cargo build --release --bin bench_throughput 2>&1 | tee build_current.log; then
+          if ! cargo build --release --bin bench-throughput 2>&1 | tee build_current.log; then
             echo "❌ Failed to build benchmark tool for $CURRENT_REF"
             exit 1
           fi
 
           echo "Running benchmarks on current..."
-          ./target/release/bench_throughput \
+          ./target/release/bench-throughput \
             --sizes "$SIZES" \
             --iterations "$ITERATIONS" \
-            --format json \
             --output benchmark_current.json
 
       - name: Compare results

Cargo.toml

@@ -45,8 +45,8 @@ path = "src/bin/bench.rs"
 
 [[bin]]
 bench = false
-name = "bench_throughput"
-path = "src/bin/bench_throughput.rs"
+name = "bench-throughput"
+path = "src/bin/bench-throughput.rs"
 
 [profile.staging]
 inherits = "dev"

docs/benchmarking.md

@@ -6,37 +6,38 @@ String Pipeline includes a throughput-based benchmarking tool for measuring perf
 
 ```bash
 # Run with default settings
-cargo run --release --bin bench_throughput
+cargo run --release --bin bench-throughput
 
 # Specify input sizes and iterations
-cargo run --release --bin bench_throughput -- --sizes 1000,5000,10000 --iterations 100
+cargo run --release --bin bench-throughput -- --sizes 1000,5000,10000 --iterations 100
 
-# Generate JSON output for comparison
-cargo run --release --bin bench_throughput -- --format json --output results.json
+# Generate JSON output to custom location
+cargo run --release --bin bench-throughput -- --output results.json
 
 # Verbose mode shows per-template details
-cargo run --release --bin bench_throughput -- --verbose
+cargo run --release --bin bench-throughput -- --verbose
 ```
 
 ## Command Line Options
 
 | Option | Short | Default | Description |
 |--------|-------|---------|-------------|
-| `--sizes` | `-s` | `100,500,1000,5000,10000,50000,100000` | Comma-separated input sizes |
-| `--iterations` | `-i` | `50` | Number of iterations per size |
-| `--format` | `-f` | `console` | Output format: `console` or `json` |
-| `--output` | `-o` | - | Output file path (for JSON format) |
+| `--sizes` | `-s` | `10000` | Comma-separated input sizes |
+| `--iterations` | `-i` | `1` | Number of iterations per size |
+| `--output` | `-o` | `$XDG_DATA_HOME/string-pipeline/benchmarks/bench-<timestamp>.json` | Override default JSON output location |
 | `--verbose` | `-v` | false | Show detailed per-template results |
 
 ## Methodology
 
 The benchmark tool measures batch processing performance:
 
 1. **Parse Phase**: Template is parsed once and timed across multiple iterations
-2. **Warmup**: Each input size runs once without timing to stabilize caches
+2. **Warmup**: Each input size runs once without timing to stabilize caches (skipped when iterations = 1)
 3. **Measurement**: Multiple iterations are timed to calculate statistics
 4. **Analysis**: Results include average, percentiles (p50, p95, p99), and standard deviation
 
+The tool always outputs both a human-readable console report and JSON data for tracking over time.
+
 ### Test Data
 
 The benchmark generates realistic absolute file paths with varying depths (2-10 levels) using common directory names,
@@ -69,12 +70,12 @@ Use the included Python script to compare two benchmark runs:
 
 ```bash
 # Run baseline benchmark
-cargo run --release --bin bench_throughput -- --format json --output baseline.json
+cargo run --release --bin bench-throughput -- --output baseline.json
 
 # Make changes to the code
 
 # Run current benchmark
-cargo run --release --bin bench_throughput -- --format json --output current.json
+cargo run --release --bin bench-throughput -- --output current.json
 
 # Compare results
 python3 scripts/compare_benchmarks.py baseline.json current.json