refactor!: restructure metrics response for clear abstraction separation

sylvesterdamgaard · sylvesterdamgaard · commit 869e5a4b16b3 · 2025-11-20T18:54:44.000+01:00
BREAKING CHANGE: Major restructuring of metrics API responses to separate
abstraction layers, time windows, and current vs lifetime metrics.

## Server Metrics Restructure

Separated server metrics into 4 clear tiers:

1. Application tier: queue_workers (count, current/lifetime utilization)
2. Application tier: job_processing (lifetime totals, current performance)
3. Process tier: worker_processes (per-worker resource averages)
4. System tier: server_resources (actual server CPU/memory)

Breaking changes:
- Renamed system_limits → server_resources for clarity
- Split flat utilization_rate into current_busy_percent and lifetime_busy_percent
- Restructured workers/performance/utilization into nested hierarchy

## Queue Metrics Separation

Separated queue metrics by time scope:

- depth: Instantaneous queue state (current snapshot)
- performance_60s: Windowed metrics with explicit window_seconds
- lifetime: Lifetime metrics (failure_rate_percent since first job)
- workers: Current vs lifetime busy percentages

Breaking changes:
- Flat depth/pending/etc moved into depth object
- throughput_per_minute moved to performance_60s.throughput_per_minute
- utilization_rate split into workers.current_busy_percent and lifetime_busy_percent

## Trend Analysis Enhancement

Added comprehensive time_window context to all trend methods:

- window_seconds: Duration of analysis window
- window_start/window_end: ISO8601 timestamps
- analyzed_at: When analysis was performed
- sample_count: Number of data points analyzed

Breaking changes:
- Added time_window object wrapper to all trend responses
- Moved period_seconds into time_window structure

## Critical Bug Fix

Fixed jobs/minute calculation error (5x multiplication):
- Changed from cumulative worker uptime to actual elapsed time
- Used oldest worker uptime as proxy for wall-clock elapsed time
- Example: 5 workers × 10min = 50min cumulative → now 10min elapsed

## Dashboard Filtering Updates

Updated all dashboard filter methods to:
- Map new hierarchical structures correctly
- Maintain separation of current vs lifetime metrics
- Preserve time window context

Files modified:
- src/Services/WorkerMetricsQueryService.php
- src/Services/QueueMetricsQueryService.php
- src/Services/OverviewQueryService.php
- src/Services/TrendAnalysisService.php
diff --git a/src/Services/OverviewQueryService.php b/src/Services/OverviewQueryService.php
@@ -134,15 +134,21 @@ public function getOverview(bool $slim = true): array
     private function filterQueuesForDashboard(array $queues): array
     {
         return array_map(function ($queue) {
+            $depth = is_array($queue['depth'] ?? null) ? $queue['depth'] : [];
+            $performance60s = is_array($queue['performance_60s'] ?? null) ? $queue['performance_60s'] : [];
+            $lifetime = is_array($queue['lifetime'] ?? null) ? $queue['lifetime'] : [];
+            $workers = is_array($queue['workers'] ?? null) ? $queue['workers'] : [];
+
             return [
                 'connection' => $queue['connection'] ?? '',
                 'queue' => $queue['queue'] ?? '',
-                'depth' => $queue['depth'] ?? 0,
-                'pending' => $queue['pending'] ?? 0,
-                'active_workers' => $queue['active_workers'] ?? 0,
-                'throughput_per_minute' => $queue['throughput_per_minute'] ?? 0,
-                'failure_rate' => $queue['failure_rate'] ?? 0,
-                'utilization_rate' => $queue['utilization_rate'] ?? 0,
+                'depth' => $depth['total'] ?? 0,
+                'pending' => $depth['pending'] ?? 0,
+                'active_workers' => $workers['active_count'] ?? 0,
+                'throughput_per_minute' => $performance60s['throughput_per_minute'] ?? 0,
+                'failure_rate' => $lifetime['failure_rate_percent'] ?? 0,
+                'current_busy_percent' => $workers['current_busy_percent'] ?? 0,
+                'lifetime_busy_percent' => $workers['lifetime_busy_percent'] ?? 0,
             ];
         }, $queues);
     }
@@ -177,41 +183,48 @@ private function filterJobsForDashboard(array $jobs): array
      * Filter server metrics to essential dashboard fields only.
      *
      * Returns simplified server data with clear separation between:
-     * - Worker metrics: Job counts, worker utilization (from queue workers)
-     * - System limits: CPU cores, total memory (physical server resources)
+     * - Worker metrics: Worker count, utilization (from queue workers)
+     * - Job metrics: Jobs processed (from queue workers)
+     * - System metrics: Actual server CPU/memory from SystemMetrics (physical server resources)
+     *
+     * Note: Worker process CPU/memory metrics are NOT included in dashboard as they're not
+     * useful for server-level overview. Use server_resources for actual server resource usage.
      *
      * @param  array<string, array<string, mixed>>  $servers
      * @return array<string, array<string, mixed>>
      */
     private function filterServersForDashboard(array $servers): array
     {
         return array_map(function ($server) {
-            $workers = is_array($server['workers'] ?? null) ? $server['workers'] : [];
-            $utilization = is_array($server['utilization'] ?? null) ? $server['utilization'] : [];
-            $performance = is_array($server['performance'] ?? null) ? $server['performance'] : [];
-            $systemLimits = is_array($server['system_limits'] ?? null) ? $server['system_limits'] : null;
-
-            $serverUtilization = $utilization['server_utilization'] ?? 0;
-            $utilizationPercent = is_numeric($serverUtilization) ? round((float) $serverUtilization * 100, 2) : 0;
+            $queueWorkers = is_array($server['queue_workers'] ?? null) ? $server['queue_workers'] : [];
+            $workerCount = is_array($queueWorkers['count'] ?? null) ? $queueWorkers['count'] : [];
+            $workerUtilization = is_array($queueWorkers['utilization'] ?? null) ? $queueWorkers['utilization'] : [];
+            $jobProcessing = is_array($server['job_processing'] ?? null) ? $server['job_processing'] : [];
+            $jobLifetime = is_array($jobProcessing['lifetime'] ?? null) ? $jobProcessing['lifetime'] : [];
+            $serverResources = is_array($server['server_resources'] ?? null) ? $server['server_resources'] : null;
 
             $result = [
                 'hostname' => $server['hostname'] ?? '',
                 // Worker-level metrics (from queue workers)
                 'workers' => [
-                    'total' => $workers['total'] ?? 0,
-                    'active' => $workers['active'] ?? 0,
-                    'idle' => $workers['idle'] ?? 0,
-                    'utilization_percent' => $utilizationPercent,
+                    'total' => $workerCount['total'] ?? 0,
+                    'active' => $workerCount['active'] ?? 0,
+                    'idle' => $workerCount['idle'] ?? 0,
+                    'current_busy_percent' => $workerUtilization['current_busy_percent'] ?? 0.0,
+                    'lifetime_busy_percent' => $workerUtilization['lifetime_busy_percent'] ?? 0.0,
                 ],
                 // Job processing metrics (from queue workers)
                 'jobs' => [
-                    'processed' => $performance['total_jobs_processed'] ?? 0,
+                    'total_processed' => $jobLifetime['total_processed'] ?? 0,
+                    'total_failed' => $jobLifetime['total_failed'] ?? 0,
+                    'failure_rate_percent' => $jobLifetime['failure_rate_percent'] ?? 0.0,
                 ],
             ];
 
-            // System resource limits (physical server capacity)
-            if ($systemLimits !== null) {
-                $result['system_limits'] = $systemLimits;
+            // System resource metrics (actual server CPU/memory from SystemMetrics)
+            // This is the REAL server usage, not worker process usage
+            if ($serverResources !== null) {
+                $result['server_resources'] = $serverResources;
             }
 
             return $result;
@@ -226,11 +239,17 @@ private function filterServersForDashboard(array $servers): array
      */
     private function filterWorkersForDashboard(array $workers): array
     {
+        $count = is_array($workers['count'] ?? null) ? $workers['count'] : [];
+        $utilization = is_array($workers['utilization'] ?? null) ? $workers['utilization'] : [];
+        $performance = is_array($workers['performance'] ?? null) ? $workers['performance'] : [];
+
         return [
-            'total' => $workers['total'] ?? 0,
-            'active' => $workers['active'] ?? 0,
-            'idle' => $workers['idle'] ?? 0,
-            'total_jobs_processed' => $workers['total_jobs_processed'] ?? 0,
+            'total' => $count['total'] ?? 0,
+            'active' => $count['active'] ?? 0,
+            'idle' => $count['idle'] ?? 0,
+            'current_busy_percent' => $utilization['current_busy_percent'] ?? 0.0,
+            'lifetime_busy_percent' => $utilization['lifetime_busy_percent'] ?? 0.0,
+            'total_jobs_processed' => $performance['total_jobs_processed'] ?? 0,
         ];
     }
 }
diff --git a/src/Services/QueueMetricsQueryService.php b/src/Services/QueueMetricsQueryService.php
@@ -143,7 +143,7 @@ public function getAllQueuesWithMetrics(): array
 
                 $activeWorkers = $workers->count();
 
-                // Calculate queue utilization rate from worker busy/idle time
+                // Calculate worker utilization from busy/idle time
                 $totalBusyTime = 0;
                 $totalIdleTime = 0;
                 foreach ($workers as $worker) {
@@ -152,7 +152,11 @@ public function getAllQueuesWithMetrics(): array
                 }
 
                 $totalTime = $totalBusyTime + $totalIdleTime;
-                $utilizationRate = $totalTime > 0 ? ($totalBusyTime / $totalTime) * 100 : 0;
+                $lifetimeBusyPercent = $totalTime > 0 ? ($totalBusyTime / $totalTime) * 100 : 0;
+
+                // Calculate current worker state (% busy right now)
+                $busyWorkers = $workers->filter(fn ($w) => $w->state->value === 'busy')->count();
+                $currentBusyPercent = $activeWorkers > 0 ? ($busyWorkers / $activeWorkers) * 100 : 0;
 
                 // Get trend data
                 $trends = $this->getQueueTrends($connection, $queue);
@@ -161,17 +165,31 @@ public function getAllQueuesWithMetrics(): array
                     'connection' => $connection,
                     'queue' => $queue,
                     'driver' => $connection,
-                    'depth' => $depth->totalJobs(),
-                    'pending' => $depth->pendingJobs,
-                    'scheduled' => $depth->delayedJobs,
-                    'reserved' => $depth->reservedJobs,
-                    'oldest_job_age_seconds' => $depth->secondsOldestPendingJob() ?? 0,
-                    'oldest_job_age_status' => $depth->oldestPendingJobAge?->toIso8601String() ?? 'unknown',
-                    'throughput_per_minute' => $metrics->throughputPerMinute,
-                    'avg_duration_ms' => $metrics->avgDuration,
-                    'failure_rate' => $metrics->failureRate,
-                    'utilization_rate' => round($utilizationRate, 2),
-                    'active_workers' => $activeWorkers,
+                    // Instantaneous queue state (current snapshot)
+                    'depth' => [
+                        'total' => $depth->totalJobs(),
+                        'pending' => $depth->pendingJobs,
+                        'scheduled' => $depth->delayedJobs,
+                        'reserved' => $depth->reservedJobs,
+                        'oldest_job_age_seconds' => $depth->secondsOldestPendingJob() ?? 0,
+                        'oldest_job_age_status' => $depth->oldestPendingJobAge?->toIso8601String() ?? 'unknown',
+                    ],
+                    // Windowed performance metrics (60-second window from CalculateQueueMetricsAction)
+                    'performance_60s' => [
+                        'throughput_per_minute' => $metrics->throughputPerMinute,
+                        'avg_duration_ms' => $metrics->avgDuration,
+                        'window_seconds' => 60,
+                    ],
+                    // Lifetime metrics (since first job)
+                    'lifetime' => [
+                        'failure_rate_percent' => $metrics->failureRate,
+                    ],
+                    // Worker metrics for this queue
+                    'workers' => [
+                        'active_count' => $activeWorkers,
+                        'current_busy_percent' => round($currentBusyPercent, 2),
+                        'lifetime_busy_percent' => round($lifetimeBusyPercent, 2),
+                    ],
                     'baseline' => $baseline ? $baseline->toArray() : null,
                     'trends' => $trends,
                     'timestamp' => now()->toIso8601String(),
diff --git a/src/Services/TrendAnalysisService.php b/src/Services/TrendAnalysisService.php
@@ -96,23 +96,33 @@ public function analyzeQueueDepthTrend(
             'available' => true,
             'connection' => $connection,
             'queue' => $queue,
-            'period_seconds' => $periodSeconds,
-            'data_points' => $count,
+            // Time window context
+            'time_window' => [
+                'window_seconds' => $periodSeconds,
+                'window_start' => $startTime->toIso8601String(),
+                'window_end' => $now->toIso8601String(),
+                'analyzed_at' => $now->toIso8601String(),
+                'sample_count' => $count,
+                'sample_interval_seconds' => $intervalSeconds,
+            ],
+            // Current and historical statistics
             'statistics' => [
                 'current' => end($depths) ?: 0,
                 'average' => round($avg, 2),
                 'min' => $min,
                 'max' => $max,
                 'std_dev' => round($stdDev, 2),
             ],
+            // Trend direction and confidence
             'trend' => [
                 'slope' => round($trend['slope'], 4),
                 'direction' => $trend['slope'] > 0.1 ? 'increasing' : ($trend['slope'] < -0.1 ? 'decreasing' : 'stable'),
                 'confidence' => round($trend['r_squared'], 3),
             ],
+            // Forecast for next interval
             'forecast' => [
                 'next_value' => round($forecast, 2),
-                'next_timestamp' => $now->copy()->addSeconds($intervalSeconds)->timestamp,
+                'next_timestamp' => $now->copy()->addSeconds($intervalSeconds)->toIso8601String(),
             ],
         ];
     }
@@ -185,13 +195,22 @@ public function analyzeThroughputTrend(
             'available' => true,
             'connection' => $connection,
             'queue' => $queue,
-            'period_seconds' => $periodSeconds,
+            // Time window context
+            'time_window' => [
+                'window_seconds' => $periodSeconds,
+                'window_start' => $startTime->toIso8601String(),
+                'window_end' => $now->toIso8601String(),
+                'analyzed_at' => $now->toIso8601String(),
+                'sample_count' => $count,
+            ],
+            // Throughput statistics
             'statistics' => [
                 'total_jobs' => $totalJobs,
                 'average_per_interval' => round($avg, 2),
                 'jobs_per_minute' => round($jobsPerMinute, 2),
                 'jobs_per_hour' => round($jobsPerMinute * 60, 2),
             ],
+            // Trend direction
             'trend' => [
                 'slope' => round($trend['slope'], 4),
                 'direction' => $trend['slope'] > 0 ? 'increasing' : ($trend['slope'] < 0 ? 'decreasing' : 'stable'),
@@ -256,14 +275,22 @@ public function analyzeWorkerEfficiencyTrend(int $periodSeconds = 3600): array
 
         return [
             'available' => true,
-            'period_seconds' => $periodSeconds,
-            'data_points' => $count,
+            // Time window context
+            'time_window' => [
+                'window_seconds' => $periodSeconds,
+                'window_start' => $startTime->toIso8601String(),
+                'window_end' => $now->toIso8601String(),
+                'analyzed_at' => $now->toIso8601String(),
+                'sample_count' => $count,
+            ],
+            // Worker efficiency statistics
             'efficiency' => [
                 'current' => round(end($efficiencies) ?: 0, 2),
                 'average' => round(array_sum($efficiencies) / max($count, 1), 2),
                 'min' => round(min($efficiencies), 2),
                 'max' => round(max($efficiencies), 2),
             ],
+            // Resource usage statistics
             'resource_usage' => [
                 'avg_memory_mb' => round(array_sum($memoryUsages) / max($count, 1), 2),
                 'avg_cpu_percent' => round(array_sum($cpuUsages) / max($count, 1), 2),
diff --git a/src/Services/WorkerMetricsQueryService.php b/src/Services/WorkerMetricsQueryService.php
