roachtest: adjust error tolerance for DiskStalledWALFailoverWithProgress

Bump throughput error tolerance from 20% to 25% and drop the last throughput
sample if it is 0 to address flakiness.

Fixes: #153328
Fixes: #152781

Author: xinhaoz

pkg/cmd/roachtest/tests/disk_stall.go

@@ -467,7 +467,7 @@ func runDiskStalledWALFailoverWithProgress(ctx context.Context, t test.Test, c c
 		operationDur      = 3 * time.Minute
 		// QPS sampling parameters.
 		sampleInterval = 10 * time.Second
-		errorTolerance = 0.2 // 20% tolerance for throughput variation.
+		errorTolerance = 0.25 // 25% tolerance for throughput variation.
 	)
 
 	t.Status("setting up disk staller")
@@ -566,20 +566,21 @@ func runDiskStalledWALFailoverWithProgress(ctx context.Context, t test.Test, c c
 				t.Fatalf("context done before workload started: %s", ctx.Err())
 			case <-workloadStarted:
 			}
+			// Wait 30s after workload starts before beginning sampling.
+			const workloadStartDelay = 30 * time.Second
+			// Calculate approximate how many samples to take. We want to account
+			// for the time waited for workload startup and we should also stop
+			// sampling ~15s before the workload starts shutting down.
+			samplingDuration := operationDur - workloadStartDelay - 15*time.Second
+			sampleCount := int(samplingDuration / sampleInterval)
 
-			// Wait 20s after workload starts before beginning sampling.
 			select {
 			case <-ctx.Done():
 				t.Fatalf("context done before workload started: %s", ctx.Err())
-			case <-time.After(30 * time.Second):
+			case <-time.After(workloadStartDelay):
 				t.Status("starting QPS sampling")
 			}
 
-			// We want to stop sampling 10s before workload ends to avoid sampling during shutdown.
-			// We'll take approx. 14 samples with this configuration.
-			samplingDuration := operationDur - 40*time.Second // 30s initial wait + 10s buffer at workload end
-			sampleCount := int(samplingDuration / sampleInterval)
-
 			sampleTimer := time.NewTicker(sampleInterval)
 			defer sampleTimer.Stop()
 
@@ -648,12 +649,22 @@ func runDiskStalledWALFailoverWithProgress(ctx context.Context, t test.Test, c c
 		// Wait for all goroutines to complete.
 		g.Wait()
 
+		if len(samples) == 0 {
+			t.Fatalf("no throughput samples collected for iteration %d", iteration)
+		}
+
 		// Validate throughput samples are within tolerance.
+		// Drop the last one if it is 0, since we can't fully sync the sampling
+		// with workload startup/shutdown, it may have been taken while the workload
+		// was shutting down.
+		if samples[len(samples)-1] == 0 {
+			samples = samples[:len(samples)-1]
+		}
 		meanThroughput := roachtestutil.GetMeanOverLastN(len(samples), samples)
 		t.Status("mean throughput for iteration", iteration, ": ", meanThroughput)
 		for _, sample := range samples {
 			require.InEpsilonf(t, meanThroughput, sample, errorTolerance,
-				"sample %f is not within tolerance of mean %f", sample, meanThroughput)
+				"sample %f is not within tolerance of mean %f\nsamples:%v", sample, meanThroughput, samples)
 		}
 		iterationMeans = append(iterationMeans, meanThroughput)
 		iteration++
@@ -671,7 +682,8 @@ func runDiskStalledWALFailoverWithProgress(ctx context.Context, t test.Test, c c
 	overallMean := roachtestutil.GetMeanOverLastN(len(iterationMeans), iterationMeans)
 	for _, mean := range iterationMeans {
 		require.InEpsilonf(t, overallMean, mean, errorTolerance,
-			"iteration mean %f is not within tolerance of overall mean %f", mean, overallMean)
+			"iteration mean %f is not within tolerance of overall mean %f\niteration means:%v", mean,
+			overallMean, iterationMeans)
 	}
 
 	data := mustGetMetrics(ctx, c, t, adminURL, install.SystemInterfaceName,