Only measure mallocCountTotal in Benchmarks

mimischi · mimischi · commit baa1e9e73da6 · 2024-11-26T15:51:16.000Z
Keeps `scalingFactor: .kilo`, but only measures `.mallocCountTotal`, as
that should be a reproducible value across different systems.
diff --git a/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift b/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift
@@ -32,27 +32,12 @@ let benchmarks = {
 
     Benchmark.defaultConfiguration = .init(
         metrics: [
-            .wallClock,
-            .cpuTotal,
-            .contextSwitches,
-            .throughput,
-            .allocatedResidentMemory,
-        ] + .arc,
+            .mallocCountTotal,
+        ],
         // We need to tell the benchmarking framework how often we are running the benchmark.
         scalingFactor: .kilo,
         maxDuration: .seconds(10_000_000),
-        maxIterations: 10,
-        thresholds: [
-            .wallClock: .init(relative: [.p90: 35]),
-            .cpuTotal: .init(relative: [.p90: 35]),
-            .allocatedResidentMemory: .init(relative: [.p90: 20]),
-            .contextSwitches: .init(relative: [.p90: 35]),
-            .throughput: .init(relative: [.p90: 35]),
-            .objectAllocCount: .init(relative: [.p90: 20]),
-            .retainCount: .init(relative: [.p90: 20]),
-            .releaseCount: .init(relative: [.p90: 20]),
-            .retainReleaseDelta: .init(relative: [.p90: 20]),
-        ]
+        maxIterations: 10
     )
 
     Benchmark.setup = {
diff --git a/Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 562561023,
-  "cpuTotal" : 210108415,
-  "objectAllocCount" : 5971,
-  "releaseCount" : 17775,
-  "retainCount" : 9638,
-  "retainReleaseDelta" : 2175,
-  "throughput" : 2,
-  "wallClock" : 638582783
+  "mallocCountTotal" : 42079
 }
diff --git a/Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 554172415,
-  "cpuTotal" : 170000000,
-  "objectAllocCount" : 6991,
-  "releaseCount" : 19823,
-  "retainCount" : 10655,
-  "retainReleaseDelta" : 2185,
-  "throughput" : 2,
-  "wallClock" : 629669887
+  "mallocCountTotal" : 59647
 }
diff --git a/Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 190971903,
-  "cpuTotal" : 460000000,
-  "objectAllocCount" : 16,
-  "releaseCount" : 48,
-  "retainCount" : 2,
-  "retainReleaseDelta" : 30,
-  "throughput" : 1,
-  "wallClock" : 1870163542
+  "mallocCountTotal" : 1000959
 }
diff --git a/Benchmarks/Thresholds/5.9/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/Benchmarks/Thresholds/5.9/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 574619647,
-  "cpuTotal" : 240000000,
-  "objectAllocCount" : 5907,
-  "releaseCount" : 16639,
-  "retainCount" : 8591,
-  "retainReleaseDelta" : 2135,
-  "throughput" : 2,
-  "wallClock" : 647495679
+  "mallocCountTotal" : 29935
 }
diff --git a/Benchmarks/Thresholds/5.9/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/Benchmarks/Thresholds/5.9/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 551026687,
-  "cpuTotal" : 180092927,
-  "objectAllocCount" : 7011,
-  "releaseCount" : 18865,
-  "retainCount" : 9668,
-  "retainReleaseDelta" : 2187,
-  "throughput" : 2,
-  "wallClock" : 639631359
+  "mallocCountTotal" : 49983
 }
diff --git a/Benchmarks/Thresholds/5.9/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/Benchmarks/Thresholds/5.9/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 265158655,
-  "cpuTotal" : 490209279,
-  "objectAllocCount" : 16,
-  "releaseCount" : 52,
-  "retainCount" : 2,
-  "retainReleaseDelta" : 34,
-  "throughput" : 1,
-  "wallClock" : 1876951039
+  "mallocCountTotal" : 1000959
 }
diff --git a/Benchmarks/Thresholds/6.0/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/Benchmarks/Thresholds/6.0/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 566231039,
-  "cpuTotal" : 200015871,
-  "objectAllocCount" : 5431,
-  "releaseCount" : 18687,
-  "retainCount" : 10863,
-  "retainReleaseDelta" : 2415,
-  "throughput" : 2,
-  "wallClock" : 649592831
+  "mallocCountTotal" : 31695
 }
diff --git a/Benchmarks/Thresholds/6.0/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/Benchmarks/Thresholds/6.0/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 533725183,
-  "cpuTotal" : 180092927,
-  "objectAllocCount" : 6447,
-  "releaseCount" : 21743,
-  "retainCount" : 12895,
-  "retainReleaseDelta" : 2417,
-  "throughput" : 2,
-  "wallClock" : 639107071
+  "mallocCountTotal" : 56831
 }
diff --git a/Benchmarks/Thresholds/6.0/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/Benchmarks/Thresholds/6.0/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 565706751,
-  "cpuTotal" : 450101247,
-  "objectAllocCount" : 16,
-  "releaseCount" : 40,
-  "retainCount" : 2,
-  "retainReleaseDelta" : 22,
-  "throughput" : 1,
-  "wallClock" : 1858076671
+  "mallocCountTotal" : 1000959
 }
diff --git a/Benchmarks/Thresholds/nightly-6.0/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/Benchmarks/Thresholds/nightly-6.0/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 582811648,
-  "cpuTotal" : 220000000,
-  "objectAllocCount" : 5379,
-  "releaseCount" : 18495,
-  "retainCount" : 10767,
-  "retainReleaseDelta" : 2341,
-  "throughput" : 2,
-  "wallClock" : 655359999
+  "mallocCountTotal" : 29295
 }
diff --git a/Benchmarks/Thresholds/nightly-6.0/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/Benchmarks/Thresholds/nightly-6.0/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 490340352,
-  "cpuTotal" : 180000000,
-  "objectAllocCount" : 6411,
-  "releaseCount" : 21615,
-  "retainCount" : 12815,
-  "retainReleaseDelta" : 2381,
-  "throughput" : 2,
-  "wallClock" : 645922815
+  "mallocCountTotal" : 48287
 }
diff --git a/Benchmarks/Thresholds/nightly-6.0/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/Benchmarks/Thresholds/nightly-6.0/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,3 @@
 {
-  "allocatedResidentMemory" : 587726847,
-  "cpuTotal" : 460000000,
-  "objectAllocCount" : 16,
-  "releaseCount" : 52,
-  "retainCount" : 2,
-  "retainReleaseDelta" : 34,
-  "throughput" : 1,
-  "wallClock" : 1866465279
+  "mallocCountTotal" : 1000447
 }

Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 562561023,`
`3`		`- "cpuTotal" : 210108415,`
`4`		`- "objectAllocCount" : 5971,`
`5`		`- "releaseCount" : 17775,`
`6`		`- "retainCount" : 9638,`
`7`		`- "retainReleaseDelta" : 2175,`
`8`		`- "throughput" : 2,`
`9`		`- "wallClock" : 638582783`
	`2`	`+ "mallocCountTotal" : 42079`
`10`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 554172415,`
`3`		`- "cpuTotal" : 170000000,`
`4`		`- "objectAllocCount" : 6991,`
`5`		`- "releaseCount" : 19823,`
`6`		`- "retainCount" : 10655,`
`7`		`- "retainReleaseDelta" : 2185,`
`8`		`- "throughput" : 2,`
`9`		`- "wallClock" : 629669887`
	`2`	`+ "mallocCountTotal" : 59647`
`10`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 190971903,`
`3`		`- "cpuTotal" : 460000000,`
`4`		`- "objectAllocCount" : 16,`
`5`		`- "releaseCount" : 48,`
`6`		`- "retainCount" : 2,`
`7`		`- "retainReleaseDelta" : 30,`
`8`		`- "throughput" : 1,`
`9`		`- "wallClock" : 1870163542`
	`2`	`+ "mallocCountTotal" : 1000959`
`10`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 574619647,`
`3`		`- "cpuTotal" : 240000000,`
`4`		`- "objectAllocCount" : 5907,`
`5`		`- "releaseCount" : 16639,`
`6`		`- "retainCount" : 8591,`
`7`		`- "retainReleaseDelta" : 2135,`
`8`		`- "throughput" : 2,`
`9`		`- "wallClock" : 647495679`
	`2`	`+ "mallocCountTotal" : 29935`
`10`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 551026687,`
`3`		`- "cpuTotal" : 180092927,`
`4`		`- "objectAllocCount" : 7011,`
`5`		`- "releaseCount" : 18865,`
`6`		`- "retainCount" : 9668,`
`7`		`- "retainReleaseDelta" : 2187,`
`8`		`- "throughput" : 2,`
`9`		`- "wallClock" : 639631359`
	`2`	`+ "mallocCountTotal" : 49983`
`10`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 265158655,`
`3`		`- "cpuTotal" : 490209279,`
`4`		`- "objectAllocCount" : 16,`
`5`		`- "releaseCount" : 52,`
`6`		`- "retainCount" : 2,`
`7`		`- "retainReleaseDelta" : 34,`
`8`		`- "throughput" : 1,`
`9`		`- "wallClock" : 1876951039`
	`2`	`+ "mallocCountTotal" : 1000959`
`10`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 566231039,`
`3`		`- "cpuTotal" : 200015871,`
`4`		`- "objectAllocCount" : 5431,`
`5`		`- "releaseCount" : 18687,`
`6`		`- "retainCount" : 10863,`
`7`		`- "retainReleaseDelta" : 2415,`
`8`		`- "throughput" : 2,`
`9`		`- "wallClock" : 649592831`
	`2`	`+ "mallocCountTotal" : 31695`
`10`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 533725183,`
`3`		`- "cpuTotal" : 180092927,`
`4`		`- "objectAllocCount" : 6447,`
`5`		`- "releaseCount" : 21743,`
`6`		`- "retainCount" : 12895,`
`7`		`- "retainReleaseDelta" : 2417,`
`8`		`- "throughput" : 2,`
`9`		`- "wallClock" : 639107071`
	`2`	`+ "mallocCountTotal" : 56831`
`10`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,3 @@`
`1`	`1`	`{`
`2`		`- "allocatedResidentMemory" : 565706751,`
`3`		`- "cpuTotal" : 450101247,`
`4`		`- "objectAllocCount" : 16,`
`5`		`- "releaseCount" : 40,`
`6`		`- "retainCount" : 2,`
`7`		`- "retainReleaseDelta" : 22,`
`8`		`- "throughput" : 1,`
`9`		`- "wallClock" : 1858076671`
	`2`	`+ "mallocCountTotal" : 1000959`
`10`	`3`	`}`