Issue #1024: Calculate baseline by the fastest benchmark

Author: Serg046

src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs

@@ -0,0 +1,13 @@
+using BenchmarkDotNet.Configs;
+using System;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [AttributeUsage(AttributeTargets.Class)]
+    public class AutomaticBaselineAttribute : Attribute, IConfigSource
+    {
+        public IConfig Config { get; }
+
+        public AutomaticBaselineAttribute(AutomaticBaselineMode mode) => Config = ManualConfig.CreateEmpty().WithAutomaticBaseline(mode);
+    }
+}

src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs

@@ -0,0 +1,8 @@
+namespace BenchmarkDotNet.Configs
+{
+    public enum AutomaticBaselineMode
+    {
+        None,
+        Fastest
+    }
+}

src/BenchmarkDotNet/Configs/DebugConfig.cs

@@ -87,5 +87,7 @@ public string ArtifactsPath
         public ConfigOptions Options => ConfigOptions.KeepBenchmarkFiles | ConfigOptions.DisableOptimizationsValidator;
 
         public IReadOnlyList<Conclusion> ConfigAnalysisConclusion => emptyConclusion;
+
+        public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None;
     }
 }

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -106,5 +106,7 @@ public string ArtifactsPath
         public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
 
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
+
+        public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None;
     }
 }

src/BenchmarkDotNet/Configs/IConfig.cs

@@ -57,5 +57,7 @@ public interface IConfig
         /// Collect any errors or warnings when composing the configuration
         /// </summary>
         IReadOnlyList<Conclusion> ConfigAnalysisConclusion { get; }
+
+        AutomaticBaselineMode AutomaticBaselineMode { get; }
     }
 }

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -53,7 +53,8 @@ internal ImmutableConfig(
             SummaryStyle summaryStyle,
             ConfigOptions options,
             TimeSpan buildTimeout,
-            IReadOnlyList<Conclusion> configAnalysisConclusion)
+            IReadOnlyList<Conclusion> configAnalysisConclusion,
+            AutomaticBaselineMode automaticBaselineMode)
         {
             columnProviders = uniqueColumnProviders;
             loggers = uniqueLoggers;
@@ -74,6 +75,7 @@ internal ImmutableConfig(
             Options = options;
             BuildTimeout = buildTimeout;
             ConfigAnalysisConclusion = configAnalysisConclusion;
+            AutomaticBaselineMode = automaticBaselineMode;
         }
 
         public ConfigUnionRule UnionRule { get; }
@@ -83,6 +85,7 @@ internal ImmutableConfig(
         [NotNull] public IOrderer Orderer { get; }
         public SummaryStyle SummaryStyle { get; }
         public TimeSpan BuildTimeout { get; }
+        public AutomaticBaselineMode AutomaticBaselineMode { get; }
 
         public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
         public IEnumerable<IExporter> GetExporters() => exporters;

src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs

@@ -71,7 +71,8 @@ public static ImmutableConfig Create(IConfig source)
                 source.SummaryStyle ?? SummaryStyle.Default,
                 source.Options,
                 source.BuildTimeout,
-                configAnalyse.AsReadOnly()
+                configAnalyse.AsReadOnly(),
+                source.AutomaticBaselineMode
             );
         }
 

src/BenchmarkDotNet/Configs/ManualConfig.cs

@@ -53,6 +53,7 @@ public class ManualConfig : IConfig
         [PublicAPI] public IOrderer Orderer { get; set; }
         [PublicAPI] public SummaryStyle SummaryStyle { get; set; }
         [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout;
+        [PublicAPI] public AutomaticBaselineMode AutomaticBaselineMode { get; private set; }
 
         public IReadOnlyList<Conclusion> ConfigAnalysisConclusion => emptyConclusion;
 
@@ -98,6 +99,12 @@ public ManualConfig WithBuildTimeout(TimeSpan buildTimeout)
             return this;
         }
 
+        public ManualConfig WithAutomaticBaseline(AutomaticBaselineMode automaticBaselineMode)
+        {
+            AutomaticBaselineMode = automaticBaselineMode;
+            return this;
+        }
+
         [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete("This method will soon be removed, please start using .AddColumn() instead.")]
         public void Add(params IColumn[] newColumns) => AddColumn(newColumns);
@@ -254,6 +261,7 @@ public void Add(IConfig config)
             columnHidingRules.AddRange(config.GetColumnHidingRules());
             Options |= config.Options;
             BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout);
+            AutomaticBaselineMode = config.AutomaticBaselineMode;
         }
 
         /// <summary>

src/BenchmarkDotNet/Reports/Summary.cs

@@ -37,6 +37,7 @@ public class Summary
         private ImmutableDictionary<BenchmarkCase, BenchmarkReport> ReportMap { get; }
         private BaseliningStrategy BaseliningStrategy { get; }
         private bool? isMultipleRuntimes;
+        private readonly BenchmarkCase inferredBaselineBenchmarkCase;
 
         public Summary(
             string title,
@@ -62,13 +63,32 @@ public Summary(
             DisplayPrecisionManager = new DisplayPrecisionManager(this);
             Orderer = GetConfiguredOrdererOrDefaultOne(reports.Select(report => report.BenchmarkCase.Config));
             BenchmarksCases = Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first
+            inferredBaselineBenchmarkCase = GetFastestBenchmarkCase(reports);
             Reports = BenchmarksCases.Select(b => ReportMap[b]).ToImmutableArray(); // we use sorted collection to re-create reports list
             BaseliningStrategy = BaseliningStrategy.Create(BenchmarksCases);
             Style = GetConfiguredSummaryStyleOrDefaultOne(BenchmarksCases).WithCultureInfo(cultureInfo);
             Table = GetTable(Style);
             AllRuntimes = BuildAllRuntimes(HostEnvironmentInfo, Reports);
         }
 
+        private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray<BenchmarkReport> reports)
+        {
+            if (reports.Any() && reports.All(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest))
+            {
+                var fastestReport = reports.First();
+                foreach (var report in reports.Skip(1))
+                {
+                    if (report.ResultStatistics.Mean < fastestReport.ResultStatistics.Mean)
+                    {
+                        fastestReport = report;
+                    }
+                }
+                return fastestReport.BenchmarkCase;
+            }
+
+            return null;
+        }
+
         [PublicAPI] public bool HasReport(BenchmarkCase benchmarkCase) => ReportMap.ContainsKey(benchmarkCase);
 
         /// <summary>
@@ -133,7 +153,11 @@ public string GetLogicalGroupKey(BenchmarkCase benchmarkCase)
             => Orderer.GetLogicalGroupKey(BenchmarksCases, benchmarkCase);
 
         public bool IsBaseline(BenchmarkCase benchmarkCase)
-            => BaseliningStrategy.IsBaseline(benchmarkCase);
+        {
+            return inferredBaselineBenchmarkCase != null
+                ? inferredBaselineBenchmarkCase == benchmarkCase
+                : BaseliningStrategy.IsBaseline(benchmarkCase);
+        }
 
         [CanBeNull]
         public BenchmarkCase GetBaseline(string logicalGroupKey)

tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs

@@ -0,0 +1,53 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using System.Linq;
+using System.Threading;
+using Xunit;
+
+namespace BenchmarkDotNet.IntegrationTests
+{
+    public class AutomaticBaselineTests
+    {
+        [Fact]
+        public void AutomaticBaselineSelectionIsCorrect()
+        {
+            var config = CreateConfig();
+
+            var summary = BenchmarkRunner.Run<BaselineSample>();
+
+            var table = summary.GetTable(SummaryStyle.Default);
+            var column = table.Columns.Single(c => c.Header == "Ratio");
+            Assert.Equal(2, column.Content.Length);
+            Assert.Equal(1.0, double.Parse(column.Content[1])); // Ratio of TwoMilliseconds
+            Assert.True(double.Parse(column.Content[0]) > 1.0); // Ratio of TwoHundredMilliseconds
+        }
+
+        [AutomaticBaseline(AutomaticBaselineMode.Fastest)]
+        public class BaselineSample
+        {
+            [Benchmark]
+            public void TwoHundredMilliseconds()
+            {
+                Thread.Sleep(200);
+            }
+
+            [Benchmark]
+            public void TwoMilliseconds()
+            {
+                Thread.Sleep(2);
+            }
+        }
+
+        private IConfig CreateConfig()
+            => ManualConfig.CreateEmpty()
+                .AddJob(Job.ShortRun
+                    .WithEvaluateOverhead(false) // no need to run idle for this test
+                    .WithWarmupCount(0) // don't run warmup to save some time for our CI runs
+                    .WithIterationCount(1)) // single iteration is enough for us
+                .AddColumnProvider(DefaultColumnProviders.Instance);
+    }
+}