Unify benchmark and testing framework

Makes it much easier and more readable to add new
benchmarking testcases.

Now just inherit from BenchmarkTestcase.

Also share code that was essentially repeated.

Author: will-keen

benchmarks/__main__.py

@@ -5,171 +5,15 @@
 Benchmark against pyvsc library for equivalent testcases.
 '''
 
-import unittest
-import timeit
-import sys
 
-from argparse import ArgumentParser
-from random import Random
+from tests.main import main
 
-from benchmarks.pyvsc.basic import vsc_basic, cr_basic
-from benchmarks.pyvsc.in_keyword import vsc_in, cr_in, cr_in_order
-from benchmarks.pyvsc.ldinstr import vsc_ldinstr
-from benchmarks.pyvsc.randlist import vscRandListSumZero, \
-    crRandListSumZero, vscRandListUnique, crRandListUnique, \
-    crRandListSumZeroFaster, crRandListUniqueFaster
-from examples.ldinstr import ldInstr
-
-
-TEST_LENGTH_MULTIPLIER = 1
-
-
-class BenchmarkTests(unittest.TestCase):
-
-    def run_one(self, name, randobj, iterations):
-        '''
-        Benchmark one object that implements the .randomize() function
-        for N iterations.
-        '''
-        start_time = timeit.default_timer()
-        for _ in range(iterations):
-            randobj.randomize()
-        end_time = timeit.default_timer()
-        total_time = end_time - start_time
-        hz = iterations / total_time
-        print(f'{self._testMethodName}: {name} took {total_time:.4g}s for {iterations} iterations ({hz:.1f}Hz)')
-        return total_time, hz
-
-    def run_benchmark(self, randobjs, iterations, check):
-        '''
-        Reusable function to run a fair benchmark between
-        two or more randomizable objects.
-
-        randobjs:   dictionary where key is name, value is an object that implements .randomize()
-        iterations: how many times to call .randomize()
-        check:      function taking a dictionary of results to check.
-        '''
-        iterations *= TEST_LENGTH_MULTIPLIER
-        results = {}
-        winner = None
-        best_hz = 0
-        for name, randobj in randobjs.items():
-            total_time, hz = self.run_one(name, randobj, iterations)
-            if hz > best_hz:
-                winner = name
-                best_hz = hz
-            # Store both total time and Hz in case the test wants to specify
-            # checks on how long should be taken in wall clock time.
-            results[name] = total_time, hz
-        print(f'{self._testMethodName}: The winner is {winner} with {best_hz:.1f}Hz!')
-        # Print summary of hz delta
-        for name, (_total_time, hz) in results.items():
-            if name == winner:
-                continue
-            speedup = best_hz / hz
-            print(f'{self._testMethodName}: {winner} was {speedup:.2f}x faster than {name}')
-        check(results)
-
-    def test_basic(self):
-        '''
-        Test basic randomizable object.
-        '''
-        randobjs = {'vsc': vsc_basic(), 'cr': cr_basic(Random(0))}
-        def check(results):
-            self.assertGreater(results['cr'][1], results['vsc'][1])
-            # This testcase is typically 40-50x faster, which may vary depending
-            # on machine. Ensure it doesn't fall below 30x.
-            speedup = results['cr'][1] / results['vsc'][1]
-            self.assertGreater(speedup, 30, "Performance has degraded!")
-        self.run_benchmark(randobjs, 100, check)
-
-    def test_in(self):
-        '''
-        Test object using 'in' keyword.
-        '''
-        randobjs = {
-            'vsc': vsc_in(),
-            'cr': cr_in(Random(0)),
-            'cr_order': cr_in_order(Random(0)),
-        }
-        def check(results):
-            self.assertGreater(results['cr'][1], results['vsc'][1])
-            # This testcase is typically 13-15x faster, which may vary depending
-            # on machine. Ensure it doesn't fall below 10x.
-            speedup = results['cr'][1] / results['vsc'][1]
-            self.assertGreater(speedup, 10, "Performance has degraded!")
-        self.run_benchmark(randobjs, 100, check)
-
-    def test_ldinstr(self):
-        '''
-        Test LD instruction example.
-        '''
-        randobjs = {
-            'vsc': vsc_ldinstr(),
-            'cr': ldInstr(Random(0)),
-        }
-        def check(results):
-            self.assertGreater(results['cr'][1], results['vsc'][1])
-            # This testcase is typically 13-15x faster, which may vary depending
-            # on machine. Ensure it doesn't fall below 10x.
-            speedup = results['cr'][1] / results['vsc'][1]
-            self.assertGreater(speedup, 10, "Performance has degraded!")
-        self.run_benchmark(randobjs, 100, check)
-
-    def test_randlist_sumzero(self):
-        '''
-        Test random list example where the list must sum to zero.
-        '''
-        randobjs = {
-            'vsc': vscRandListSumZero(),
-            'cr': crRandListSumZero(Random(0)),
-            'cr_faster': crRandListSumZeroFaster(Random(0)),
-        }
-        def check(results):
-            self.assertGreater(results['cr'][1], results['vsc'][1])
-            self.assertGreater(results['cr_faster'][1], results['vsc'][1])
-            # This testcase is typically 20x faster, which may vary depending
-            # on machine. Ensure it doesn't fall below 15x.
-            speedup = results['cr'][1] / results['vsc'][1]
-            self.assertGreater(speedup, 15, "Performance has degraded!")
-            speedup = results['cr_faster'][1] / results['vsc'][1]
-            self.assertGreater(speedup, 15, "Performance has degraded!")
-        self.run_benchmark(randobjs, 100, check)
-
-    def test_randlist_unique(self):
-        '''
-        Test random list example where the list must be unique.
-        '''
-        randobjs = {
-            'vsc': vscRandListUnique(),
-            'cr': crRandListUnique(Random(0)),
-            'cr_faster': crRandListUniqueFaster(Random(0)),
-        }
-        def check(results):
-            self.assertGreater(results['cr_faster'][1], results['vsc'][1])
-            self.assertGreater(results['cr'][1], results['vsc'][1])
-            # With the naive solver, this testcase is typically 3-4x faster,
-            # which may vary depending on machine. Ensure it doesn't fall
-            # below 2x.
-            speedup = results['cr'][1] / results['vsc'][1]
-            self.assertGreater(speedup, 2, "Performance has degraded!")
-            # This testcase is typically 10-13x faster, which may vary depending
-            # on machine. Ensure it doesn't fall below 10x.
-            speedup = results['cr_faster'][1] / results['vsc'][1]
-            self.assertGreater(speedup, 10, "Performance has degraded!")
-        self.run_benchmark(randobjs, 100, check)
-
-
-def parse_args():
-    parser = ArgumentParser(description='Run unit tests for constrainedrandom library')
-    parser.add_argument('--length-mul', type=int, default=1, help='Multiplier for test length, when desiring greater certainty on performance.')
-    args, extra = parser.parse_known_args()
-    return args, extra
+from benchmarks.pyvsc.basic import VSCBasic
+from benchmarks.pyvsc.in_keyword import VSCIn
+from benchmarks.pyvsc.ldinstr import VSCInstr
+from benchmarks.pyvsc.randlist import VSCRandListSumZero
+from benchmarks.pyvsc.randlist import VSCRandListUnique
 
 
 if __name__ == "__main__":
-    args, extra = parse_args()
-    TEST_LENGTH_MULTIPLIER = args.length_mul
-    # Reconstruct argv
-    argv = [sys.argv[0]] + extra
-    unittest.main(argv=argv)
+    main()

benchmarks/benchmark_utils.py

@@ -0,0 +1,60 @@
+import random
+from typing import Any, Dict
+
+from tests.perf_utils import PERF_DICT
+from tests.testutils import TestBase
+
+
+class BenchmarkTestCase(TestBase):
+    '''
+    Class to be overridden to provide benchmark
+    test cases.
+    '''
+
+    def test_benchmark(self):
+        '''
+        Reusable function to run a fair benchmark between
+        two or more randomizable objects.
+        '''
+        results = {}
+        winner = None
+        best_hz = 0
+        randobjs = self.get_randobjs()
+        for name, randobj in randobjs.items():
+            # Attempt fairness by re-seeding each time
+            random.seed(0)
+            full_name = self.__class__.__name__ + f".{name}"
+            _result, perf_result = self.randomize_and_time(randobj, self.iterations, name=full_name)
+            if perf_result['hz'] > best_hz:
+                winner = name
+                best_hz = perf_result['hz']
+            # Store both total time and Hz in case the test wants to specify
+            # checks on how long should be taken in wall clock time.
+            results[name] = perf_result
+        print(f'{self._testMethodName}: The winner is {winner} with {best_hz:.2f}Hz!')
+        # Print summary of hz delta
+        for name, perf_result in results.items():
+            if name == winner:
+                continue
+            speedup = best_hz / perf_result['hz']
+            print(f'{self._testMethodName}: {winner} was {speedup:.2f}x faster than {name}')
+        self.check_perf(results)
+
+    def check_perf(self, perf_results: Dict[str, PERF_DICT]):
+        '''
+        Implement this to check the expected performance results
+        for a given testcase.
+
+        perf_results: Performance results as captured.
+        '''
+        raise NotImplementedError("Benchmark test cases should check performance")
+
+    def get_randobjs(self) -> Dict[str, Any]:
+        '''
+        Implement this to return the objects to test.
+
+        Returns a dictionary with index as the string name
+        of the random object, value as the object which implements
+        `.randomize()`.
+        '''
+        raise NotImplementedError("Benchmark test cases must implement get_randobjs()")

benchmarks/pyvsc/basic.py

@@ -1,13 +1,11 @@
 # SPDX-License-Identifier: MIT
 # Copyright (c) 2023 Imagination Technologies Ltd. All Rights Reserved
 
-'''
-Basic random object from pyvsc documentation.
-'''
-
 from constrainedrandom import RandObj
 import vsc
 
+from ..benchmark_utils import BenchmarkTestCase
+
 
 @vsc.randobj
 class vsc_basic(object):
@@ -25,11 +23,48 @@ def ab_c(self):
 
 class cr_basic(RandObj):
 
-    def __init__(self, random):
-        super().__init__(random)
+    def __init__(self):
+        super().__init__()
         self.add_rand_var('a', bits=8)
         self.add_rand_var('b', bits=8, order=1)
         self.add_rand_var('c', bits=8)
         self.add_rand_var('d', bits=8)
 
         self.add_constraint(lambda a, b : a < b, ('a', 'b'))
+
+
+class cr_basic_class(RandObj):
+
+    def __init__(self):
+        super().__init__()
+        self.add_rand_var('a', bits=8)
+        self.add_rand_var('b', bits=8, order=1)
+        self.add_rand_var('c', bits=8)
+        self.add_rand_var('d', bits=8)
+        self.add_constraint(self.ab_c, ('a', 'b'))
+
+    def ab_c(self, a, b):
+        return a < b
+
+
+class VSCBasic(BenchmarkTestCase):
+    '''
+    Basic random object from pyvsc documentation.
+    '''
+
+    def get_randobjs(self):
+        return {
+            'vsc': vsc_basic(),
+            'cr': cr_basic(),
+            'cr_class': cr_basic_class(),
+        }
+
+    def check_perf(self, perf_results):
+        self.assertGreater(perf_results['cr']['hz'], perf_results['vsc']['hz'])
+        self.assertGreater(perf_results['cr_class']['hz'], perf_results['vsc']['hz'])
+        # This testcase is typically 40-50x faster, which may vary depending
+        # on machine. Ensure it doesn't fall below 30x.
+        speedup = perf_results['cr']['hz'] / perf_results['vsc']['hz']
+        self.assertGreater(speedup, 30, "Performance has degraded!")
+        speedup = perf_results['cr_class']['hz'] / perf_results['vsc']['hz']
+        self.assertGreater(speedup, 30, "Performance has degraded!")

benchmarks/pyvsc/in_keyword.py

@@ -1,13 +1,11 @@
 # SPDX-License-Identifier: MIT
 # Copyright (c) 2023 Imagination Technologies Ltd. All Rights Reserved
 
-'''
-Random object using 'in' keyword from pyvsc documentation.
-'''
-
 from constrainedrandom import RandObj
 import vsc
 
+from ..benchmark_utils import BenchmarkTestCase
+
 
 @vsc.randobj
 class vsc_in(object):
@@ -34,8 +32,8 @@ class cr_in(RandObj):
     Basic implementation, does thes same thing as vsc_in. No ordering hints.
     '''
 
-    def __init__(self, random):
-        super().__init__(random)
+    def __init__(self):
+        super().__init__()
 
         self.add_rand_var('a', domain=[1,2] + list(range(4,8)))
         self.add_rand_var('b', bits=8, constraints=(lambda b : b != 0,))
@@ -56,8 +54,8 @@ class cr_in_order(RandObj):
     cr_in, but with ordering hints.
     '''
 
-    def __init__(self, random):
-        super().__init__(random)
+    def __init__(self):
+        super().__init__()
 
         self.add_rand_var('a', domain=[1,2] + list(range(4,8)), order=0)
         self.add_rand_var('b', bits=8, constraints=(lambda b : b != 0,), order=2)
@@ -71,3 +69,26 @@ def c_lt_d(c, d):
         def b_in_range(b, c, d):
             return b in range(c, d)
         self.add_constraint(b_in_range, ('b', 'c', 'd'))
+
+
+class VSCIn(BenchmarkTestCase):
+    '''
+    Random object using 'in' keyword from pyvsc documentation.
+    '''
+
+    def get_randobjs(self):
+        return {
+            'vsc': vsc_in(),
+            'cr': cr_in(),
+            'cr_order': cr_in_order(),
+        }
+
+    def check_perf(self, results):
+        self.assertGreater(results['cr']['hz'], results['vsc']['hz'])
+        self.assertGreater(results['cr_order']['hz'], results['vsc']['hz'])
+        # This testcase is typically 13-15x faster, which may vary depending
+        # on machine. Ensure it doesn't fall below 10x.
+        speedup = results['cr']['hz'] / results['vsc']['hz']
+        self.assertGreater(speedup, 10, "Performance has degraded!")
+        speedup = results['cr_order']['hz'] / results['vsc']['hz']
+        self.assertGreater(speedup, 10, "Performance has degraded!")