diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index f4f1a751c..6163e7bd1 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -210,6 +210,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--parallel` or `--no-parallel` argument. public var parallel: Bool? + /// The maximum number of test tasks to run in parallel. + public var experimentalMaximumParallelizationWidth: Int? + /// The value of the `--symbolicate-backtraces` argument. public var symbolicateBacktraces: String? @@ -336,6 +339,7 @@ extension __CommandLineArguments_v0: Codable { enum CodingKeys: String, CodingKey { case listTests case parallel + case experimentalMaximumParallelizationWidth case symbolicateBacktraces case verbose case veryVerbose @@ -485,6 +489,10 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum if args.contains("--no-parallel") { result.parallel = false } + if let maximumParallelizationWidth = args.argumentValue(forLabel: "--experimental-maximum-parallelization-width").flatMap(Int.init) { + // TODO: decide if we want to repurpose --num-workers for this use case? + result.experimentalMaximumParallelizationWidth = maximumParallelizationWidth + } // Whether or not to symbolicate backtraces in the event stream. if let symbolicateBacktraces = args.argumentValue(forLabel: "--symbolicate-backtraces") { @@ -545,7 +553,22 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr var configuration = Configuration() // Parallelization (on by default) - configuration.isParallelizationEnabled = args.parallel ?? true + if let parallel = args.parallel, !parallel { + configuration.isParallelizationEnabled = parallel + } else { + var maximumParallelizationWidth = args.experimentalMaximumParallelizationWidth + if maximumParallelizationWidth == nil && Test.current == nil { + // Don't check the environment variable when a current test is set (which + // presumably means we're running our own unit tests). + maximumParallelizationWidth = Environment.variable(named: "SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH").flatMap(Int.init) + } + if let maximumParallelizationWidth { + if maximumParallelizationWidth < 1 { + throw _EntryPointError.invalidArgument("--experimental-maximum-parallelization-width", value: String(describing: maximumParallelizationWidth)) + } + configuration.maximumParallelizationWidth = maximumParallelizationWidth + } + } // Whether or not to symbolicate backtraces in the event stream. if let symbolicateBacktraces = args.symbolicateBacktraces { diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index b7836e3e1..b60688731 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -90,6 +90,7 @@ add_library(Testing Support/Graph.swift Support/JSON.swift Support/Locked.swift + Support/Serializer.swift Support/VersionNumber.swift Support/Versions.swift Discovery+Macro.swift diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 12b8827de..e0fe009ba 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -8,6 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +private import _TestingInternals + /// A type containing settings for preparing and running tests. @_spi(ForToolsIntegrationOnly) public struct Configuration: Sendable { @@ -18,7 +20,33 @@ public struct Configuration: Sendable { // MARK: - Parallelization /// Whether or not to parallelize the execution of tests and test cases. - public var isParallelizationEnabled: Bool = true + /// + /// - Note: Setting the value of this property implicitly sets the value of + /// the experimental ``maximumParallelizationWidth`` property. + public var isParallelizationEnabled: Bool { + get { + maximumParallelizationWidth > 1 + } + set { + maximumParallelizationWidth = newValue ? defaultParallelizationWidth : 1 + } + } + + /// The maximum width of parallelization. + /// + /// The value of this property determines how many tests (or rather, test + /// cases) will run in parallel. + /// + /// @Comment { + /// The default value of this property is equal to twice the number of CPU + /// cores reported by the operating system, or `Int.max` if that value is + /// not available. + /// } + /// + /// - Note: Setting the value of this property implicitly sets the value of + /// the ``isParallelizationEnabled`` property. + @_spi(Experimental) + public var maximumParallelizationWidth: Int = defaultParallelizationWidth /// How to symbolicate backtraces captured during a test run. /// diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 8adf7e088..a6a76189e 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -66,6 +66,20 @@ extension Runner { .current ?? .init() } + /// Context to apply to a test run. + /// + /// Instances of this type are passed directly to the various functions in + /// this file and represent context for the run itself. As such, they are not + /// task-local nor are they meant to change as the test run progresses. + /// + /// This type is distinct from ``Configuration`` which _can_ change on a + /// per-test basis. If you find yourself wanting to modify a property of this + /// type at runtime, it may be better-suited for ``Configuration`` instead. + private struct _Context: Sendable { + /// A serializer used to reduce parallelism among test cases. + var testCaseSerializer: Serializer? + } + /// Apply the custom scope for any test scope providers of the traits /// associated with a specified test by calling their /// ``TestScoping/provideScope(for:testCase:performing:)`` function. @@ -179,6 +193,7 @@ extension Runner { /// /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, is to be run. + /// - context: Context for the test run. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. @@ -193,7 +208,7 @@ extension Runner { /// ## See Also /// /// - ``Runner/run()`` - private static func _runStep(atRootOf stepGraph: Graph) async throws { + private static func _runStep(atRootOf stepGraph: Graph, context: _Context) async throws { // Whether to send a `.testEnded` event at the end of running this step. // Some steps' actions may not require a final event to be sent — for // example, a skip event only sends `.testSkipped`. @@ -250,18 +265,18 @@ extension Runner { try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { - await _runTestCases(testCases, within: step) + await _runTestCases(testCases, within: step, context: context) } // Run the children of this test (i.e. the tests in this suite.) - try await _runChildren(of: stepGraph) + try await _runChildren(of: stepGraph, context: context) } } } } else { // There is no test at this node in the graph, so just skip down to the // child nodes. - try await _runChildren(of: stepGraph) + try await _runChildren(of: stepGraph, context: context) } } @@ -286,10 +301,11 @@ extension Runner { /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, will be used to /// find children to run. + /// - context: Context for the test run. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. - private static func _runChildren(of stepGraph: Graph) async throws { + private static func _runChildren(of stepGraph: Graph, context: _Context) async throws { let childGraphs = if _configuration.isParallelizationEnabled { // Explicitly shuffle the steps to help detect accidental dependencies // between tests due to their ordering. @@ -331,7 +347,7 @@ extension Runner { // Run the child nodes. try await _forEach(in: childGraphs.lazy.map(\.value), namingTasksWith: taskNamer) { childGraph in - try await _runStep(atRootOf: childGraph) + try await _runStep(atRootOf: childGraph, context: context) } } @@ -340,12 +356,15 @@ extension Runner { /// - Parameters: /// - testCases: The test cases to be run. /// - step: The runner plan step associated with this test case. + /// - context: Context for the test run. /// /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. - private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async { + private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step, context: _Context) async { + let configuration = _configuration + // Apply the configuration's test case filter. - let testCaseFilter = _configuration.testCaseFilter + let testCaseFilter = configuration.testCaseFilter let testCases = testCases.lazy.filter { testCase in testCaseFilter(testCase, step.test) } @@ -359,7 +378,13 @@ extension Runner { } await _forEach(in: testCases.enumerated(), namingTasksWith: taskNamer) { _, testCase in - await _runTestCase(testCase, within: step) + if let testCaseSerializer = context.testCaseSerializer { + // Note that if .serialized is applied to an inner scope, we still use + // this serializer (if set) so that we don't overcommit. + await testCaseSerializer.run { await _runTestCase(testCase, within: step, context: context) } + } else { + await _runTestCase(testCase, within: step, context: context) + } } } @@ -368,10 +393,11 @@ extension Runner { /// - Parameters: /// - testCase: The test case to run. /// - step: The runner plan step associated with this test case. + /// - context: Context for the test run. /// /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. - private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async { + private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: _Context) async { let configuration = _configuration Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) @@ -431,6 +457,21 @@ extension Runner { eventHandler(event, context) } + // Context to pass into the test run. We intentionally don't pass the Runner + // itself (implicitly as `self` nor as an argument) because we don't want to + // accidentally depend on e.g. the `configuration` property rather than the + // current configuration. + let context: _Context = { + var context = _Context() + + let maximumParallelizationWidth = runner.configuration.maximumParallelizationWidth + if maximumParallelizationWidth > 1 && maximumParallelizationWidth < .max { + context.testCaseSerializer = Serializer(maximumWidth: runner.configuration.maximumParallelizationWidth) + } + + return context + }() + await Configuration.withCurrent(runner.configuration) { // Post an event for every test in the test plan being run. These events // are turned into JSON objects if JSON output is enabled. @@ -457,7 +498,7 @@ extension Runner { taskAction = "running iteration #\(iterationIndex + 1)" } _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: taskAction)) { - try? await _runStep(atRootOf: runner.plan.stepGraph) + try? await _runStep(atRootOf: runner.plan.stepGraph, context: context) } await taskGroup.waitForAll() } diff --git a/Sources/Testing/Support/Serializer.swift b/Sources/Testing/Support/Serializer.swift new file mode 100644 index 000000000..96adfca7c --- /dev/null +++ b/Sources/Testing/Support/Serializer.swift @@ -0,0 +1,93 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +/// The number of CPU cores on the current system, or `nil` if that +/// information is not available. +var cpuCoreCount: Int? { +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) + return Int(sysconf(Int32(_SC_NPROCESSORS_CONF))) +#elseif os(Windows) + var siInfo = SYSTEM_INFO() + GetSystemInfo(&siInfo) + return Int(siInfo.dwNumberOfProcessors) +#elseif os(WASI) + return 1 +#else +#warning("Platform-specific implementation missing: CPU core count unavailable") + return nil +#endif +} + +/// The default parallelization width when parallelized testing is enabled. +var defaultParallelizationWidth: Int { + // cpuCoreCount.map { max(1, $0) * 2 } ?? .max + .max +} + +/// A type whose instances can run a series of work items in strict order. +/// +/// When a work item is scheduled on an instance of this type, it runs after any +/// previously-scheduled work items. If it suspends, subsequently-scheduled work +/// items do not start running; they must wait until the suspended work item +/// either returns or throws an error. +/// +/// This type is not part of the public interface of the testing library. +final actor Serializer { + /// The maximum number of work items that may run concurrently. + nonisolated let maximumWidth: Int + + /// The number of scheduled work items, including any currently running. + private var _currentWidth = 0 + + /// Continuations for any scheduled work items that haven't started yet. + private var _continuations = [CheckedContinuation]() + + init(maximumWidth: Int = 1) { + precondition(maximumWidth >= 1, "Invalid serializer width \(maximumWidth).") + self.maximumWidth = maximumWidth + } + + /// Run a work item serially after any previously-scheduled work items. + /// + /// - Parameters: + /// - workItem: A closure to run. + /// + /// - Returns: Whatever is returned from `workItem`. + /// + /// - Throws: Whatever is thrown by `workItem`. + func run(_ workItem: @isolated(any) @Sendable () async throws -> R) async rethrows -> R where R: Sendable { + _currentWidth += 1 + defer { + // Resume the next scheduled closure. + if !_continuations.isEmpty { + let continuation = _continuations.removeFirst() + continuation.resume() + } + + _currentWidth -= 1 + } + + await withCheckedContinuation { continuation in + if _currentWidth <= maximumWidth { + // Nothing else was scheduled, so we can resume immediately. + continuation.resume() + } else { + // Something was scheduled, so add the continuation to the + // list. When it resumes, we can run. + _continuations.append(continuation) + } + } + + return try await workItem() + } +} + diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 4668fbb25..2d7001e8f 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -59,6 +59,25 @@ struct SwiftPMTests { #expect(!configuration.isParallelizationEnabled) } + @Test("--experimental-maximum-parallelization-width argument") + func maximumParallelizationWidth() throws { + var configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "12345"]) + #expect(configuration.isParallelizationEnabled) + #expect(configuration.maximumParallelizationWidth == 12345) + + configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "1"]) + #expect(!configuration.isParallelizationEnabled) + #expect(configuration.maximumParallelizationWidth == 1) + + configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "\(Int.max)"]) + #expect(configuration.isParallelizationEnabled) + #expect(configuration.maximumParallelizationWidth == .max) + + #expect(throws: (any Error).self) { + _ = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "0"]) + } + } + @Test("--symbolicate-backtraces argument", arguments: [ (String?.none, Backtrace.SymbolicationMode?.none),