diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index bebf05eb9..ddeda2262 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -29,6 +29,8 @@ add_library(Testing Attachments/Attachable.swift Attachments/AttachableWrapper.swift Attachments/Attachment.swift + Confirmations/Confirmation.swift + Confirmations/Polling.swift Events/Clock.swift Events/Event.swift Events/Recorder/Event.AdvancedConsoleOutputRecorder.swift @@ -47,7 +49,6 @@ add_library(Testing Expectations/Expectation.swift Expectations/Expectation+Macro.swift Expectations/ExpectationChecking+Macro.swift - Issues/Confirmation.swift Issues/ErrorSnapshot.swift Issues/Issue.swift Issues/Issue+Recording.swift @@ -109,6 +110,7 @@ add_library(Testing Traits/HiddenTrait.swift Traits/IssueHandlingTrait.swift Traits/ParallelizationTrait.swift + Traits/PollingConfigurationTrait.swift Traits/Tags/Tag.Color.swift Traits/Tags/Tag.Color+Loading.swift Traits/Tags/Tag.List.swift diff --git a/Sources/Testing/Issues/Confirmation.swift b/Sources/Testing/Confirmations/Confirmation.swift similarity index 100% rename from Sources/Testing/Issues/Confirmation.swift rename to Sources/Testing/Confirmations/Confirmation.swift diff --git a/Sources/Testing/Confirmations/Polling.swift b/Sources/Testing/Confirmations/Polling.swift new file mode 100644 index 000000000..60f123766 --- /dev/null +++ b/Sources/Testing/Confirmations/Polling.swift @@ -0,0 +1,477 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// Default values for polling confirmations. +@available(_clockAPI, *) +private let _defaultPollingConfiguration = ( + pollingDuration: Duration.seconds(1), + pollingInterval: Duration.milliseconds(1) +) + +/// A type defining when to stop polling. +/// This also determines what happens if the duration elapses during polling. +@_spi(Experimental) +public enum PollingStopCondition: Sendable, Equatable, Codable { + /// Evaluates the expression until the first time it passes + /// If it does not pass once by the time the duration is reached, then a + /// failure will be reported. + case firstPass + + /// Evaluates the expression until the first time it returns fails. + /// If the expression fails, then a failure will be reported. + /// If the expression only passes before the duration is reached, then + /// no failure will be reported. + /// If the expression does not finish evaluating before the duration is + /// reached, then a failure will be reported. + case stopsPassing +} + +/// A type describing an error thrown when polling fails. +@_spi(Experimental) +public struct PollingFailedError: Error, Sendable, Codable { + /// A type describing why polling failed + public enum Reason: Sendable, Codable { + /// The polling failed because it was cancelled using `Task.cancel`. + case cancelled + + /// The polling failed because the stop condition failed. + case stopConditionFailed(PollingStopCondition) + } + + /// A user-specified comment describing this confirmation + public var comment: Comment? + + /// Why polling failed, either cancelled, or because the stop condition failed. + public var reason: Reason + + /// A ``SourceContext`` indicating where and how this confirmation was called + @_spi(ForToolsIntegrationOnly) + public var sourceContext: SourceContext + + /// Initialize an instance of this type with the specified details + /// + /// - Parameters: + /// - comment: A user-specified comment describing this confirmation. + /// Defaults to `nil`. + /// - reason: The reason why polling failed. + /// - sourceContext: A ``SourceContext`` indicating where and how this + /// confirmation was called. + init( + comment: Comment? = nil, + reason: Reason, + sourceContext: SourceContext, + ) { + self.comment = comment + self.reason = reason + self.sourceContext = sourceContext + } +} + +extension PollingFailedError: CustomIssueRepresentable { + func customize(_ issue: consuming Issue) -> Issue { + if let comment { + issue.comments.append(comment) + } + issue.kind = .pollingConfirmationFailed( + reason: reason + ) + issue.sourceContext = sourceContext + return issue + } +} + +/// Poll expression within the duration based on the given stop condition +/// +/// - Parameters: +/// - comment: A user-specified comment describing this confirmation. +/// - stopCondition: When to stop polling. +/// - duration: The expected length of time to continue polling for. +/// This value does not incorporate the time to run `body`, and may not +/// correspond to the wall-clock time that polling lasts for, especially on +/// highly-loaded systems with a lot of tests running. +/// If nil, this uses whatever value is specified under the last +/// ``PollingConfirmationConfigurationTrait`` added to the test or suite +/// with a matching stopCondition. +/// If no such trait has been added, then polling will be attempted for +/// about 1 second before recording an issue. +/// `duration` must be greater than or equal to `interval`. +/// - interval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``PollingConfirmationConfigurationTrait`` added to the test or suite +/// with a matching stopCondition. +/// If no such trait has been added, then polling will wait at least +/// 1 millisecond between polling attempts. +/// `interval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The location in source where the confirmation was called. +/// - body: The function to invoke. The expression is considered to pass if +/// the `body` returns true. Similarly, the expression is considered to fail +/// if `body` returns false. +/// +/// - Throws: A `PollingFailedError` if the `body` does not return true within +/// the polling duration. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(_clockAPI, *) +public func confirmation( + _ comment: Comment? = nil, + until stopCondition: PollingStopCondition, + within duration: Duration? = nil, + pollingEvery interval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async throws { + let poller = Poller( + stopCondition: stopCondition, + duration: stopCondition.duration(with: duration), + interval: stopCondition.interval(with: interval), + comment: comment, + sourceContext: SourceContext( + backtrace: .current(), + sourceLocation: sourceLocation + ) + ) + try await poller.evaluate(isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } +} + +/// Confirm that some expression eventually returns a non-nil value +/// +/// - Parameters: +/// - comment: A user-specified comment describing this confirmation. +/// - stopCondition: When to stop polling. +/// - duration: The expected length of time to continue polling for. +/// This value does not incorporate the time to run `body`, and may not +/// correspond to the wall-clock time that polling lasts for, especially on +/// highly-loaded systems with a lot of tests running. +/// If nil, this uses whatever value is specified under the last +/// ``PollingConfirmationConfigurationTrait`` added to the test or suite +/// with a matching stopCondition. +/// If no such trait has been added, then polling will be attempted for +/// about 1 second before recording an issue. +/// `duration` must be greater than or equal to `interval`. +/// - interval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``PollingConfirmationConfigurationTrait`` added to the test or suite +/// with a matching stopCondition. +/// If no such trait has been added, then polling will wait at least +/// 1 millisecond between polling attempts. +/// `interval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The location in source where the confirmation was called. +/// - body: The function to invoke. The expression is considered to pass if +/// the `body` returns a non-nil value. Similarly, the expression is +/// considered to fail if `body` returns nil. +/// +/// - Throws: A `PollingFailedError` if the `body` does not return true within +/// the polling duration. +/// +/// - Returns: The last non-nil value returned by `body`. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(_clockAPI, *) +@discardableResult +public func confirmation( + _ comment: Comment? = nil, + until stopCondition: PollingStopCondition, + within duration: Duration? = nil, + pollingEvery interval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> sending R? +) async throws -> R { + let poller = Poller( + stopCondition: stopCondition, + duration: stopCondition.duration(with: duration), + interval: stopCondition.interval(with: interval), + comment: comment, + sourceContext: SourceContext( + backtrace: .current(), + sourceLocation: sourceLocation + ) + ) + return try await poller.evaluateOptional(isolation: isolation) { + do { + return try await body() + } catch { + return nil + } + } +} + +/// A helper function to de-duplicate the logic of grabbing configuration from +/// either the passed-in value (if given), the hardcoded default, and the +/// appropriate configuration trait. +/// +/// The provided value, if non-nil is returned. Otherwise, this looks for +/// the last `TraitKind` specified, and if one exists, returns the value +/// as determined by `keyPath`. +/// If the provided value is nil, and no configuration trait has been applied, +/// then this returns the value specified in `default`. +/// +/// - Parameters: +/// - providedValue: The value provided by the test author when calling +/// `confirmPassesEventually` or `confirmAlwaysPasses`. +/// - default: The harded coded default value, as defined in +/// `_defaultPollingConfiguration`. +/// - keyPath: The keyPath mapping from `TraitKind` to the value type. +/// +/// - Returns: The value to use. +private func getValueFromTrait( + providedValue: Value?, + default: Value, + _ keyPath: KeyPath, + where filter: @escaping (TraitKind) -> Bool +) -> Value { + if let providedValue { return providedValue } + guard let test = Test.current else { return `default` } + let possibleTraits = test.traits.compactMap { $0 as? TraitKind } + .filter(filter) + let traitValues = possibleTraits.compactMap { $0[keyPath: keyPath] } + return traitValues.last ?? `default` +} + +extension PollingStopCondition { + /// The result of processing polling. + enum PollingProcessResult { + /// Continue to poll. + case continuePolling + /// Polling succeeded. + case succeeded(R) + /// Polling failed. + case failed + } + /// Process the result of a polled expression and decide whether to continue + /// polling. + /// + /// - Parameters: + /// - expressionResult: The result of the polled expression. + /// - wasLastPollingAttempt: If this was the last time we're attempting to + /// poll. + /// + /// - Returns: A process result. Whether to continue polling, stop because + /// polling failed, or stop because polling succeeded. + fileprivate func process( + expressionResult result: R?, + wasLastPollingAttempt: Bool + ) -> PollingProcessResult { + switch self { + case .firstPass: + if let result { + return .succeeded(result) + } else if wasLastPollingAttempt { + return .failed + } else { + return .continuePolling + } + case .stopsPassing: + if let result { + if wasLastPollingAttempt { + return .succeeded(result) + } else { + return .continuePolling + } + } else { + return .failed + } + } + } + + /// Determine the polling duration to use for the given provided value. + /// Based on ``getValueFromTrait``, this falls back using + /// ``_defaultPollingConfiguration.pollingInterval`` and + /// ``PollingUntilFirstPassConfigurationTrait``. + @available(_clockAPI, *) + fileprivate func duration(with provided: Duration?) -> Duration { + getValueFromTrait( + providedValue: provided, + default: _defaultPollingConfiguration.pollingDuration, + \PollingConfirmationConfigurationTrait.duration, + where: { $0.stopCondition == self } + ) + } + + /// Determine the polling interval to use for the given provided value. + /// Based on ``getValueFromTrait``, this falls back using + /// ``_defaultPollingConfiguration.pollingInterval`` and + /// ``PollingUntilFirstPassConfigurationTrait``. + @available(_clockAPI, *) + fileprivate func interval(with provided: Duration?) -> Duration { + getValueFromTrait( + providedValue: provided, + default: _defaultPollingConfiguration.pollingInterval, + \PollingConfirmationConfigurationTrait.interval, + where: { $0.stopCondition == self } + ) + } +} + +/// A type for managing polling +@available(_clockAPI, *) +private struct Poller { + /// The stop condition to follow + let stopCondition: PollingStopCondition + + /// Approximately how long to poll for + let duration: Duration + + /// The minimum waiting period between polling + let interval: Duration + + /// A user-specified comment describing this confirmation + let comment: Comment? + + /// A ``SourceContext`` indicating where and how this confirmation was called + let sourceContext: SourceContext + + /// Evaluate polling, throwing an error if polling fails. + /// + /// - Parameters: + /// - isolation: The actor isolation to use. + /// - body: The expression to poll. + /// + /// - Throws: A ``PollingFailedError`` if polling doesn't pass. + /// + /// - Returns: Whether or not polling passed. + /// + /// - Side effects: If polling fails (see ``PollingStopCondition``), then + /// this will record an issue. + @discardableResult func evaluate( + isolation: isolated (any Actor)?, + _ body: @escaping () async -> Bool + ) async throws -> Bool { + try await evaluateOptional(isolation: isolation) { + if await body() { + // return any non-nil value. + return true + } else { + return nil + } + } != nil + } + + /// Evaluate polling, throwing an error if polling fails. + /// + /// - Parameters: + /// - isolation: The actor isolation to use. + /// - body: The expression to poll. + /// + /// - Throws: A ``PollingFailedError`` if polling doesn't pass. + /// + /// - Returns: the last non-nil value returned by `body`. + /// + /// - Side effects: If polling fails (see ``PollingStopCondition``), then + /// this will record an issue. + @discardableResult func evaluateOptional( + isolation: isolated (any Actor)?, + _ body: @escaping () async -> sending R? + ) async throws -> R { + precondition(interval > Duration.zero) + precondition(duration >= interval) + + let iterations = Int(exactly: + max(duration.seconds() / interval.seconds(), 1).rounded() + ) ?? Int.max + // if Int(exactly:) returns nil, then that generally means the value is too + // large. In which case, we should fall back to Int.max. + + let failureReason: PollingFailedError.Reason + switch await poll( + iterations: iterations, + isolation: isolation, + expression: body + ) { + case let .succeeded(value): + return value + case .cancelled: + failureReason = .cancelled + case .failed: + failureReason = .stopConditionFailed(stopCondition) + } + throw PollingFailedError( + comment: comment, + reason: failureReason, + sourceContext: sourceContext + ) + } + + /// The result of polling. + private enum PollingResult { + /// Polling was cancelled using `Task.Cancel`. This is treated as a failure. + case cancelled + /// The stop condition failed. + case failed + /// The stop condition passed. + case succeeded(R) + } + + /// This function contains the logic for continuously polling an expression, + /// as well as processing the results of that expression. + /// + /// - Parameters: + /// - iterations: The maximum amount of times to continue polling. + /// - isolation: The actor isolation to use. + /// - expression: An expression to continuously evaluate. + /// + /// - Returns: The most recent value if the polling succeeded, else nil. + private func poll( + iterations: Int, + isolation: isolated (any Actor)?, + expression: @escaping () async -> sending R? + ) async -> PollingResult { + for iteration in 0.. Double { + let secondsComponent = Double(components.seconds) + let attosecondsComponent = Double(components.attoseconds) * 1e-18 + return secondsComponent + attosecondsComponent + } +} diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index beeca101e..859566f15 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -38,6 +38,19 @@ public struct Issue: Sendable { /// confirmed too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: any RangeExpression & Sendable) + /// An issue due to a polling confirmation having failed. + /// + /// - Parameters: + /// - reason: The ``PollingFailedError.Reason`` behind why the polling + /// confirmation failed. + /// + /// This issue can occur when calling ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` + /// or + /// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` + /// whenever the polling fails, as described in ``PollingStopCondition``. + @_spi(Experimental) + case pollingConfirmationFailed(reason: PollingFailedError.Reason) + /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. /// @@ -295,6 +308,8 @@ extension Issue.Kind: CustomStringConvertible { } } return "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(String(describingForTest: expected)) time(s)" + case .pollingConfirmationFailed: + return "Polling confirmation failed" case let .errorCaught(error): return "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -434,6 +449,15 @@ extension Issue.Kind { /// too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: Int) + /// An issue due to a polling confirmation having failed. + /// + /// This issue can occur when calling ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` + /// or + /// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` + /// whenever the polling fails, as described in ``PollingStopCondition``. + @_spi(Experimental) + case pollingConfirmationFailed + /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. /// @@ -477,6 +501,8 @@ extension Issue.Kind { .expectationFailed(Expectation.Snapshot(snapshotting: expectation)) case .confirmationMiscounted: .unconditional + case .pollingConfirmationFailed: + .pollingConfirmationFailed case let .errorCaught(error), let .valueAttachmentFailed(error): .errorCaught(ErrorSnapshot(snapshotting: error)) case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -495,6 +521,7 @@ extension Issue.Kind { case unconditional case expectationFailed case confirmationMiscounted + case pollingConfirmationFailed case errorCaught case timeLimitExceeded case knownIssueNotRecorded @@ -567,6 +594,8 @@ extension Issue.Kind { forKey: .confirmationMiscounted) try confirmationMiscountedContainer.encode(actual, forKey: .actual) try confirmationMiscountedContainer.encode(expected, forKey: .expected) + case .pollingConfirmationFailed: + try container.encode(true, forKey: .pollingConfirmationFailed) case let .errorCaught(error): var errorCaughtContainer = container.nestedContainer(keyedBy: _CodingKeys._ErrorCaughtKeys.self, forKey: .errorCaught) try errorCaughtContainer.encode(error, forKey: .error) @@ -622,6 +651,8 @@ extension Issue.Kind.Snapshot: CustomStringConvertible { } case let .confirmationMiscounted(actual: actual, expected: expected): "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(expected.counting("time"))" + case .pollingConfirmationFailed: + "Polling confirmation failed" case let .errorCaught(error): "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): diff --git a/Sources/Testing/Traits/PollingConfigurationTrait.swift b/Sources/Testing/Traits/PollingConfigurationTrait.swift new file mode 100644 index 000000000..48ca5baa3 --- /dev/null +++ b/Sources/Testing/Traits/PollingConfigurationTrait.swift @@ -0,0 +1,79 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A trait to provide a default polling configuration to all usages of +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` +/// and +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` +/// within a test or suite using the specified stop condition. +/// +/// To add this trait to a test, use the ``Trait/pollingConfirmationDefaults`` +/// function. +@_spi(Experimental) +@available(_clockAPI, *) +public struct PollingConfirmationConfigurationTrait: TestTrait, SuiteTrait { + /// The stop condition to this configuration is valid for + public var stopCondition: PollingStopCondition + + /// How long to continue polling for. If nil, this will fall back to the next + /// inner-most `PollingConfirmationConfigurationTrait.duration` value for this + /// stop condition. + /// If no non-nil values are found, then it will use 1 second. + public var duration: Duration? + + /// The minimum amount of time to wait between polling attempts. If nil, this + /// will fall back to earlier `PollingConfirmationConfigurationTrait.interval` + /// values for this stop condition. If no non-nil values are found, then it + /// will use 1 millisecond. + public var interval: Duration? + + public var isRecursive: Bool { true } + + public init( + stopCondition: PollingStopCondition, + duration: Duration?, + interval: Duration? + ) { + self.stopCondition = stopCondition + self.duration = duration + self.interval = interval + } +} + +@_spi(Experimental) +@available(_clockAPI, *) +extension Trait where Self == PollingConfirmationConfigurationTrait { + /// Specifies defaults for polling confirmations in the test or suite. + /// + /// - Parameters: + /// - stopCondition: The `PollingStopCondition` this trait applies to. + /// - duration: The expected length of time to continue polling for. + /// This value may not correspond to the wall-clock time that polling + /// lasts for, especially on highly-loaded systems with a lot of tests + /// running. + /// If nil, polling will be attempted for approximately 1 second. + /// If specified, `duration` must be greater than or equal to `interval`. + /// - interval: The minimum amount of time to wait between polling + /// attempts. + /// If nil, polling will wait at least 1 millisecond between polling + /// attempts. + /// If specified, `interval` must be greater than 0. + public static func pollingConfirmationDefaults( + until stopCondition: PollingStopCondition, + within duration: Duration? = nil, + pollingEvery interval: Duration? = nil + ) -> Self { + PollingConfirmationConfigurationTrait( + stopCondition: stopCondition, + duration: duration, + interval: interval + ) + } +} diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift new file mode 100644 index 000000000..4e8697b10 --- /dev/null +++ b/Tests/TestingTests/PollingTests.swift @@ -0,0 +1,567 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +struct `Polling Confirmation Tests` { + struct `with PollingStopCondition.firstPass` { + let stop = PollingStopCondition.firstPass + + @available(_clockAPI, *) + @Test func `simple passing expressions`() async throws { + try await confirmation(until: stop) { true } + + let value = try await confirmation(until: stop) { 1 } + + #expect(value == 1) + } + + @available(_clockAPI, *) + @Test func `simple failing expressions`() async throws { + var issues = await runTest { + try await confirmation(until: stop) { false } + } + issues += await runTest { + _ = try await confirmation(until: stop) { Optional.none } + } + #expect(issues.count == 2) + #expect(issues.allSatisfy { + if case .pollingConfirmationFailed = $0.kind { + return true + } else { + return false + } + }) + } + + @available(_clockAPI, *) + @Test + func `returning false in a closure returning Optional is considered a pass`() async throws { + try await confirmation(until: stop) { () -> Bool? in + return false + } + } + + @available(_clockAPI, *) + @Test + func `When the value changes from false to true during execution`() async throws { + let incrementor = Incrementor() + + try await confirmation(until: stop) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + } + + @available(_clockAPI, *) + @Test func `Thrown errors are treated as returning false`() async throws { + let issues = await runTest { + try await confirmation(until: stop) { + throw PollingTestSampleError.ohNo + } + } + #expect(issues.count == 1) + } + + @available(_clockAPI, *) + @Test + func `Calculates how many times to poll based on the duration & interval`() async { + let incrementor = Incrementor() + _ = await runTest { + // this test will intentionally fail. + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() == 0 + } + } + #expect(await incrementor.count == 1000) + } + + @Suite( + .pollingConfirmationDefaults( + until: .firstPass, + within: .milliseconds(100) + ) + ) + struct `Configuration traits` { + let stop = PollingStopCondition.firstPass + + @available(_clockAPI, *) + @Test + func `When no test or callsite configuration provided, uses the suite configuration`() async { + let incrementor = Incrementor() + var test = Test { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 100) + } + + @available(_clockAPI, *) + @Test( + .pollingConfirmationDefaults( + until: .stopsPassing, + within: .milliseconds( + 500 + ) + ) + ) + func `Ignore trait configurations that don't match the stop condition`() async { + let incrementor = Incrementor() + var test = Test { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 100) + } + + @available(_clockAPI, *) + @Test( + .pollingConfirmationDefaults( + until: .firstPass, + within: .milliseconds(10) + ) + ) + func `When test configuration provided, uses the test configuration`() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 10) + } + + @available(_clockAPI, *) + @Test( + .pollingConfirmationDefaults( + until: .firstPass, + within: .milliseconds(10) + ) + ) + func `When callsite configuration provided, uses that`() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + try await confirmation( + until: stop, + within: .milliseconds(50), + pollingEvery: .milliseconds(1) + ) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 50) + } + + @available(_clockAPI, *) + @Test func `Allows duration to be equal to interval`() async throws { + let incrementor = Incrementor() + try await confirmation( + until: stop, + within: .milliseconds(100), + pollingEvery: .milliseconds(100) + ) { + _ = await incrementor.increment() + return true + } + + #expect(await incrementor.count == 1) + } + +#if !SWT_NO_EXIT_TESTS + @available(_clockAPI, *) + @Test + func `Requires duration be greater than or equal to interval`() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .firstPass, + within: .seconds(1), + pollingEvery: .milliseconds(1100) + ) { true } + } + } + + @available(_clockAPI, *) + @Test func `Requires interval be greater than 0`() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .firstPass, + pollingEvery: .seconds(0) + ) { true } + } + } + + @available(_clockAPI, *) + @Test func `Handles extremely large polling iterations`() async throws { + await #expect(processExitsWith: .success) { + try await confirmation( + until: .firstPass, + within: .seconds(Int.max), + pollingEvery: .nanoseconds(1) + ) { true } + } + } +#endif + } + } + + struct `with PollingStopCondition.stopsPassing` { + let stop = PollingStopCondition.stopsPassing + @available(_clockAPI, *) + @Test func `Simple passing expressions`() async throws { + try await confirmation(until: stop) { true } + let value = try await confirmation(until: stop) { 1 } + + #expect(value == 1) + } + + @available(_clockAPI, *) + @Test func `Simple failing expressions`() async { + var issues = await runTest { + try await confirmation(until: stop) { false } + } + issues += await runTest { + _ = try await confirmation(until: stop) { Optional.none } + } + #expect(issues.count == 2) + #expect(issues.allSatisfy { + if case .pollingConfirmationFailed = $0.kind { + return true + } else { + return false + } + }) + } + + @available(_clockAPI, *) + @Test + func `returning false in a closure returning Optional is considered a pass`() async throws { + try await confirmation(until: stop) { () -> Bool? in + return false + } + } + + @available(_clockAPI, *) + @Test func `if the closure starts off as true, but becomes false`() async { + let incrementor = Incrementor() + let issues = await runTest { + try await confirmation(until: stop) { + await incrementor.increment() == 2 + // this will pass only on the first invocation + // This checks that we fail the test if it starts failing later + // during polling + } + } + #expect(issues.count == 1) + } + + @available(_clockAPI, *) + @Test func `if the closure continues to pass`() async throws { + let incrementor = Incrementor() + + try await confirmation(until: stop) { + _ = await incrementor.increment() + return true + } + + #expect(await incrementor.count > 1) + } + + @available(_clockAPI, *) + @Test func `Thrown errors will automatically exit & fail`() async { + let issues = await runTest { + try await confirmation(until: stop) { + throw PollingTestSampleError.ohNo + } + } + #expect(issues.count == 1) + } + + @available(_clockAPI, *) + @Test + func `Calculates how many times to poll based on the duration & interval`() async throws { + let incrementor = Incrementor() + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 1000) + } + + @Suite( + .pollingConfirmationDefaults( + until: .stopsPassing, + within: .milliseconds(100) + ) + ) + struct `Configuration traits` { + let stop = PollingStopCondition.stopsPassing + + @available(_clockAPI, *) + @Test + func `"When no test/callsite configuration, it uses the suite configuration"`() async throws { + let incrementor = Incrementor() + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() != 0 + } + let count = await incrementor.count + #expect(count == 100) + } + + @available(_clockAPI, *) + @Test( + .pollingConfirmationDefaults( + until: .firstPass, + within: .milliseconds( + 500 + ) + ) + ) + func `Ignore trait configurations that don't match the stop condition`() async throws { + let incrementor = Incrementor() + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() != 0 + } + let count = await incrementor.count + #expect(count == 100) + } + + @available(_clockAPI, *) + @Test( + .pollingConfirmationDefaults( + until: .stopsPassing, + within: .milliseconds(10) + ) + ) + func `When test configuration provided, uses the test configuration`() async throws { + let incrementor = Incrementor() + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() != 0 + } + let count = await incrementor.count + #expect(await count == 10) + } + + @available(_clockAPI, *) + @Test( + .pollingConfirmationDefaults( + until: .stopsPassing, + within: .milliseconds(10) + ) + ) + func `When callsite configuration provided, uses that`() async throws { + let incrementor = Incrementor() + try await confirmation( + until: stop, + within: .milliseconds(50), + pollingEvery: .milliseconds(1) + ) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 50) + } + + @available(_clockAPI, *) + @Test func `Allows duration to be equal to interval`() async throws { + let incrementor = Incrementor() + try await confirmation( + until: stop, + within: .milliseconds(100), + pollingEvery: .milliseconds(100) + ) { + _ = await incrementor.increment() + return true + } + + #expect(await incrementor.count == 1) + } + +#if !SWT_NO_EXIT_TESTS + @available(_clockAPI, *) + @Test + func `Requires duration be greater than or equal to interval`() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .stopsPassing, + within: .seconds(1), + pollingEvery: .milliseconds(1100) + ) { true } + } + } + + @available(_clockAPI, *) + @Test func `Requires duration be greater than 0`() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .stopsPassing, + within: .seconds(0) + ) { true } + } + } + + @available(_clockAPI, *) + @Test func `Requires interval be greater than 0`() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .stopsPassing, + pollingEvery: .seconds(0) + ) { true } + } + } +#endif + } + } + + @Suite(.disabled("time-sensitive")) + struct `Duration Tests` { + struct `with PollingStopCondition.firstPass` { + let stop = PollingStopCondition.firstPass + let delta = Duration.milliseconds(100) + + @available(_clockAPI, *) + @Test func `Simple passing expressions`() async throws { + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { true } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @available(_clockAPI, *) + @Test func `Simple failing expressions`() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + try await confirmation(until: stop) { false } + } + #expect(issues.count == 1) + } + #expect(duration.isCloseTo(other: .seconds(2), within: delta)) + } + + @available(_clockAPI, *) + @Test + func `When the value changes from false to true during execution`() async throws { + let incrementor = Incrementor() + + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @available(_clockAPI, *) + @Test func `Doesn't wait after the last iteration`() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + try await confirmation( + until: stop, + within: .seconds(10), + pollingEvery: .seconds(1) // Wait a long time to handle jitter. + ) { false } + } + #expect(issues.count == 1) + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } + } + + struct `with PollingStopCondition.stopsPassing` { + let stop = PollingStopCondition.stopsPassing + let delta = Duration.milliseconds(100) + + @available(_clockAPI, *) + @Test func `Simple passing expressions`() async throws { + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { true } + } + #expect(duration.isCloseTo(other: .seconds(2), within: delta)) + } + + @available(_clockAPI, *) + @Test func `Simple failing expressions`() async { + let duration = await Test.Clock().measure { + _ = await runTest { + try await confirmation(until: stop) { false } + } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @available(_clockAPI, *) + @Test + func `Doesn't wait after the last iteration`() async throws { + let duration = try await Test.Clock().measure { + try await confirmation( + until: stop, + within: .seconds(10), + pollingEvery: .seconds(1) // Wait a long time to handle jitter. + ) { true } + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } + } + } +} + +private enum PollingTestSampleError: Error { + case ohNo + case secondCase +} + +@available(_clockAPI, *) +extension DurationProtocol { + fileprivate func isCloseTo(other: Self, within delta: Self) -> Bool { + var distance = self - other + if (distance < Self.zero) { + distance *= -1 + } + return distance <= delta + } +} + +private actor Incrementor { + var count = 0 + func increment() -> Int { + count += 1 + return count + } +} diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 05bb05dc8..49f4c8413 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -94,6 +94,55 @@ func runTestFunction(named name: String, in containingType: Any.Type, configurat await runner.run() } +/// Create a ``Test`` instance for the expression and run it, returning any +/// issues recorded. +/// +/// - Parameters: +/// - testFunction: The test expression to run +/// +/// - Returns: The list of issues recorded. +@discardableResult +func runTest( + testFunction: @escaping @Sendable () async throws -> Void +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + issues.withLock { + $0.append(issue) + } + } + } + await Test(testFunction: testFunction).run(configuration: configuration) + return issues.rawValue +} + +/// Runs the passed-in `Test`, returning any issues recorded. +/// +/// - Parameters: +/// - test: The test to run +/// +/// - Returns: The list of issues recorded. +@discardableResult +func runTest( + test: Test +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + issues.withLock { + $0.append(issue) + } + } + } + await test.run(configuration: configuration) + return issues.rawValue +} + extension Runner { /// Initialize an instance of this type that runs the free test function /// named `testName` in the module specified in `fileID`.