Skip to content

Commit df7a48a

Browse files
committed
Fix stream pipeline chaining
1 parent c2d0925 commit df7a48a

File tree

7 files changed

+69
-44
lines changed

7 files changed

+69
-44
lines changed

TensorStack.Common/Video/VideoStream.cs

Lines changed: 0 additions & 33 deletions
This file was deleted.
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
// Copyright (c) TensorStack. All rights reserved.
22
// Licensed under the Apache 2.0 License.
3+
using System.Collections.Generic;
34
using TensorStack.Common.Video;
45

56
namespace TensorStack.Extractors.Common
67
{
78
public record ExtractorStreamOptions : ExtractorOptions
89
{
9-
public VideoStream Stream { get; }
10+
public IAsyncEnumerable<VideoFrame> Stream { get; }
1011
}
1112
}

TensorStack.Upscaler/Common/UpscaleOptions.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright (c) TensorStack. All rights reserved.
22
// Licensed under the Apache 2.0 License.
3+
using System.Collections.Generic;
34
using TensorStack.Common;
45
using TensorStack.Common.Pipeline;
56
using TensorStack.Common.Tensor;
@@ -64,6 +65,6 @@ public sealed record UpscaleStreamOptions : UpscaleOptions
6465
/// <summary>
6566
/// Gets the stream input.
6667
/// </summary>
67-
public VideoStream Stream { get; init; }
68+
public IAsyncEnumerable<VideoFrame> Stream { get; init; }
6869
}
6970
}

TensorStack.Upscaler/README.md

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ You can use the TensorStack.Audio package to restore audio from the source video
153153
```
154154
---
155155

156+
156157
## Tiling Support
157158
Tiling allows images and video frames to be processed in smaller sections (tiles) instead of all at once. This helps reduce memory usage and can improve performance when working with very large images or high-resolution videos.
158159

@@ -182,4 +183,57 @@ Here is a list of some known and tested models compatible with `TensorStack.Upsc
182183
- [Xenova/swin2SR-classical-sr-x2-64](https://huggingface.co/Xenova/swin2SR-classical-sr-x2-64)
183184
- [Xenova/swin2SR-classical-sr-x4-64](https://huggingface.co/Xenova/swin2SR-classical-sr-x4-64)
184185
- [Neus/GFPGANv1.4](https://huggingface.co/Neus/GFPGANv1.4)
185-
- [TensorStack/Upscale-amuse](https://huggingface.co/TensorStack/Upscale-amuse)
186+
- [TensorStack/Upscale-amuse](https://huggingface.co/TensorStack/Upscale-amuse)
187+
188+
---
189+
190+
## Combining Pipelines
191+
TensorStack supports chaining multiple pipelines together using `IAsyncEnumerable` streams.
192+
This allows complex video processing workflows—such as upscaling and frame interpolation—to be executed efficiently in a **single pass** without storing intermediate results in memory.
193+
194+
In the following example, the video is upscaled by **** and its frame rate is increased by ****:
195+
```csharp
196+
[nuget: TensorStack.Upscaler]
197+
[nuget: TensorStack.Providers.DML]
198+
[nuget: TensorStack.Video.Windows]
199+
200+
// Create Provider
201+
var provider = Provider.GetProvider();
202+
203+
// Upscaler Config
204+
var upscaleConfig = new UpscalerConfig
205+
{
206+
ScaleFactor = 4,
207+
ExecutionProvider = provider,
208+
Normalization = Normalization.ZeroToOne,
209+
Path = @"M:\Models\RealESR-General-4x\model.onnx"
210+
}
211+
212+
// Create Pipelines
213+
using (var upscalePipeline = UpscalePipeline.Create(upscaleConfig))
214+
using (var interpolationPipeline = InterpolationPipeline.Create(provider))
215+
{
216+
// Read Stream [512 x 512 @ 8fps]
217+
var videoInput = new VideoInputStream("Input.mp4");
218+
var videoStream = videoInput.GetAsync();
219+
220+
// Upscale Stream
221+
videoStream = upscalePipeline.RunAsync(new UpscaleStreamOptions
222+
{
223+
Stream = videoStream
224+
});
225+
226+
// Interpolate Stream
227+
videoStream = interpolationPipeline.RunAsync(new InterpolationStreamOptions
228+
{
229+
Multiplier = 3,
230+
Stream = videoStream,
231+
FrameRate = videoInput.FrameRate,
232+
FrameCount = videoInput.FrameCount
233+
});
234+
235+
// Save Steam [2048 x 2048 @ 24fps]
236+
await videoStream.SaveAync("Output.mp4");
237+
}
238+
239+
```

TensorStack.Video.Windows/VideoInputStream.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,9 @@ private VideoInputStream(VideoInfo videoInfo)
3939
/// <param name="frameRateOverride">The frame rate.</param>
4040
/// <param name="cancellationToken">The cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
4141
/// <returns>IAsyncEnumerable&lt;ImageFrame&gt;.</returns>
42-
public VideoStream GetAsync(int? widthOverride = default, int? heightOverride = default, float? frameRateOverride = default, ResizeMode resizeMode = ResizeMode.Stretch, CancellationToken cancellationToken = default)
42+
public IAsyncEnumerable<VideoFrame> GetAsync(int? widthOverride = default, int? heightOverride = default, float? frameRateOverride = default, ResizeMode resizeMode = ResizeMode.Stretch, CancellationToken cancellationToken = default)
4343
{
44-
var stream = VideoManager.ReadStreamAsync(SourceFile, frameRateOverride, widthOverride, heightOverride, resizeMode, cancellationToken);
45-
return new VideoStream(stream, FrameCount, frameRateOverride ?? FrameRate, widthOverride ?? Width, heightOverride ?? Height);
44+
return VideoManager.ReadStreamAsync(SourceFile, frameRateOverride, widthOverride, heightOverride, resizeMode, cancellationToken);
4645
}
4746

4847

TensorStack.Video/Common/InterpolationOptions.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using TensorStack.Common.Pipeline;
1+
using System.Collections.Generic;
2+
using TensorStack.Common.Pipeline;
23
using TensorStack.Common.Tensor;
34
using TensorStack.Common.Video;
45

@@ -29,6 +30,8 @@ public sealed record InterpolationVideoOptions : InterpolationOptions
2930
/// </summary>
3031
public sealed record InterpolationStreamOptions : InterpolationOptions
3132
{
32-
public VideoStream Stream { get; init; }
33+
public int FrameCount { get; init; }
34+
public float FrameRate { get; init; }
35+
public IAsyncEnumerable<VideoFrame> Stream { get; init; }
3336
}
3437
}

TensorStack.Video/Pipelines/InterpolationPipeline.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,11 +102,11 @@ public async Task<VideoTensor> RunAsync(InterpolationVideoOptions options, IProg
102102
public async IAsyncEnumerable<VideoFrame> RunAsync(InterpolationStreamOptions options, IProgress<RunProgress> progressCallback = default, [EnumeratorCancellation] CancellationToken cancellationToken = default)
103103
{
104104
var frameIndex = 0;
105-
var totalFrames = options.Stream.FrameCount * options.Multiplier;
106-
var newFrameRate = options.Stream.FrameRate * options.Multiplier;
105+
var totalFrames = options.FrameCount * options.Multiplier;
106+
var newFrameRate = options.FrameRate * options.Multiplier;
107107

108108
var previousFrame = default(ImageTensor);
109-
var extraFramePositions = GetFlowEstimationKeyFrames(options.Stream.FrameCount, options.Multiplier);
109+
var extraFramePositions = GetFlowEstimationKeyFrames(options.FrameCount, options.Multiplier);
110110
await foreach (var frame in options.Stream)
111111
{
112112
var currentFrame = frame.Frame.CloneAs();

0 commit comments

Comments
 (0)