Video Interpolation pipeline

Author: saddam213

TensorStack.Common/ModelSession.cs

@@ -237,10 +237,12 @@ protected virtual void Dispose(bool disposing)
 
             if (disposing)
             {
-                _allocator?.Dispose();
-                _options?.Dispose();
                 _session?.Dispose();
+                _options?.Dispose();
+                _allocator?.Dispose();
                 _session = null;
+                _options = null;
+                _allocator = null;
             }
 
             disposed = true;

TensorStack.Common/Video/VideoStream.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.Threading;
+
+namespace TensorStack.Common.Video
+{
+    public class VideoStream : IAsyncEnumerable<VideoFrame>
+    {
+        private readonly IAsyncEnumerable<VideoFrame> _stream;
+        private readonly int _width;
+        private readonly int _height;
+        private readonly float _frameRate;
+        private readonly int _frameCount;
+
+        public VideoStream(IAsyncEnumerable<VideoFrame> stream, int frameCount, float frameRate, int width, int height)
+        {
+            _stream = stream;
+            _frameCount = frameCount;
+            _frameRate = frameRate;
+            _width = width;
+            _height = height;
+        }
+
+        public int Width => _width;
+        public int Height => _height;
+        public float FrameRate => _frameRate;
+        public int FrameCount => _frameCount;
+
+        public IAsyncEnumerator<VideoFrame> GetAsyncEnumerator(CancellationToken cancellationToken = default)
+        {
+            return _stream.GetAsyncEnumerator(cancellationToken);
+        }
+    }
+}

TensorStack.Extractors/Common/BackgroundImageOptions.cs

@@ -1,6 +1,5 @@
 // Copyright (c) TensorStack. All rights reserved.
 // Licensed under the Apache 2.0 License.
-using TensorStack.Common;
 using TensorStack.Common.Pipeline;
 using TensorStack.Common.Tensor;
 

TensorStack.Extractors/Common/ExtractorImageOptions.cs

@@ -6,6 +6,6 @@ namespace TensorStack.Extractors.Common
 {
     public record ExtractorImageOptions : ExtractorOptions
     {
-        public ImageTensor Input { get; init; }
+        public ImageTensor Image { get; init; }
     }
 }

TensorStack.Extractors/Common/ExtractorStreamOptions.cs

@@ -1,13 +1,11 @@
 // Copyright (c) TensorStack. All rights reserved.
 // Licensed under the Apache 2.0 License.
-using System.Collections.Generic;
-using TensorStack.Common;
 using TensorStack.Common.Video;
 
 namespace TensorStack.Extractors.Common
 {
     public record ExtractorStreamOptions : ExtractorOptions
     {
-        public IAsyncEnumerable<VideoFrame> Input { get; }
+        public VideoStream Stream { get; }
     }
 }

TensorStack.Extractors/Common/ExtractorVideoOptions.cs

@@ -7,6 +7,6 @@ namespace TensorStack.Extractors.Common
 {
     public record ExtractorVideoOptions : ExtractorOptions
     {
-        public VideoTensor Input { get; }
+        public VideoTensor Video { get; }
     }
 }

TensorStack.Extractors/Pipelines/ExtractorPipeline.cs

@@ -65,16 +65,16 @@ public async Task<ImageTensor> RunAsync(ExtractorImageOptions options, IProgress
         {
             var timestamp = RunProgress.GetTimestamp();
             if (_extractorModel.Normalization == Normalization.ZeroToOne)
-                options.Input.NormalizeZeroToOne();
+                options.Image.NormalizeZeroToOne();
 
-            var resultTensor = await ExtractInternalAsync(options.Input, options, cancellationToken);
+            var resultTensor = await ExtractInternalAsync(options.Image, options, cancellationToken);
             NormalizeResult(resultTensor, options.IsInverted);
 
             if (_extractorModel.Normalization == Normalization.ZeroToOne)
-                options.Input.NormalizeOneToOne();
+                options.Image.NormalizeOneToOne();
 
             if (options.MergeInput)
-                resultTensor = MergeResult(options.Input, resultTensor);
+                resultTensor = MergeResult(options.Image, resultTensor);
 
             progressCallback?.Report(new RunProgress(timestamp));
             return resultTensor;
@@ -93,10 +93,10 @@ public async Task<VideoTensor> RunAsync(ExtractorVideoOptions options, IProgress
         {
             var timestamp = RunProgress.GetTimestamp();
             if (_extractorModel.Normalization == Normalization.ZeroToOne)
-                options.Input.NormalizeZeroToOne();
+                options.Video.NormalizeZeroToOne();
 
             var results = new List<ImageTensor>();
-            foreach (var frame in options.Input.GetFrames())
+            foreach (var frame in options.Video.GetFrames())
             {
                 var frameTime = Stopwatch.GetTimestamp();
                 var resultTensor = await ExtractInternalAsync(frame, options, cancellationToken);
@@ -107,10 +107,10 @@ public async Task<VideoTensor> RunAsync(ExtractorVideoOptions options, IProgress
                     resultTensor = MergeResult(frame, resultTensor);
 
                 results.Add(resultTensor);
-                progressCallback?.Report(new RunProgress(results.Count, options.Input.Frames, frameTime));
+                progressCallback?.Report(new RunProgress(results.Count, options.Video.Frames, frameTime));
             }
 
-            var resultVideoTensor = new VideoTensor(results.Join(), options.Input.FrameRate);
+            var resultVideoTensor = new VideoTensor(results.Join(), options.Video.FrameRate);
             progressCallback?.Report(new RunProgress(timestamp));
             return resultVideoTensor;
         }
@@ -129,7 +129,7 @@ public async IAsyncEnumerable<VideoFrame> RunAsync(ExtractorStreamOptions option
         {
             var frameCount = 0;
             var timestamp = RunProgress.GetTimestamp();
-            await foreach (var videoFrame in options.Input)
+            await foreach (var videoFrame in options.Stream)
             {
                 var frameTime = Stopwatch.GetTimestamp();
                 if (_extractorModel.Normalization == Normalization.ZeroToOne)

TensorStack.Upscaler/Common/UpscaleImageOptions.cs

@@ -2,6 +2,8 @@
 // Licensed under the Apache 2.0 License.
 using TensorStack.Common;
 using TensorStack.Common.Pipeline;
+using TensorStack.Common.Tensor;
+using TensorStack.Common.Video;
 
 namespace TensorStack.Upscaler.Common
 {
@@ -25,4 +27,43 @@ public abstract record UpscaleOptions : IRunOptions
         /// </summary>
         public int TileOverlap { get; init; }
     }
+
+
+
+    /// <summary>
+    /// Image UpscaleOptions.
+    /// </summary>
+    public sealed record UpscaleImageOptions : UpscaleOptions
+    {
+        /// <summary>
+        /// Gets the image input.
+        /// </summary>
+        public ImageTensor Image { get; init; }
+    }
+
+
+
+    /// <summary>
+    /// Video UpscaleOptions.
+    /// </summary>
+    public sealed record UpscaleVideoOptions : UpscaleOptions
+    {
+        /// <summary>
+        /// Gets the video input.
+        /// </summary>
+        public VideoTensor Video { get; init; }
+    }
+
+
+
+    /// <summary>
+    /// Stream UpscaleOptions.
+    /// </summary>
+    public sealed record UpscaleStreamOptions : UpscaleOptions
+    {
+        /// <summary>
+        /// Gets the stream input.
+        /// </summary>
+        public VideoStream Stream { get; init; }
+    }
 }