Tidy nuget pipeline, Upscale docs

saddam213 · saddam213 · commit 071d7410d89c · 2025-10-12T10:52:47.000+13:00
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -1,8 +1,8 @@
 <Project>
   <PropertyGroup>
-    <Version>0.1.1</Version>
+    <Version>0.1.7</Version>
     <Company>TensorStack</Company>
-    <Copyright>TensorStack 2025</Copyright>
+    <Copyright>TensorStack - 2025</Copyright>
     <RepositoryUrl>https://github.com/TensorStack-AI/TensorStack</RepositoryUrl>
     <PackageTags>onnx;onnx-runtime;</PackageTags>
     <Authors>sa_ddam213</Authors>
diff --git a/TensorStack.Audio.Windows/TensorStack.Audio.Windows.csproj b/TensorStack.Audio.Windows/TensorStack.Audio.Windows.csproj
@@ -1,7 +1,7 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net9.0-windows</TargetFramework>
+    <TargetFramework>net9.0-windows10.0.17763.0</TargetFramework>
     <PlatformTarget>x64</PlatformTarget>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
   </PropertyGroup>
@@ -38,6 +38,16 @@
       <Pack>True</Pack>
       <PackagePath>\</PackagePath>
     </None>
+    <Content Include="ffmpeg.exe">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <PackageCopyToOutput>true</PackageCopyToOutput>
+      <Pack>true</Pack>
+    </Content>
+    <Content Include="ffprobe.exe">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <PackageCopyToOutput>true</PackageCopyToOutput>
+      <Pack>true</Pack>
+    </Content>
   </ItemGroup>
 
 </Project>
diff --git a/TensorStack.Audio.Windows/ffmpeg.exe b/TensorStack.Audio.Windows/ffmpeg.exe
diff --git a/TensorStack.Audio.Windows/ffprobe.exe b/TensorStack.Audio.Windows/ffprobe.exe
diff --git a/TensorStack.Common/Common/FileHelper.cs b/TensorStack.Common/Common/FileHelper.cs
@@ -40,6 +40,7 @@ public static string RandomFileName(string extension)
 
         public static string RandomFileName(string directory, string extension)
         {
+            Directory.CreateDirectory(directory);
             return Path.Combine(directory, RandomFileName(extension));
         }
     }
diff --git a/TensorStack.Common/ModelConfig.cs b/TensorStack.Common/ModelConfig.cs
@@ -6,18 +6,24 @@ namespace TensorStack.Common
 {
     public record ModelConfig
     {
+        private ExecutionProvider _executionProvider;
+
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
         public string Path { get; set; }
 
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
         public bool IsOptimizationSupported { get; set; }
 
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public ExecutionProvider ExecutionProvider { get; private set; }
+        public ExecutionProvider ExecutionProvider
+        {
+            get { return _executionProvider; }
+            init { _executionProvider = value; }
+        }
 
         public virtual void SetProvider(ExecutionProvider executionProvider)
         {
-            ExecutionProvider = executionProvider;
+            _executionProvider = executionProvider;
         }
     }
 }
diff --git a/TensorStack.Image.Bitmap/TensorStack.Image.Bitmap.csproj b/TensorStack.Image.Bitmap/TensorStack.Image.Bitmap.csproj
@@ -1,7 +1,7 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net9.0-windows</TargetFramework>
+    <TargetFramework>net9.0-windows10.0.17763.0</TargetFramework>
     <RuntimeIdentifier>win-x64</RuntimeIdentifier>
     <PlatformTarget>x64</PlatformTarget>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
diff --git a/TensorStack.Image.BitmapImage/TensorStack.Image.BitmapImage.csproj b/TensorStack.Image.BitmapImage/TensorStack.Image.BitmapImage.csproj
@@ -1,7 +1,7 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net9.0-windows</TargetFramework>
+    <TargetFramework>net9.0-windows10.0.17763.0</TargetFramework>
     <RuntimeIdentifier>win-x64</RuntimeIdentifier>
     <PlatformTarget>x64</PlatformTarget>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
diff --git a/TensorStack.TextGeneration/Pipelines/Phi/Phi3Pipeline.cs b/TensorStack.TextGeneration/Pipelines/Phi/Phi3Pipeline.cs
@@ -123,7 +123,7 @@ protected override async Task<Sequence> InitializeAsync(GenerateOptions options)
             var inputIds = TokenizerOutput.InputIds;
             var positionIds = GetPositionIds(modelMetadata, 0, position);
             var attentionMask = new Tensor<long>([1, position], 1);
-            RunDecoderInternalAsync(modelMetadata, sequence, inputIds, positionIds, attentionMask, false);
+            RunDecoderInternal(modelMetadata, sequence, inputIds, positionIds, attentionMask, false);
             return sequence;
         }
 
@@ -140,7 +140,7 @@ protected override async Task<Tensor<float>> RunDecoderAsync(Sequence sequence)
             var inputIds = new Tensor<long>([1, 1], sequence.Tokens[^1]);
             var positionIds = GetPositionIds(modelMetadata, position);
             var attentionMask = new Tensor<long>([1, position], 1);
-            return RunDecoderInternalAsync(modelMetadata, sequence, inputIds, positionIds, attentionMask, true);
+            return RunDecoderInternal(modelMetadata, sequence, inputIds, positionIds, attentionMask, true);
         }
 
 
@@ -153,7 +153,7 @@ protected override async Task<Tensor<float>> RunDecoderAsync(Sequence sequence)
         /// <param name="positionIds">The position ids.</param>
         /// <param name="attentionMask">The attention mask.</param>
         /// <param name="useBranchCache">if set to <c>true</c> [use branch cache].</param>
-        private Tensor<float> RunDecoderInternalAsync(ModelMetadata modelMetadata, Sequence sequence, Tensor<long> inputIds, Tensor<long> positionIds, Tensor<long> attentionMask, bool useBranchCache)
+        private Tensor<float> RunDecoderInternal(ModelMetadata modelMetadata, Sequence sequence, Tensor<long> inputIds, Tensor<long> positionIds, Tensor<long> attentionMask, bool useBranchCache)
         {
             using (var parameters = new ModelParameters(modelMetadata))
             {
diff --git a/TensorStack.TextGeneration/Tokenizers/BPETokenizer.cs b/TensorStack.TextGeneration/Tokenizers/BPETokenizer.cs
@@ -116,7 +116,10 @@ public string Decode(int token, bool considerSpecialTokens = false)
         /// <param name="token">The token.</param>
         public string Decode(long token, bool considerSpecialTokens = false)
         {
-            return VocabularyMap[token];
+            if (!considerSpecialTokens && SpecialTokensMap.ContainsKey(token))
+                return string.Empty;
+
+            return TokensToString([VocabularyMap[token]]);
         }
 
 
diff --git a/TensorStack.Upscaler/README.md b/TensorStack.Upscaler/README.md
@@ -1,107 +1,185 @@
 # TensorStack.Upscaler
+High-performance ONNX image and video upscaling for AI-generated content or other media. Supports multiple models and provides both buffered and streaming video modes for flexible memory usage.
 
-## Upscale Models
-Below is a small list of known/tested upscale models
-* https://huggingface.co/wuminghao/swinir
-* https://huggingface.co/rocca/swin-ir-onnx
-* https://huggingface.co/Xenova/swin2SR-classical-sr-x2-64
-* https://huggingface.co/Xenova/swin2SR-classical-sr-x4-64
-* https://huggingface.co/Neus/GFPGANv1.4
+---
+
+## Quick Start
 
+This minimal example demonstrates how to upscale an image and a video using `TensorStack.Upscaler`.
 
-# Image Example
 ```csharp
-// Upscaler config
-var config = new UpscalerConfig("swin-2sr-classical-64-x2.onnx", Provider.DirectML)
+[nuget: TensorStack.Upscaler]
+[nuget: TensorStack.Providers.DML]
+[nuget: TensorStack.Image.Bitmap]
+[nuget: TensorStack.Video.Windows]
+[nuget: TensorStack.Audio.Windows]
+[model: https://huggingface.co/TensorStack/Upscale-amuse/resolve/main/RealESR-General-4x/model.onnx]
+
+async Task QuickStartAsync()
 {
-    ScaleFactor = 2,
-    Normalization = Normalization.ZeroToOne
-};
+    // 1. Create the Upscale Pipeline
+    var pipeline = UpscalePipeline.Create(new UpscalerConfig
+    {
+        ScaleFactor = 4,
+        Normalization = Normalization.ZeroToOne,
+        ExecutionProvider = Provider.GetProvider(),
+        Path = @"M:\Models\RealESR-General-4x\model.onnx"
+    });
+
+    // 2. Upscale an Image
+    var inputImage = new ImageInput("Input.png");
+    var upscaledImage = await pipeline.RunAsync(new UpscaleImageOptions
+    {
+        Image = inputImage,
+        TileMode = TileMode.None
+    });
+    await upscaledImage.SaveAsync("Output.png");
+
+    // 3. Upscale a Video (Streaming mode)
+    var inputStream = await VideoInputStream.CreateAsync("Input.mp4");
+    var upscaledVideo = pipeline.RunAsync(new UpscaleStreamOptions
+    {
+        Stream = inputStream.GetAsync(),
+        TileMode = TileMode.None
+    });
+    await upscaledVideo.SaveAsync("Output.mp4");
+
+    // 4. Add audio from the source video (optional)
+    await AudioManager.AddAudioAsync("Output.mp4", "Input.mp4");
+}
+```
+
+---
+
+
+## Creating an Upscale Pipeline
+
+```csharp
+[nuget: TensorStack.Upscaler]
+[nuget: TensorStack.Providers.DML]
 
-// Create Pipeline
-using (var pipeline = UpscalePipeline.Create(config))
+// Create the pipeline
+var pipeline = UpscalePipeline.Create(new UpscalerConfig
 {
-    // Load Pipeline
-    await pipeline.LoadAsync();
+    ScaleFactor = 4,
+    Normalization = Normalization.ZeroToOne,
+    ExecutionProvider = Provider.GetProvider(),
+    Path = @"M:\Models\RealESR-General-4x\model.onnx"
+});
+```
 
-    // Read input image
-    var input = new ImageInput("Input.png");
+**Configuration Options:**
 
-    // Options
-    var options = new UpscaleImageOptions(input);
+- `ScaleFactor` — Upscale factor (e.g., 2x, 4x)  
+- `Normalization` — Input value normalization (`ZeroToOne` or `OneToOne`)  
+- `ExecutionProvider` — Hardware provider (CPU, GPU, DirectML, etc.)  
+- `Path` — Path to the ONNX model  
 
-    // Run Upscale Pipeline
-    var outputTensor = await pipeline.RunAsync(options);
+---
 
-    // Save Output image
-    await outputTensor.SaveAsync("Output.png");
-}
+
+
+## Upscale Image
+```csharp
+    [nuget: TensorStack.Image.Bitmap]
+
+    // Read Image
+    var inputImage = new ImageInput("Input.png");
+
+    // Upscale Image
+    var output = await pipeline.RunAsync(new UpscaleImageOptions
+    {
+        Image = inputImage,
+        TileMode = TileMode.None
+    });
+
+    // Write Image
+    await output.SaveAsync("Output.png");
 ```
 
+---
 
 
-# Video Example
+## Upscale Video (Buffered)
+Buffers all frames in memory. Suitable for short-duration videos, AI-generated content, low-resolution videos, or GIFs.
 ```csharp
-// Upscaler config
-var config = new UpscalerConfig("swin-2sr-classical-64-x2.onnx", Provider.DirectML)
-{
-    ScaleFactor = 2,
-    Normalization = Normalization.ZeroToOne
-};
+    [nuget: TensorStack.Video.Windows]
 
-// Create Pipeline
-using (var pipeline = UpscalePipeline.Create(config))
-{
-    // Load Pipeline
-    await pipeline.LoadAsync();
+    // Read Video
+    var inputVideo = await VideoInput.CreateAsync("Input.gif");
 
-    // Read input video
-    var video = new VideoInput("Input.mp4");
+    // Upscale Video
+    var outputVideo = await pipeline.RunAsync(new UpscaleVideoOptions
+    {  
+        Video = inputVideo,
+        TileMode = TileMode.None
+    });
 
-    // Get video input
-    var input = await video.GetTensorAsync();
+    // Write Video
+    await outputVideo.SaveAync("Output.mp4");
+```
 
-    // Options
-    var options = new UpscaleVideoOptions(input);
+---
 
-    // Run Upscale Pipeline
-    var outputTensor = await pipeline.RunAsync(options);
+## Upscale Video (Stream)
+Processes frames one-by-one for minimal memory usage. Ideal for high-resolution or long-duration videos.
+```csharp
+    [nuget: TensorStack.Video.Windows]
 
-    // Save Output video
-    await outputTensor.SaveAync("Output.mp4");
-}
+    // Read Stream
+    var inputStream = await VideoInputStream.CreateAsync("Input.mp4");
+
+    // Upscale Stream
+    var outputStream = pipeline.RunAsync(new UpscaleStreamOptions
+    {
+        Stream = inputStream.GetAsync(),
+        TileMode = TileMode.None
+    });
+
+    // Write Stream
+    await outputStream.SaveAync("Output.mp4");
 ```
 
+---
 
+## Audio Support
+TensorStack.Video only processes video frames, so audio will be missing from the final result.
 
-# Video Stream Example
+You can use the TensorStack.Audio package to restore audio from the source video:
 ```csharp
-// Upscaler config
-var config = new UpscalerConfig("swin-2sr-classical-64-x2.onnx", Provider.DirectML)
-{
-    ScaleFactor = 2,
-    Normalization = Normalization.ZeroToOne
-};
+    [nuget: TensorStack.Audio.Windows]
 
-// Create Pipeline
-using (var pipeline = UpscalePipeline.Create(config))
-{
-    // Load Pipeline
-    await pipeline.LoadAsync();
+    await AudioManager.AddAudioAsync("TargetVideo.mp4", "SourceVideo.mp4");
+```
+---
 
-    // Read input video
-    var video = new VideoInput("Input.mp4");
+## Tiling Support
+Tiling allows images and video frames to be processed in smaller sections (tiles) instead of all at once. This helps reduce memory usage and can improve performance when working with very large images or high-resolution videos.
 
-    // Get video stream
-    var videoStream = video.GetStreamAsync();
+The `TileMode` determines how these tiles are handled:
 
-    // Options
-    var options = new UpscaleStreamOptions(videoStream);
+* **None:** Processes the entire image/frame in a single pass.
+* **Overlap:** Tiles have overlapping edges to avoid visible seams.
+* **Blend:** Overlapping tiles are blended together for smooth transitions.
+* **Clip:** Tiles are cut without blending.
+* **Clip + Blend:** Combines clipping and blending for high-quality results.
 
-    // Get Upscale stream
-    videoStream = pipeline.RunAsync(options);
+Additional options include:
 
-    // Save Video Steam
-    await videoStream.SaveAync("Output.mp4");
-}
-```
+* **MaxTileSize:** The maximum size of each tile in pixels. Smaller tiles reduce memory usage but may take longer to process.
+* **TileOverlap:** The number of overlapping pixels between tiles. More overlap can prevent visible seams and improve output quality.
+
+Adjusting these settings allows you to balance memory usage, processing speed, and visual quality for your upscaling tasks.
+
+---
+
+## Upscale Models
+
+Here is a list of some known and tested models compatible with `TensorStack.Upscaler`:
+
+- [wuminghao/swinir](https://huggingface.co/wuminghao/swinir)  
+- [rocca/swin-ir-onnx](https://huggingface.co/rocca/swin-ir-onnx)  
+- [Xenova/swin2SR-classical-sr-x2-64](https://huggingface.co/Xenova/swin2SR-classical-sr-x2-64)  
+- [Xenova/swin2SR-classical-sr-x4-64](https://huggingface.co/Xenova/swin2SR-classical-sr-x4-64)  
+- [Neus/GFPGANv1.4](https://huggingface.co/Neus/GFPGANv1.4)  
+- [TensorStack/Upscale-amuse](https://huggingface.co/TensorStack/Upscale-amuse)  
diff --git a/TensorStack.Video.Windows/Extensions.cs b/TensorStack.Video.Windows/Extensions.cs
diff --git a/TensorStack.Video.Windows/TensorStack.Video.Windows.csproj b/TensorStack.Video.Windows/TensorStack.Video.Windows.csproj

Original file line number	Diff line number	Diff line change
`@@ -40,6 +40,7 @@ public static string RandomFileName(string extension)`
`40`	`40`
`41`	`41`	`public static string RandomFileName(string directory, string extension)`
`42`	`42`	`{`
	`43`	`+ Directory.CreateDirectory(directory);`
`43`	`44`	`return Path.Combine(directory, RandomFileName(extension));`
`44`	`45`	`}`
`45`	`46`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,18 +6,24 @@ namespace TensorStack.Common`
`6`	`6`	`{`
`7`	`7`	`public record ModelConfig`
`8`	`8`	`{`
	`9`	`+ private ExecutionProvider _executionProvider;`
	`10`	`+`
`9`	`11`	`[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]`
`10`	`12`	`public string Path { get; set; }`
`11`	`13`
`12`	`14`	`[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]`
`13`	`15`	`public bool IsOptimizationSupported { get; set; }`
`14`	`16`
`15`	`17`	`[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]`
`16`		`- public ExecutionProvider ExecutionProvider { get; private set; }`
	`18`	`+ public ExecutionProvider ExecutionProvider`
	`19`	`+ {`
	`20`	`+ get { return _executionProvider; }`
	`21`	`+ init { _executionProvider = value; }`
	`22`	`+ }`
`17`	`23`
`18`	`24`	`public virtual void SetProvider(ExecutionProvider executionProvider)`
`19`	`25`	`{`
`20`		`- ExecutionProvider = executionProvider;`
	`26`	`+ _executionProvider = executionProvider;`
`21`	`27`	`}`
`22`	`28`	`}`
`23`	`29`	`}`
Original file line number	Diff line number	Diff line change
`@@ -123,7 +123,7 @@ protected override async Task<Sequence> InitializeAsync(GenerateOptions options)`
`123`	`123`	`var inputIds = TokenizerOutput.InputIds;`
`124`	`124`	`var positionIds = GetPositionIds(modelMetadata, 0, position);`
`125`	`125`	`var attentionMask = new Tensor<long>([1, position], 1);`
`126`		`- RunDecoderInternalAsync(modelMetadata, sequence, inputIds, positionIds, attentionMask, false);`
	`126`	`+ RunDecoderInternal(modelMetadata, sequence, inputIds, positionIds, attentionMask, false);`
`127`	`127`	`return sequence;`
`128`	`128`	`}`
`129`	`129`
`@@ -140,7 +140,7 @@ protected override async Task<Tensor<float>> RunDecoderAsync(Sequence sequence)`
`140`	`140`	`var inputIds = new Tensor<long>([1, 1], sequence.Tokens[^1]);`
`141`	`141`	`var positionIds = GetPositionIds(modelMetadata, position);`
`142`	`142`	`var attentionMask = new Tensor<long>([1, position], 1);`
`143`		`- return RunDecoderInternalAsync(modelMetadata, sequence, inputIds, positionIds, attentionMask, true);`
	`143`	`+ return RunDecoderInternal(modelMetadata, sequence, inputIds, positionIds, attentionMask, true);`
`144`	`144`	`}`
`145`	`145`
`146`	`146`
`@@ -153,7 +153,7 @@ protected override async Task<Tensor<float>> RunDecoderAsync(Sequence sequence)`
`153`	`153`	`/// <param name="positionIds">The position ids.</param>`
`154`	`154`	`/// <param name="attentionMask">The attention mask.</param>`
`155`	`155`	`/// <param name="useBranchCache">if set to <c>true</c> [use branch cache].</param>`
`156`		`- private Tensor<float> RunDecoderInternalAsync(ModelMetadata modelMetadata, Sequence sequence, Tensor<long> inputIds, Tensor<long> positionIds, Tensor<long> attentionMask, bool useBranchCache)`
	`156`	`+ private Tensor<float> RunDecoderInternal(ModelMetadata modelMetadata, Sequence sequence, Tensor<long> inputIds, Tensor<long> positionIds, Tensor<long> attentionMask, bool useBranchCache)`
`157`	`157`	`{`
`158`	`158`	`using (var parameters = new ModelParameters(modelMetadata))`
`159`	`159`	`{`
Original file line number	Diff line number	Diff line change
`@@ -116,7 +116,10 @@ public string Decode(int token, bool considerSpecialTokens = false)`
`116`	`116`	`/// <param name="token">The token.</param>`
`117`	`117`	`public string Decode(long token, bool considerSpecialTokens = false)`
`118`	`118`	`{`
`119`		`- return VocabularyMap[token];`
	`119`	`+ if (!considerSpecialTokens && SpecialTokensMap.ContainsKey(token))`
	`120`	`+ return string.Empty;`
	`121`	`+`
	`122`	`+ return TokensToString([VocabularyMap[token]]);`
`120`	`123`	`}`
`121`	`124`
`122`	`125`