Pose extractor pipeline

Author: saddam213

Examples/TensorStack.Example.Extractors/Services/BackgroundService.cs

@@ -133,7 +133,7 @@ public async Task<ImageInput> ExecuteAsync(BackgroundImageRequest options)
                     var imageTensor = await Task.Run(() => _currentPipeline.RunAsync(new BackgroundImageOptions
                     {
                         Mode = options.Mode,
-                        Input = options.Image
+                        Image = options.Image
                     }, cancellationToken: _cancellationTokenSource.Token));
 
                     if (options.IsTransparentSupported)
@@ -169,7 +169,7 @@ async Task<VideoFrame> FrameProcessor(VideoFrame frame)
                     {
                         var processedFrame = await _currentPipeline.RunAsync(new BackgroundImageOptions
                         {
-                            Input = frame.Frame,
+                            Image = frame.Frame,
                             Mode = options.Mode
                         }, cancellationToken: cancellationToken);
 

TensorStack.Common/Common/Enums.cs

@@ -58,7 +58,12 @@ public enum ResizeMode
         /// <summary>
         /// Center Crop Image
         /// </summary>
-        Crop = 1
+        Crop = 1,
+
+        /// <summary>
+        /// LetterBox Center Image
+        /// </summary>
+        LetterBox = 2
     }
 
 

TensorStack.Common/Extensions/TensorExtensions.cs

@@ -1019,6 +1019,45 @@ public static ImageTensor ResizeImage(this ImageTensor sourceImage, int targetWi
         }
 
 
+        /// <summary>
+        /// Gets the crop coordinates.
+        /// </summary>
+        /// <param name="sourceHeight">Height of the source.</param>
+        /// <param name="sourceWidth">Width of the source.</param>
+        /// <param name="targetHeight">Height of the target.</param>
+        /// <param name="targetWidth">Width of the target.</param>
+        /// <param name="resizeMode">The resize mode.</param>
+        /// <returns>CoordinateBox&lt;System.Int32&gt;.</returns>
+        private static CoordinateBox<int> GetCropCoordinates(int sourceHeight, int sourceWidth, int targetHeight, int targetWidth, ResizeMode resizeMode)
+        {
+            var cropX = 0;
+            var cropY = 0;
+            var croppedWidth = targetWidth;
+            var croppedHeight = targetHeight;
+            if (resizeMode == ResizeMode.Crop)
+            {
+                var scaleX = (float)targetWidth / sourceWidth;
+                var scaleY = (float)targetHeight / sourceHeight;
+                var scaleFactor = Math.Max(scaleX, scaleY);
+                croppedWidth = (int)(sourceWidth * scaleFactor);
+                croppedHeight = (int)(sourceHeight * scaleFactor);
+                cropX = Math.Abs(Math.Max((croppedWidth - targetWidth) / 2, 0));
+                cropY = Math.Abs(Math.Max((croppedHeight - targetHeight) / 2, 0));
+            }
+            else if (resizeMode == ResizeMode.LetterBox)
+            {
+                var scaleX = (float)targetWidth / sourceWidth;
+                var scaleY = (float)targetHeight / sourceHeight;
+                var scaleFactor = Math.Min(scaleX, scaleY);
+                croppedWidth = (int)(sourceWidth * scaleFactor);
+                croppedHeight = (int)(sourceHeight * scaleFactor);
+                cropX = -(targetWidth - croppedWidth) / 2;
+                cropY = -(targetHeight - croppedHeight) / 2;
+            }
+            return new CoordinateBox<int>(cropX, cropY, croppedWidth, croppedHeight);
+        }
+
+
         /// <summary>
         /// Resizes the specified ImageTensor (Bilinear)
         /// </summary>
@@ -1032,15 +1071,20 @@ private static ImageTensor ResizeImageBilinear(ImageTensor sourceImage, int targ
             var sourceHeight = sourceImage.Dimensions[2];
             var sourceWidth = sourceImage.Dimensions[3];
             var cropSize = GetCropCoordinates(sourceHeight, sourceWidth, targetHeight, targetWidth, resizeMode);
-            var destination = new ImageTensor(new[] { 1, channels, targetHeight, targetWidth });
-            Parallel.For(0, channels, c =>
+            var destination = new ImageTensor([1, channels, targetHeight, targetWidth]);
+            if (resizeMode == ResizeMode.LetterBox)
+                destination.Fill(0f);
+
+            var scaleY = (float)(sourceHeight - 1) / (cropSize.MaxY - 1);
+            var sclaeX = (float)(sourceWidth - 1) / (cropSize.MaxX - 1);
+            Parallel.For(0, cropSize.MaxY, h =>
             {
-                for (int h = 0; h < cropSize.MaxY; h++)
+                for (var c = 0; c < channels; c++)
                 {
                     for (int w = 0; w < cropSize.MaxX; w++)
                     {
-                        var y = h * (float)(sourceHeight - 1) / (cropSize.MaxY - 1);
-                        var x = w * (float)(sourceWidth - 1) / (cropSize.MaxX - 1);
+                        var y = h * scaleY;
+                        var x = w * sclaeX;
 
                         var y0 = (int)Math.Floor(y);
                         var x0 = (int)Math.Floor(x);
@@ -1084,42 +1128,46 @@ private static ImageTensor ResizeImageBicubic(ImageTensor sourceImage, int targe
             var sourceHeight = sourceImage.Dimensions[2];
             var sourceWidth = sourceImage.Dimensions[3];
             var cropSize = GetCropCoordinates(sourceHeight, sourceWidth, targetHeight, targetWidth, resizeMode);
-            var destination = new ImageTensor(new[] { 1, channels, targetHeight, targetWidth });
-            Parallel.For(0, channels, c =>
+            var destination = new ImageTensor([1, channels, targetHeight, targetWidth]);
+            if (resizeMode == ResizeMode.LetterBox)
+                destination.Fill(0f);
+
+            var scaleX = (float)sourceWidth / cropSize.MaxX;
+            var scaleY = (float)sourceHeight / cropSize.MaxY;
+            Parallel.For(0, cropSize.MaxY, h =>
             {
-                for (int h = 0; h < cropSize.MaxY; h++)
+                for (var c = 0; c < channels; c++)
                 {
                     for (int w = 0; w < cropSize.MaxX; w++)
                     {
-                        float y = h * (float)(sourceHeight - 1) / (cropSize.MaxY - 1);
-                        float x = w * (float)(sourceWidth - 1) / (cropSize.MaxX - 1);
+                        float srcY = (h + 0.5f) * scaleY - 0.5f;
+                        float srcX = (w + 0.5f) * scaleX - 0.5f;
 
-                        int yInt = (int)Math.Floor(y);
-                        int xInt = (int)Math.Floor(x);
-                        float yFrac = y - yInt;
-                        float xFrac = x - xInt;
+                        int yInt = (int)Math.Floor(srcY);
+                        int xInt = (int)Math.Floor(srcX);
+                        float yFrac = srcY - yInt;
+                        float xFrac = srcX - xInt;
+                        float pixelValue = 0f;
 
-                        float[] colVals = new float[4];
-
-                        for (int i = -1; i <= 2; i++)
+                        // 2D bicubic: sum over 16 neighbors
+                        for (int m = -1; m <= 2; m++)
                         {
-                            int yi = Math.Clamp(yInt + i, 0, sourceHeight - 1);
-                            float[] rowVals = new float[4];
+                            int yi = MirrorIndex(yInt + m, sourceHeight);
+                            float wY = MitchellNetravali(m - yFrac);
 
-                            for (int j = -1; j <= 2; j++)
+                            for (int n = -1; n <= 2; n++)
                             {
-                                int xi = Math.Clamp(xInt + j, 0, sourceWidth - 1);
-                                rowVals[j + 1] = sourceImage[0, c, yi, xi];
+                                int xi = MirrorIndex(xInt + n, sourceWidth);
+                                float wX = MitchellNetravali(n - xFrac);
+                                pixelValue += sourceImage[0, c, yi, xi] * wX * wY;
                             }
-
-                            colVals[i + 1] = CubicInterpolate(rowVals[0], rowVals[1], rowVals[2], rowVals[3], xFrac);
                         }
 
-                        var targetY = h - cropSize.MinY;
-                        var targetX = w - cropSize.MinX;
+                        int targetY = h - cropSize.MinY;
+                        int targetX = w - cropSize.MinX;
                         if (targetX >= 0 && targetY >= 0 && targetY < targetHeight && targetX < targetWidth)
                         {
-                            destination[0, c, h, w] = CubicInterpolate(colVals[0], colVals[1], colVals[2], colVals[3], yFrac);
+                            destination[0, c, targetY, targetX] = pixelValue;
                         }
                     }
                 }
@@ -1130,50 +1178,38 @@ private static ImageTensor ResizeImageBicubic(ImageTensor sourceImage, int targe
 
 
         /// <summary>
-        /// Cubic interpolate.
+        /// Mitchell-Netravali kernel (sharper, natural bicubic)
         /// </summary>
-        /// <param name="v0">The v0.</param>
-        /// <param name="v1">The v1.</param>
-        /// <param name="v2">The v2.</param>
-        /// <param name="v3">The v3.</param>
-        /// <param name="fraction">The fraction.</param>
+        /// <param name="value">The value.</param>
         /// <returns>System.Single.</returns>
-        private static float CubicInterpolate(float v0, float v1, float v2, float v3, float fraction)
+        private static float MitchellNetravali(float value)
         {
-            float A = (-0.5f * v0) + (1.5f * v1) - (1.5f * v2) + (0.5f * v3);
-            float B = (v0 * -1.0f) + (v1 * 2.5f) - (v2 * 2.0f) + (v3 * 0.5f);
-            float C = (-0.5f * v0) + (0.5f * v2);
-            float D = v1;
-            return A * (fraction * fraction * fraction) + B * (fraction * fraction) + C * fraction + D;
+            value = Math.Abs(value);
+            const float B = 1f / 3f;
+            const float C = 1f / 3f;
+
+            if (value < 1f)
+                return ((12 - 9 * B - 6 * C) * (value * value * value) + (-18 + 12 * B + 6 * C) * (value * value) + (6 - 2 * B)) / 6f;
+            else if (value < 2f)
+                return ((-B - 6 * C) * (value * value * value) + (6 * B + 30 * C) * (value * value) + (-12 * B - 48 * C) * value + (8 * B + 24 * C)) / 6f;
+            else
+                return 0f;
         }
 
 
         /// <summary>
-        /// Gets the crop coordinates.
+        /// Mirror padding helper
         /// </summary>
-        /// <param name="sourceHeight">Height of the source.</param>
-        /// <param name="sourceWidth">Width of the source.</param>
-        /// <param name="targetHeight">Height of the target.</param>
-        /// <param name="targetWidth">Width of the target.</param>
-        /// <param name="resizeMode">The resize mode.</param>
-        /// <returns>CoordinateBox&lt;System.Int32&gt;.</returns>
-        private static CoordinateBox<int> GetCropCoordinates(int sourceHeight, int sourceWidth, int targetHeight, int targetWidth, ResizeMode resizeMode)
+        /// <param name="i">The i.</param>
+        /// <param name="max">The maximum.</param>
+        /// <returns>System.Int32.</returns>
+        private static int MirrorIndex(int i, int max)
         {
-            var cropX = 0;
-            var cropY = 0;
-            var croppedWidth = targetWidth;
-            var croppedHeight = targetHeight;
-            if (resizeMode == ResizeMode.Crop)
-            {
-                var scaleX = (float)targetWidth / sourceWidth;
-                var scaleY = (float)targetHeight / sourceHeight;
-                var scaleFactor = Math.Max(scaleX, scaleY);
-                croppedWidth = (int)(sourceWidth * scaleFactor);
-                croppedHeight = (int)(sourceHeight * scaleFactor);
-                cropX = Math.Abs(Math.Max((croppedWidth - targetWidth) / 2, 0));
-                cropY = Math.Abs(Math.Max((croppedHeight - targetHeight) / 2, 0));
-            }
-            return new CoordinateBox<int>(cropX, cropY, croppedWidth, croppedHeight);
+            if (i < 0)
+                return -i;
+            if (i >= max)
+                return 2 * max - i - 2;
+            return i;
         }
     }
 }

TensorStack.Common/Tensor/ImageTensor.cs

@@ -133,9 +133,9 @@ public void UpdateAlphaChannel(ImageTensor tensor)
         /// <param name="width">The target width in pixels.</param>
         /// <param name="height">The target height in pixels..</param>
         /// <param name="resizeMode">The resize mode.</param>
-        public void Resize(int width, int height, ResizeMode resizeMode)
+        public void Resize(int width, int height, ResizeMode resizeMode, ResizeMethod resizeMethod = ResizeMethod.Bilinear)
         {
-           UpdateTensor(this.ResizeImage(width, height, resizeMode));
+           UpdateTensor(this.ResizeImage(width, height, resizeMode, resizeMethod));
         }
 
 

TensorStack.Common/Vision/CLIPImage.cs

@@ -16,10 +16,9 @@ public class CLIPImage
         /// <param name="width">The width.</param>
         /// <param name="height">The height.</param>
         /// <returns>ImageTensor.</returns>
-        public static ImageTensor Process(ImageTensor input, int width = 224, int height = 224)
+        public static ImageTensor Process(ImageTensor input, int width = 224, int height = 224, ResizeMode ResizeMode = ResizeMode.Stretch)
         {
-            var options = new CLIPImageOptions(width, height);
-            return Process(input, options);
+            return Process(input, new CLIPImageOptions(width, height, ResizeMode));
         }
 
 
@@ -32,8 +31,7 @@ public static ImageTensor Process(ImageTensor input, int width = 224, int height
         public static ImageTensor Process(ImageTensor input, CLIPImageOptions options)
         {
             options ??= new CLIPImageOptions();
-            var resultTensor = input.ResizeImage(options.Width, options.Height, ResizeMode.Stretch);
-            resultTensor.NormalizeZeroToOne();
+            var resultTensor = input.ResizeImage(options.Width, options.Height, options.ResizeMode, options.ResizeMethod);
             for (int x = 0; x < resultTensor.Width; x++)
             {
                 for (int y = 0; y < resultTensor.Height; y++)

TensorStack.Common/Vision/CLIPImageOptions.cs

@@ -2,7 +2,7 @@
 // Licensed under the Apache 2.0 License.
 namespace TensorStack.Common.Vision
 {
-    public record CLIPImageOptions(int Width = 224, int Height = 224)
+    public record CLIPImageOptions(int Width = 224, int Height = 224, ResizeMode ResizeMode = ResizeMode.Stretch, ResizeMethod ResizeMethod = ResizeMethod.Bilinear)
     {
         /// <summary>
         /// The Mean to use if normalizing the image.

TensorStack.Extractors/Common/BackgroundImageOptions.cs

@@ -18,9 +18,10 @@ public record BackgroundImageOptions : IRunOptions
         /// <summary>
         /// Gets the input.
         /// </summary>
-        public ImageTensor Input { get; init; }
+        public ImageTensor Image { get; init; }
     }
 
+
     public enum BackgroundMode
     {
         MaskBackground = 0,

TensorStack.Extractors/Common/ExtractorImageOptions.cs

@@ -1,7 +1,10 @@
 // Copyright (c) TensorStack. All rights reserved.
 // Licensed under the Apache 2.0 License.
+using System.Collections.Generic;
 using TensorStack.Common;
 using TensorStack.Common.Pipeline;
+using TensorStack.Common.Tensor;
+using TensorStack.Common.Video;
 
 namespace TensorStack.Extractors.Common
 {
@@ -35,4 +38,40 @@ public record ExtractorOptions : IRunOptions
         /// </summary>
         public bool IsInverted { get; init; }
     }
+
+
+    /// <summary>
+    /// Image ExtractorOptions.
+    /// </summary>
+    public record ExtractorImageOptions : ExtractorOptions
+    {
+        /// <summary>
+        /// Gets the image.
+        /// </summary>
+        public ImageTensor Image { get; init; }
+    }
+
+
+    /// <summary>
+    /// Video ExtractorOptions.
+    /// </summary>
+    public record ExtractorVideoOptions : ExtractorOptions
+    {
+        /// <summary>
+        /// Gets the video.
+        /// </summary>
+        public VideoTensor Video { get; init; }
+    }
+
+
+    /// <summary>
+    /// Stream ExtractorOptions.
+    /// </summary>
+    public record ExtractorStreamOptions : ExtractorOptions
+    {
+        /// <summary>
+        /// Gets the stream.
+        /// </summary>
+        public IAsyncEnumerable<VideoFrame> Stream { get; init; }
+    }
 }