Merge pull request #187 from JingyuanZhang/master

JingyuanZhang · web-flow · commit 15332de1f379 · 2021-07-16T00:00:45.000+08:00
feat(core): update mediaProcessor api and test
diff --git a/packages/paddlejs-core/__tests__/spec/mediaProcessor.test.ts b/packages/paddlejs-core/__tests__/spec/mediaProcessor.test.ts
@@ -17,7 +17,6 @@ describe('test mediaProcessor with scale and targetSize', () => {
     const mediaParams = {
         gapFillWith: '#000',
         mean: [0.485, 0.456, 0.406],
-        scale: 256,
         std: [0.229, 0.224, 0.225],
         feedShape: { fw: 224, fh: 224 },
         targetShape: [1, 3, 224, 224],
@@ -29,62 +28,22 @@ describe('test mediaProcessor with scale and targetSize', () => {
     processror.pixelWidth = 800;
     processror.pixelHeight = 600;
 
-    test('test api resizeAndFitTargetSize', () => {
-        const imagedata = processror.resizeAndFitTargetSize(img, mediaParams);
-        expect(imagedata.width).toBe(224);
-        expect(imagedata.data.slice(0, 4).join(',')).toEqual('0,0,0,255');
-    });
+    const imageDataInfo = {
+        dx: 0,
+        dy: 0,
+        dWidth: 224,
+        dHeight: 224
+    };
 
-    test('test api fitToTargetSize', () => {
-        const res = processror.fitToTargetSize(img, mediaParams);
-        expect(res.sw).toBe(224);
-    });
-
-    test('test api reSize', () => {
-        const res = processror.reSize(img, mediaParams);
-        expect(res.sw).toBe(256);
-        expect(res.sh).toBe(256);
-        processror.pixelWidth = 600;
-        processror.pixelHeight = 800;
-        const res1 = processror.reSize(img, mediaParams);
-        expect(res1.sw).toBe(256);
-        expect(res1.sh).toBe(256);
-        processror.pixelWidth = 700;
-        processror.pixelHeight = 700;
-        const res2 = processror.reSize(img, mediaParams);
-        expect(res2.sw).toBe(256);
-        expect(res2.sh).toBe(256);
-    });
 
     test('test api allReshapeToRGB', () => {
-        const imagedata = processror.resizeAndFitTargetSize(img, mediaParams);
+        processror.fitToTargetSize(img, mediaParams);
+        const imagedata = processror.getImageData(imageDataInfo);
         const data = processror.allReshapeToRGB(imagedata, mediaParams);
         expect(data.length).toBe(150528);
     });
 
-    test('test api grayscale', () => {
-        const imagedata = processror.resizeAndFitTargetSize(img, mediaParams);
-        const avg = (imagedata.data[0] + imagedata.data[1] + imagedata.data[2]) / 3;
-        const data = processror.grayscale(imagedata);
-        expect(data[0]).toBe(avg);
-        expect(data[3]).toBe(255);
-    });
 
-    test('test api reshape', () => {
-        const imagedata = {
-            data: new Array(10 * 12 * 4).fill(0)
-        };
-        const data = processror.reshape(imagedata, {
-            width: 5,
-            height: 6,
-            mean: [0.485, 0.456, 0.406],
-            std: [0.229, 0.224, 0.225]
-        }, {
-            sw: 10,
-            sh: 12
-        });
-        expect(data.length).toBe(90);
-    });
 
     test('test api process', () => {
         const data = processror.process(img, mediaParams);
diff --git a/packages/paddlejs-core/src/commons/interface.ts b/packages/paddlejs-core/src/commons/interface.ts
@@ -40,7 +40,6 @@ export interface ModelConfig {
     mean?: number[];
     std?: number[];
     bgr?: boolean;
-    scale?: number;
     type?: GraphType; // model type
     needPreheat?: boolean;
     plugins?: { // tranform graph plugins
diff --git a/packages/paddlejs-core/src/mediaProcessor.ts b/packages/paddlejs-core/src/mediaProcessor.ts
@@ -14,8 +14,8 @@ export default class MediaProcessor {
     std: number[] = [1, 1, 1];
     bgr: boolean = false;
     result: Float32Array | number[] = [];
-    pixelWidth: number = 224;
-    pixelHeight: number = 224;
+    pixelWidth: number = 1;
+    pixelHeight: number = 1;
     inputFeed: InputFeed[] = [];
 
     constructor() {
@@ -28,17 +28,20 @@ export default class MediaProcessor {
      * @param inputs
      */
     process(media, modelConfig): InputFeed[] {
-        const { feedShape, fill, targetSize, scale, mean, std, bgr } = modelConfig;
+        const { feedShape, fill, mean, std, bgr } = modelConfig;
         const { fh, fw } = feedShape;
         const input = media;
 
+
         const params = {
             gapFillWith: fill || this.gapFillWith,
             mean: mean || this.mean,
             std: std || this.std,
             bgr: bgr || this.bgr,
-            scale,
-            targetSize,
+            targetSize: {
+                width: fw,
+                height: fh
+            },
             targetShape: [1, 3, fh, fw]
         };
 
@@ -52,7 +55,12 @@ export default class MediaProcessor {
 
     fromPixels(pixels, opt): InputFeed[] {
         let data: ImageData | number[] | Float32Array = [];
-        let scaleSize;
+        const imageDataInfo = {
+            dx: 0,
+            dy: 0,
+            dWidth: opt.targetSize.width,
+            dHeight: opt.targetSize.height
+        };
 
         if (!(pixels instanceof HTMLImageElement
             || pixels instanceof HTMLVideoElement
@@ -66,79 +74,28 @@ export default class MediaProcessor {
 
         this.pixelWidth = pixels.width;
         this.pixelHeight = pixels.height;
-        if (opt.scale && opt.targetSize) {
-            data = this.resizeAndFitTargetSize(pixels, opt);
-        }
-        else if (opt.targetSize) { // 如果有 targetSize，就是装在目标宽高里的模式
-            scaleSize = this.fitToTargetSize(pixels, opt);
-            data = this.getImageData(0, 0, scaleSize);
-        }
-        else {
-            scaleSize = this.reSize(pixels, opt);
-            data = this.getImageData(0, 0, scaleSize);
-        }
-        if (opt.gray) {
-            data = this.grayscale(data);
-        }
 
-        if (opt.reShape) {
-            data = this.reshape(data, opt, scaleSize);
-        }
+        this.fitToTargetSize(pixels, opt);
+        data = this.getImageData(imageDataInfo);
 
         // process imageData in webgl
         if (env.get('webgl_feed_process')) {
             data = Float32Array.from((data as ImageData).data);
             return [{
-                data: data,
+                data,
                 shape: [1, 4, opt.targetShape[2], opt.targetShape[3]],
                 name: 'image'
             }] as InputFeed[];
         }
 
         data = this.allReshapeToRGB(data, opt) as number[];
         return [{
-            data: data,
+            data,
             shape: opt.targetShape || opt.shape,
             name: 'image'
         }] as InputFeed[];
     }
 
-    /**
-     * crop图像&重新设定图片tensor形状
-     * @param shape
-     */
-    reshape(imageData, opt, scaleSize) {
-        const { sw, sh } = scaleSize;
-        const { width, height } = opt;
-        const hPadding = Math.ceil((sw - width) / 2);
-        const vPadding = Math.ceil((sh - height) / 2);
-
-        const data = imageData.data;
-        // channel RGB
-        const red: number[] = [];
-        const green: number[] = [];
-        const blue: number[] = [];
-        // 平均数
-        const mean = opt.mean;
-        // 标准差
-        const std = opt.std;
-        // 考虑channel因素获取数据
-        for (let i = 0; i < data.length; i += 4) {
-            // img_mean 0.485, 0.456, 0.406
-            // img_std 0.229, 0.224, 0.225
-            const index = i / 4;
-            const vIndex = Math.floor(index / sw);
-            const hIndex = index - (vIndex * sw) - 1;
-            if (hIndex >= hPadding && hIndex < (hPadding + width)
-                && vIndex >= vPadding && vIndex < (vPadding + height)) {
-                red.push(((data[i] / 255) - mean[0]) / std[0]); // red
-                green.push(((data[i + 1] / 255) - mean[1]) / std[1]); // green
-                blue.push(((data[i + 2] / 255) - mean[2]) / std[2]); // blue
-            }
-        }
-        // 转成 GPU 加速 NCHW 格式
-        return red.concat(green.concat(blue));
-    }
 
     /**
      * 全部转rgb * H * W
@@ -174,60 +131,6 @@ export default class MediaProcessor {
         return result;
     }
 
-    /**
-     * 根据scale缩放图像
-     * @param image
-     * @param params
-     * @return {Object} 缩放后的尺寸
-     */
-    reSize(image, params) {
-        // 原始图片宽高
-        const width = this.pixelWidth;
-        const height = this.pixelHeight;
-        // 缩放后的宽高
-        let sw = width;
-        let sh = height;
-        sw = sh = params.scale;
-
-        this.targetContext.canvas.width = sw;
-        this.targetContext.canvas.height = sh;
-        this.targetContext.drawImage(
-            image, 0, 0, sw, sh);
-        return { sw, sh };
-    }
-
-    /**
-     * 根据scale缩放图像并且缩放成目标尺寸并居中
-     */
-    resizeAndFitTargetSize(image, params) {
-        // 原始图片宽高
-        const width = this.pixelWidth;
-        const height = this.pixelHeight;
-        // 缩放后的宽高
-        let sw = width;
-        let sh = height;
-        // 最小边缩放到scale
-        if (width < height) {
-            sw = params.scale;
-            sh = Math.round(sw * height / width);
-        }
-        else {
-            sh = params.scale;
-            sw = Math.round(sh * width / height);
-        }
-
-        this.targetContext.canvas.width = sw;
-        this.targetContext.canvas.height = sh;
-        const targetWidth = params.targetSize.width;
-        const targetHeight = params.targetSize.height;
-        this.targetContext.drawImage(image, 0, 0, sw, sh);
-        const x = (sw - targetWidth) / 2;
-        const y = (sh - targetHeight) / 2;
-        sw = targetWidth;
-        sh = targetHeight;
-        const data = this.getImageData(x, y, { sw, sh });
-        return data;
-    }
 
     /**
      * 缩放成目标尺寸并居中
@@ -257,37 +160,18 @@ export default class MediaProcessor {
         }
 
         this.targetContext.drawImage(image, x, y, sw, sh);
-
-        return { sw: targetWidth, sh: targetHeight };
     }
 
     /**
      * 获取图像内容
      * @param pixels
      * @returns {Uint8ClampedArray}
      */
-    getImageData(x, y, scaleSize) {
+    getImageData(imageDataInfo) {
 
-        const { sw, sh } = scaleSize;
+        const { dx, dy, dWidth, dHeight } = imageDataInfo;
         // 复制画布上指定矩形的像素数据
-        return this.targetContext.getImageData(x, y, sw, sh);
+        return this.targetContext.getImageData(dx, dy, dWidth, dHeight);
     }
 
-    /**
-     * 计算灰度图
-     * @param imageData
-     * @returns {*}
-     */
-    grayscale(imageData) {
-        const data = imageData.data;
-
-        for (let i = 0; i < data.length; i += 4) {
-            // 3 channel 灰度处理无空间压缩
-            const avg = (data[i] + data[i + 1] + data[i + 2]) / 3;
-            data[i] = avg; // red
-            data[i + 1] = avg; // green
-            data[i + 2] = avg; // blue
-        }
-        return data;
-    }
 }
diff --git a/packages/paddlejs-models/gesture/package-lock.json b/packages/paddlejs-models/gesture/package-lock.json
diff --git a/packages/paddlejs-models/gesture/package.json b/packages/paddlejs-models/gesture/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@paddlejs-models/gesture",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "",
   "main": "lib/index",
   "scripts": {

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@paddlejs-models/gesture",`
`3`		`- "version": "0.0.1",`
	`3`	`+ "version": "0.0.3",`
`4`	`4`	`"description": "",`
`5`	`5`	`"main": "lib/index",`
`6`	`6`	`"scripts": {`