fix: unify camera permission handling, detection background handling, detection sig, support processEveryNthFrame, onTorch handling

closes: #13
closes: #20
closes: #21
closes: #22
closes: #23
closes: #24

Author: triniwiz

apps/demo/src/plugin-demos/mlkit-core.ts

@@ -8,69 +8,72 @@ import { ObjectResult } from '@nativescript/mlkit-object-detection';
 import { PoseResult } from '@nativescript/mlkit-pose-detection';
 import { TextResult } from '@nativescript/mlkit-text-recognition';
 export function navigatingTo(args: EventData) {
-	const page = <Page>args.object;
-	page.bindingContext = new DemoModel();
+  const page = <Page>args.object;
+  page.bindingContext = new DemoModel();
 }
 
 export class DemoModel extends DemoSharedMlkitCore {
-	camera: MLKitView;
-	detectorType = "all";
-	isPaused = false;
-	onLoaded(args) {
-		this.camera = args.object;
-	}
+  camera: MLKitView;
+  detectorType = DetectionType.Barcode;
+  isPaused = false;
+  torchOn = false;
+  onLoaded(args) {
+    this.camera = args.object;
+    this.set('isPaused', this.camera.pause);
+	this.set('torchOn', this.camera.torchOn);
+  }
 
-	onDetection(event: DetectionEvent) {
-		console.log('onDetection', event.data, event.type);
-	}
+  onDetection(event: DetectionEvent) {
+    console.log('onDetection', event.data, event.type);
+    if (event.type === DetectionType.Barcode) {
+      const first = event.data[0] as BarcodeResult;
+      console.log('bounds', first.bounds);
+    }
+  }
 
-	toggleCamera() {
-		this.camera.toggleCamera();
-	}
+  toggleCamera() {
+    this.camera.toggleCamera();
+  }
 
-	requestPermission() {
-		this.camera.requestCameraPermission();
-	}
+  toggleTorch() {
+    this.camera.torchOn = !this.camera.torchOn
+	this.set('torchOn', this.camera.torchOn);
+  }
 
-	changeType(args) {
-		Dialogs.action('Change Detector Type', 'Cancel', [
-			'all',
-			'barcode',
-			'digitalInk (unsupport atm)',
-			'face',
-			'image',
-			'object',
-			'pose',
-			'text',
-			'none'
-		]).then(value => {
-			if (value === 'Cancel') { return }
-			if (value.indexOf('digitalInk') > -1) {
-				Dialogs.alert('digitalInk is currently unsupported')
-				this.set('detectorType', 'none');
-			} else {
-				this.set('detectorType', value);
-			}
-		})
-	}
+  requestPermission() {
+    this.camera.requestCameraPermission();
+  }
 
-	togglePause(args) {
-		this.camera.pause = !this.camera.pause;
-		this.set('isPaused', this.camera.pause);
-	}
+  changeType(args) {
+    Dialogs.action('Change Detector Type', 'Cancel', ['all', 'barcode', 'digitalInk (unsupport atm)', 'face', 'image', 'object', 'pose', 'text', 'none']).then((value) => {
+      if (value === 'Cancel') {
+        return;
+      }
+      if (value.indexOf('digitalInk') > -1) {
+        Dialogs.alert('digitalInk is currently unsupported');
+        this.set('detectorType', 'none');
+      } else {
+        this.set('detectorType', value);
+      }
+    });
+  }
 
-	async processStill(args) {
-		try {
-			const src = await ImageSource.fromUrl('https://www.jqueryscript.net/images/jQuery-Plugin-To-Generate-International-Article-Number-Barcode-EAN13.jpg');
-			
-			console.log(src.android)
-			const result = await detectWithStillImage(src, {
-				detectorType: DetectionType.Barcode
-			});
-			console.log('processStill', result.barcode[0]);
-		} catch (e) {
-			console.log(e);
-		}
-	}
-}
+  togglePause(args) {
+    this.camera.pause = !this.camera.pause;
+    this.set('isPaused', this.camera.pause);
+  }
+
+  async processStill(args) {
+    try {
+      const src = await ImageSource.fromUrl('https://www.jqueryscript.net/images/jQuery-Plugin-To-Generate-International-Article-Number-Barcode-EAN13.jpg');
 
+      console.log(src.android);
+      const result = await detectWithStillImage(src, {
+        detectorType: DetectionType.Barcode,
+      });
+      console.log('processStill', result.barcode[0]);
+    } catch (e) {
+      console.log(e);
+    }
+  }
+}

apps/demo/src/plugin-demos/mlkit-core.xml

@@ -3,14 +3,16 @@
         <ActionBar title="mlkit-core" icon="" class="action-bar">
         </ActionBar>
     </Page.actionBar>
-    <GridLayout rows="*,auto,auto,auto,auto,auto,auto,auto" height="100%">
-       <ui:MLKitView pause="false" height="100%" rowSpan="3"  loaded="{{ onLoaded }}" cameraPosition="back" detectionType="{{ detectorType }}" detection="{{ onDetection }}"/>
+    <GridLayout rows="*,auto,auto,auto,auto,auto,auto,auto,auto,auto" height="100%">
+       <ui:MLKitView pause="true" height="100%" rowSpan="3"  loaded="{{ onLoaded }}" cameraPosition="back" detectionType="{{ detectorType }}" detection="{{ onDetection }}"/>
        <Button row="1" height="40" text="Toggle Camera" tap="{{ toggleCamera }}" />
-       <Button row="2" height="40" text="Request Camera Permission" tap="{{ requestPermission }}" />
-       <Label row="3" text="{{'Detecting ' + detectorType }}"/>
-       <Button row="4" height="40" text="Change Detector Type" tap="{{ changeType }}" />
-       <Label row="5" text="{{'isPaused : ' + isPaused }}"/>
-       <Button row="6" height="40" text="Toggle Pause" tap="{{ togglePause }}" />
-       <Button row="7" height="40" text="Process Still imAGE" tap="{{ processStill }}" />
+       <Label color="red" row="2" text="{{'TorchOn ' + torchOn }}"/>
+       <Button row="3" height="40" text="Toggle Torch" tap="{{ toggleTorch }}" />
+       <Button row="4" height="40" text="Request Camera Permission" tap="{{ requestPermission }}" />
+       <Label row="5" text="{{'Detecting ' + detectorType }}"/>
+       <Button row="6" height="40" text="Change Detector Type" tap="{{ changeType }}" />
+       <Label row="7" text="{{'isPaused : ' + isPaused }}"/>
+       <Button row="8" height="40" text="Toggle Pause" tap="{{ togglePause }}" />
+       <Button row="9" height="40" text="Process Still imAGE" tap="{{ processStill }}" />
     </GridLayout>
 </Page>

packages/mlkit-barcode-scanning/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nativescript/mlkit-barcode-scanning",
-  "version": "1.0.0-alpha.4",
+  "version": "1.0.0-alpha.5",
   "description": "NativeScript MLKit Barcode Scanner module",
   "main": "index",
   "typings": "index.d.ts",

packages/mlkit-barcode-scanning/platforms/android/include.gradle

@@ -1,4 +1,4 @@
 dependencies {
-    implementation 'io.github.triniwiz.fancycamera:barcodeScanning:1.0.0-alpha.3'
+    implementation 'io.github.triniwiz.fancycamera:barcodeScanning:1.0.0-alpha.4'
     implementation 'com.google.mlkit:barcode-scanning:17.0.1'
 }

packages/mlkit-core/README.md

@@ -77,7 +77,7 @@ import { DetectionType, DetectionEvent } from '@nativescript/mlkit-core';
 import { BarcodeResult } from '@nativescript/mlkit-barcode-scanning';
 onDetection(event: DetectionEvent){
     if(event.type === DetectionType.Barcode){
-        const barcode: BarcodeResult = data;
+        const barcode: BarcodeResult[] = event.data;
     }
 }
 ```
@@ -93,7 +93,7 @@ import { DetectionType, DetectionEvent } from '@nativescript/mlkit-core';
 import { FaceResult } from '@nativescript/mlkit-face-detection';
 onDetection(event: DetectionEvent){
     if(event.type === DetectionType.Face){
-        const faces: FaceResult[] = data;
+        const faces: FaceResult[] = event.data;
     }
 }
 ```
@@ -110,7 +110,7 @@ import { DetectionType, DetectionEvent } from '@nativescript/mlkit-core';
 import { ImageLabelingResult } from '@nativescript/mlkit-image-labeling';
 onDetection(event: DetectionEvent){
     if(event.type === DetectionType.Image){
-        const labels: ImageLabelingResult[] = data;
+        const labels: ImageLabelingResult[] = event.data;
     }
 }
 ```
@@ -127,7 +127,7 @@ import { DetectionType, DetectionEvent } from '@nativescript/mlkit-core';
 import { ObjectResult } from '@nativescript/mlkit-object-detection'
 onDetection(event: DetectionEvent){
     if(event.type === DetectionType.Object){
-        const objects: ObjectResult[] = data;
+        const objects: ObjectResult[] = event.data;
     }
 }
 ```
@@ -143,7 +143,7 @@ import { DetectionType, DetectionEvent } from '@nativescript/mlkit-core';
 import { PoseResult } from '@nativescript/mlkit-pose-detection';
 onDetection(event: DetectionEvent){
     if(event.type === DetectionType.Pose){
-        const poses: PoseResult[] = data;
+        const poses: PoseResult = event.data;
     }
 }
 ```
@@ -160,7 +160,7 @@ import { DetectionType, DetectionEvent } from '@nativescript/mlkit-core';
 import { TextResult } from '@nativescript/mlkit-text-recognition';
 onDetection(event: DetectionEvent){
     if(event.type === DetectionType.Text){
-        const text: TextResult[] = data;
+        const text: TextResult = event.data;
     }
 }
 ```

packages/mlkit-core/common.ts

@@ -45,7 +45,7 @@ export enum FaceDetectionPerformanceMode {
 
 @CSSType('MLKitView')
 export class MLKitViewBase extends ContainerView {
-    public static detectionEvent = 'detection';
+    static detectionEvent = 'detection';
     cameraPosition: CameraPosition;
     detectionType: DetectionType;
     barcodeFormats: BarcodeFormats[];
@@ -57,6 +57,7 @@ export class MLKitViewBase extends ContainerView {
     objectDetectionClassify: boolean;
     torchOn: boolean;
     pause: boolean;
+    processEveryNthFrame: number;
 }
 
 export const cameraPositionProperty = new Property<MLKitViewBase, CameraPosition>({
@@ -140,11 +141,17 @@ export const torchOnProperty = new Property<MLKitViewBase, boolean>({
 torchOnProperty.register(MLKitViewBase);
 
 
-
 export const pauseProperty = new Property<MLKitViewBase, boolean>({
     name: 'pause',
     defaultValue: false,
     valueConverter: booleanConverter
 });
 
 pauseProperty.register(MLKitViewBase);
+
+export const processEveryNthFrameProperty = new Property<MLKitViewBase, number>({
+    name: 'processEveryNthFrame',
+    defaultValue: 0
+});
+
+processEveryNthFrameProperty.register(MLKitViewBase);

packages/mlkit-core/index.android.ts

@@ -1,4 +1,4 @@
-import { BarcodeFormats, barcodeFormatsProperty, CameraPosition, cameraPositionProperty, DetectionType, detectionTypeProperty, faceDetectionMinFaceSizeProperty, faceDetectionPerformanceModeProperty, faceDetectionTrackingEnabledProperty, imageLabelerConfidenceThresholdProperty, MLKitViewBase, objectDetectionClassifyProperty, objectDetectionMultipleProperty, pauseProperty, torchOnProperty } from "./common";
+import { BarcodeFormats, barcodeFormatsProperty, CameraPosition, cameraPositionProperty, DetectionType, detectionTypeProperty, faceDetectionMinFaceSizeProperty, faceDetectionPerformanceModeProperty, faceDetectionTrackingEnabledProperty, imageLabelerConfidenceThresholdProperty, MLKitViewBase, objectDetectionClassifyProperty, objectDetectionMultipleProperty, pauseProperty, processEveryNthFrameProperty, torchOnProperty } from "./common";
 import { Application, Device, Utils, AndroidActivityRequestPermissionsEventData, ImageSource } from '@nativescript/core';
 import lazy from '@nativescript/core/utils/lazy';
 import { StillImageDetectionOptions } from ".";
@@ -149,6 +149,12 @@ export class MLKitView extends MLKitViewBase {
         this.#setListeners();
     }
 
+    [processEveryNthFrameProperty.setNative](value){
+        if(this.#camera){
+            this.#camera.setProcessEveryNthFrame(value);
+        }
+    }
+
     initNativeView() {
         super.initNativeView();
         this.#setListeners();
@@ -319,6 +325,7 @@ export class MLKitView extends MLKitViewBase {
         }
     }
 
+
     [barcodeFormatsProperty.setNative](value: BarcodeFormats[]) {
         if (!BARCODE_SCANNER_SUPPORTED()) {
             return;

packages/mlkit-core/index.d.ts

@@ -1,47 +1,48 @@
-import { MLKitViewBase, DetectionType, BarcodeFormats } from "./common";
+import { MLKitViewBase, DetectionType, BarcodeFormats } from './common';
 import { EventData } from '@nativescript/core';
 
-
 export interface StillImageDetectionOptions {
-    detectorType: DetectionType
-    barcodeScanning?: {
-        barcodeFormat?: [BarcodeFormats]
-    }
-    faceDetection?: {
-        faceTracking?: boolean
-        minimumFaceSize: ?number
-        detectionMode?: "fast" | "accurate"
-        landmarkMode?: "all" | "none"
-        contourMode?: "all" | "none"
-        classificationMode?: "all" | "none"
-    }
-    imageLabeling?: {
-        confidenceThreshold?: number
-    }
-    objectDetection?: {
-        multiple: boolean;
-        classification: boolean;
-    }
-    selfieSegmentation?: {
-        enableRawSizeMask?: boolean;
-        smoothingRatio?: number;
-    }
+  detectorType: DetectionType;
+  barcodeScanning?: {
+    barcodeFormat?: [BarcodeFormats];
+  };
+  faceDetection?: {
+    faceTracking?: boolean;
+    minimumFaceSize: ?number;
+    detectionMode?: 'fast' | 'accurate';
+    landmarkMode?: 'all' | 'none';
+    contourMode?: 'all' | 'none';
+    classificationMode?: 'all' | 'none';
+  };
+  imageLabeling?: {
+    confidenceThreshold?: number;
+  };
+  objectDetection?: {
+    multiple: boolean;
+    classification: boolean;
+  };
+  selfieSegmentation?: {
+    enableRawSizeMask?: boolean;
+    smoothingRatio?: number;
+  };
 }
 
 export interface DetectionEvent extends EventData {
-    data: { [key: string]: any };
-    type: DetectionType
+  data: { [key: string]: any } | Array<{ [key: string]: any }>;
+  type: DetectionType;
 }
+
+
 export { BarcodeFormats, barcodeFormatsProperty, CameraPosition, cameraPositionProperty, DetectionType, faceDetectionMinFaceSizeProperty, faceDetectionPerformanceModeProperty, faceDetectionTrackingEnabledProperty, imageLabelerConfidenceThresholdProperty as imageLablerConfidenceThresholdProperty, objectDetectionClassifyProperty, objectDetectionMultipleProperty } from './common';
 export declare class MLKitView extends MLKitViewBase {
-    static isAvailable(): boolean;
-    static detectionEvent: string;
-    stopPreview(): void;
-    toggleCamera(): void;
-    startPreview(): void;
-    requestCameraPermission(): Promise<void>;
-    hasCameraPermission(): boolean;
-    on(event: 'detection', callback: (args: DetectionEvent) => void, thisArg?: any);
+  static isAvailable(): boolean;
+  static detectionEvent: string;
+  stopPreview(): void;
+  toggleCamera(): void;
+  startPreview(): void;
+  requestCameraPermission(): Promise<void>;
+  hasCameraPermission(): boolean;
+  on(event: 'detection', callback: (args: DetectionEvent) => void, thisArg?: any);
 }
 
-export function detectWithStillImage(image: any, options?: StillImageDetectionOptions): Promise<{ [key: string]: any }>
+export function detectWithStillImage(image: any, options?: StillImageDetectionOptions): Promise<{ [key: string]: any }>;