Flutter - Isolate : NoSuchMethodError: The method 'toRawHandle' was called on null

Question

Goal

I'm trying to perform real-time object detection with a Flutter app, using Tensorflow 2, with a SSD Mobilenet V2 model

• I managed to get this work, using this git repo
• However, I am encountering some latencies on the camera output display, so I decided to call the detection method inside a Isolate

What I've tried

• The recognition method is Tflite.detectObjectOnFrame() from tflite plugin
• As far as I understand, I have to use a particular Isolate in order to use plugin methods in another thread, so I'm using the isolate_handler plugin

The Code

Camera.dart

import 'dart:async';
import 'dart:typed_data';

import 'package:flutter/material.dart';
import 'package:camera/camera.dart';
import 'package:tflite/tflite.dart';
import 'package:isolate_handler/isolate_handler.dart';
import 'dart:math' as math;

typedef void Callback(List<dynamic> list, int h, int w);

class Camera extends StatefulWidget {
  final List<CameraDescription> cameras;
  final Callback setRecognitions;

  Camera(this.cameras, this.setRecognitions);

  @override
  _CameraState createState() => new _CameraState();
}

class _CameraState extends State<Camera> {
  CameraController controller;
  bool isDetecting = false;

  final isolates = IsolateHandler();
  int startTime;

  @override
  void initState() {
    super.initState();

    if (widget.cameras == null || widget.cameras.length < 1) {
      print('No camera is found');
    } else {
      controller = new CameraController(
        widget.cameras[0],
        ResolutionPreset.high,
      );
      controller.initialize().then((_) {
        if (!mounted) {
          return;
        }
        setState(() {});

        controller.startImageStream((CameraImage img) {
          if (!isDetecting) {
            isDetecting = true;

            startTime = new DateTime.now().millisecondsSinceEpoch;

            //HERE IS THE ISOLATE
            isolates.spawn<List<dynamic>>(
                entryPoint,
                name: 'object_detection',
                onInitialized: () => isolates.send(img, to: 'object_detection'));
            }
          }
        );
      });
    }

  }

  @override
  void dispose() {
    controller?.dispose();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    if (controller == null || !controller.value.isInitialized) {
      return Container();
    }

    var tmp = MediaQuery.of(context).size;
    var screenH = math.max(tmp.height, tmp.width);
    var screenW = math.min(tmp.height, tmp.width);
    tmp = controller.value.previewSize;
    var previewH = math.max(tmp.height, tmp.width);
    var previewW = math.min(tmp.height, tmp.width);
    var screenRatio = screenH / screenW;
    var previewRatio = previewH / previewW;

    return OverflowBox(
      maxHeight:
          screenRatio > previewRatio ? screenH : screenW / previewW * previewH,
      maxWidth:
          screenRatio > previewRatio ? screenH / previewH * previewW : screenW,
      child: CameraPreview(controller),
    );
  }

  void entryPoint(Map<String, dynamic> context) {
    final messenger = HandledIsolate.initialize(context);

    messenger.listen((img) async {
      // HERE IS THE METHOD I USE FROM TFLITE PLUGIN
      var recognitions = Tflite.detectObjectOnFrame(
        bytesList: img.planes.map((plane) {
          return plane.bytes;
        }).toList(),
        model: "SSDMobileNet",
        imageHeight: img.height,
        imageWidth: img.width,
        imageMean: 127.5,
        imageStd: 127.5,
        numResultsPerClass: 1,
        threshold: 0.4,
      );
      messenger.send(recognitions);
    });
  }

  void setRecognitions(List<dynamic> recognitions) {
    int endTime = new DateTime.now().millisecondsSinceEpoch;
    print("Detection took ${endTime - startTime}");

    widget.setRecognitions(recognitions, /*img.height, img.width*/1280, 720);

    isDetecting = false;
    isolates.kill('object_detection');
  }
}

The error I get

E/flutter ( 7960): [ERROR:flutter/lib/ui/ui_dart_state.cc(177)] Unhandled Exception: NoSuchMethodError: The method 'toRawHandle' was called on null.
E/flutter ( 7960): Receiver: null
E/flutter ( 7960): Tried calling: toRawHandle()
E/flutter ( 7960): #0      Object.noSuchMethod (dart:core-patch/object_patch.dart:51:5)
E/flutter ( 7960): #1      FlutterIsolate.spawn (package:flutter_isolate/flutter_isolate.dart:25:55)
E/flutter ( 7960): #2      HandledIsolate._init (package:isolate_handler/src/handled_isolate.dart:174:37)
E/flutter ( 7960): #3      new HandledIsolate (package:isolate_handler/src/handled_isolate.dart:108:5)
E/flutter ( 7960): #4      IsolateHandler.spawn (package:isolate_handler/isolate_handler.dart:167:22)
E/flutter ( 7960): #5      _CameraState.initState.<anonymous closure>.<anonymous closure> (package:flutter_realtime_detection/camera.dart:56:22)
E/flutter ( 7960): #6      CameraController.startImageStream.<anonymous closure> (package:camera/camera.dart:412:20)
E/flutter ( 7960): #7      _rootRunUnary (dart:async/zone.dart:1198:47)
E/flutter ( 7960): #8      _CustomZone.runUnary (dart:async/zone.dart:1100:19)
E/flutter ( 7960): #9      _CustomZone.runUnaryGuarded (dart:async/zone.dart:1005:7)
E/flutter ( 7960): #10     _BufferingStreamSubscription._sendData (dart:async/stream_impl.dart:357:11)
E/flutter ( 7960): #11     _DelayedData.perform (dart:async/stream_impl.dart:611:14)
E/flutter ( 7960): #12     _StreamImplEvents.handleNext (dart:async/stream_impl.dart:730:11)
E/flutter ( 7960): #13     _PendingEvents.schedule.<anonymous closure> (dart:async/stream_impl.dart:687:7)
E/flutter ( 7960): #14     _rootRun (dart:async/zone.dart:1182:47)
E/flutter ( 7960): #15     _CustomZone.run (dart:async/zone.dart:1093:19)
E/flutter ( 7960): #16     _CustomZone.runGuarded (dart:async/zone.dart:997:7)
E/flutter ( 7960): #17     _CustomZone.bindCallbackGuarded.<anonymous closure> (dart:async/zone.dart:1037:23)
E/flutter ( 7960): #18     _rootRun (dart:async/zone.dart:1190:13)
E/flutter ( 7960): #19     _CustomZone.run (dart:async/zone.dart:1093:19)
E/flutter ( 7960): #20     _CustomZone.runGuarded (dart:async/zone.dart:997:7)
E/flutter ( 7960): #21     _CustomZone.bindCallbackGuarded.<anonymous closure> (dart:async/zone.dart:1037:23)
E/flutter ( 7960): #22     _microtaskLoop (dart:async/schedule_microtask.dart:41:21)
E/flutter ( 7960): #23     _startMicrotaskLoop (dart:async/schedule_microtask.dart:50:5)

Any idea of what I am doing wrong ? Thank you.

Answer 1

Isolates don't share memory between them, which means you can only use primitive values and static/top-level functions. The entryPoint is not a static/top-level function in your case.

Check out the easy_isolate plugin, it provides an easy way to work with isolates with well-explained documentation. But doesn't call platform plugins.

https://pub.dev/packages/easy_isolate