From 11bf729eea12ccc7935d583b750291023c0ee326 Mon Sep 17 00:00:00 2001 From: Levent Koch Date: Sat, 19 Sep 2020 12:36:58 +0200 Subject: [PATCH 1/8] adding coral tensorflow plugin --- plugins/tensorflow-coral/.gitignore | 4 + plugins/tensorflow-coral/INSTALL.sh | 30 ++++ plugins/tensorflow-coral/README.md | 72 ++++++++ plugins/tensorflow-coral/conf.sample.json | 9 + plugins/tensorflow-coral/detect.py | 163 ++++++++++++++++++ plugins/tensorflow-coral/detect_image.py | 119 +++++++++++++ plugins/tensorflow-coral/package.json | 33 ++++ .../tensorflow-coral/shinobi-tensorflow.js | 122 +++++++++++++ 8 files changed, 552 insertions(+) create mode 100644 plugins/tensorflow-coral/.gitignore create mode 100755 plugins/tensorflow-coral/INSTALL.sh create mode 100644 plugins/tensorflow-coral/README.md create mode 100644 plugins/tensorflow-coral/conf.sample.json create mode 100644 plugins/tensorflow-coral/detect.py create mode 100644 plugins/tensorflow-coral/detect_image.py create mode 100644 plugins/tensorflow-coral/package.json create mode 100644 plugins/tensorflow-coral/shinobi-tensorflow.js diff --git a/plugins/tensorflow-coral/.gitignore b/plugins/tensorflow-coral/.gitignore new file mode 100644 index 00000000..f6c52d28 --- /dev/null +++ b/plugins/tensorflow-coral/.gitignore @@ -0,0 +1,4 @@ +conf.json +dist +models +.env diff --git a/plugins/tensorflow-coral/INSTALL.sh b/plugins/tensorflow-coral/INSTALL.sh new file mode 100755 index 00000000..40ebc13b --- /dev/null +++ b/plugins/tensorflow-coral/INSTALL.sh @@ -0,0 +1,30 @@ +#!/bin/bash +DIR=`dirname $0` +echo "Installing coral dependencies..." +echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | sudo tee /etc/apt/sources.list.d/coral-edgetpu.list +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +sudo apt-get update +sudo apt-get install libedgetpu1-max +echo "Coral dependencies installed." +echo "Getting coral object detection models..." +mkdir -p models +wget "https://github.com/google-coral/edgetpu/raw/master/test_data/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite" +mv ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite models/ +wget "https://dl.google.com/coral/canned_models/coco_labels.txt" +mv coco_labels.txt models/ +echo "Models downloaded." + + +npm install yarn -g --unsafe-perm --force +npm install --unsafe-perm +if [ ! -e "./conf.json" ]; then + echo "Creating conf.json" + sudo cp conf.sample.json conf.json +else + echo "conf.json already exists..." +fi +echo "Adding Random Plugin Key to Main Configuration" +node $DIR/../../tools/modifyConfigurationForPlugin.js tensorflow-coral key=$(head -c 64 < /dev/urandom | sha256sum | awk '{print substr($1,1,60)}') + +echo "!!!IMPORTANT!!!" +echo "IF YOU DON'T HAVE INSTALLED CORAL DEPENDENCIES BEFORE, YOU NEED TO PLUG OUT AND THEN PLUG IN YOUR CORAL USB ACCELERATOR BEFORE USING THIS PLUGIN" diff --git a/plugins/tensorflow-coral/README.md b/plugins/tensorflow-coral/README.md new file mode 100644 index 00000000..56471ba6 --- /dev/null +++ b/plugins/tensorflow-coral/README.md @@ -0,0 +1,72 @@ +# TensorFlowCoral.js + +**Ubuntu and CentOS only** + +Go to the Shinobi directory. **/home/Shinobi** is the default directory. + +``` +cd /home/Shinobi/plugins/tensorflow +``` + +Copy the config file. + +``` +sh INSTALL.sh +``` + +IF YOU DON'T HAVE INSTALLED CORAL DEPENDENCIES BEFORE, YOU NEED TO PLUG OUT AND THEN PLUG IN YOUR CORAL USB ACCELERATOR BEFORE USING THIS PLUGIN! + +Start the plugin. + +``` +pm2 start shinobi-tensorflow.js +``` + +Doing this will reveal options in the monitor configuration. Shinobi does not need to be restarted when a plugin is initiated or stopped. + +## Run the plugin as a Host +> The main app (Shinobi) will be the client and the plugin will be the host. The purpose of allowing this method is so that you can use one plugin for multiple Shinobi instances. Allowing you to easily manage connections without starting multiple processes. + +Edit your plugins configuration file. Set the `hostPort` **to be different** than the `listening port for camera.js`. + +``` +nano conf.json +``` + +Here is a sample of a Host configuration for the plugin. + - `plug` is the name of the plugin corresponding in the main configuration file. + - `https` choose if you want to use SSL or not. Default is `false`. + - `hostPort` can be any available port number. **Don't make this the same port number as Shinobi.** Default is `8082`. + - `type` tells the main application (Shinobi) what kind of plugin it is. In this case it is a detector. + +``` +{ + "plug":"Tensorflow", + "hostPort":8082, + "key":"Tensorflow123123", + "mode":"host", + "type":"detector" +} +``` + +Now modify the **main configuration file** located in the main directory of Shinobi. + +``` +nano conf.json +``` + +Add the `plugins` array if you don't already have it. Add the following *object inside the array*. + +``` + "plugins":[ + { + "id" : "Tensorflow", + "https" : false, + "host" : "localhost", + "port" : 8082, + "key" : "Tensorflow123123", + "mode" : "host", + "type" : "detector" + } + ], +``` diff --git a/plugins/tensorflow-coral/conf.sample.json b/plugins/tensorflow-coral/conf.sample.json new file mode 100644 index 00000000..b832b7bb --- /dev/null +++ b/plugins/tensorflow-coral/conf.sample.json @@ -0,0 +1,9 @@ +{ + "plug":"TensorflowCoral", + "host":"localhost", + "port":8080, + "hostPort":8082, + "key":"change_this_to_something_very_random____make_sure_to_match__/plugins/tensorflow-coral/conf.json", + "mode":"client", + "type":"detector" +} diff --git a/plugins/tensorflow-coral/detect.py b/plugins/tensorflow-coral/detect.py new file mode 100644 index 00000000..7f35b6b5 --- /dev/null +++ b/plugins/tensorflow-coral/detect.py @@ -0,0 +1,163 @@ +# Lint as: python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions to work with detection models.""" + +import collections +import numpy as np + +Object = collections.namedtuple('Object', ['id', 'score', 'bbox']) + + +class BBox(collections.namedtuple('BBox', ['xmin', 'ymin', 'xmax', 'ymax'])): + """Bounding box. + + Represents a rectangle which sides are either vertical or horizontal, parallel + to the x or y axis. + """ + __slots__ = () + + @property + def width(self): + """Returns bounding box width.""" + return self.xmax - self.xmin + + @property + def height(self): + """Returns bounding box height.""" + return self.ymax - self.ymin + + @property + def area(self): + """Returns bound box area.""" + return self.width * self.height + + @property + def valid(self): + """Returns whether bounding box is valid or not. + + Valid bounding box has xmin <= xmax and ymin <= ymax which is equivalent to + width >= 0 and height >= 0. + """ + return self.width >= 0 and self.height >= 0 + + def scale(self, sx, sy): + """Returns scaled bounding box.""" + return BBox(xmin=sx * self.xmin, + ymin=sy * self.ymin, + xmax=sx * self.xmax, + ymax=sy * self.ymax) + + def translate(self, dx, dy): + """Returns translated bounding box.""" + return BBox(xmin=dx + self.xmin, + ymin=dy + self.ymin, + xmax=dx + self.xmax, + ymax=dy + self.ymax) + + def map(self, f): + """Returns bounding box modified by applying f for each coordinate.""" + return BBox(xmin=f(self.xmin), + ymin=f(self.ymin), + xmax=f(self.xmax), + ymax=f(self.ymax)) + + @staticmethod + def intersect(a, b): + """Returns the intersection of two bounding boxes (may be invalid).""" + return BBox(xmin=max(a.xmin, b.xmin), + ymin=max(a.ymin, b.ymin), + xmax=min(a.xmax, b.xmax), + ymax=min(a.ymax, b.ymax)) + + @staticmethod + def union(a, b): + """Returns the union of two bounding boxes (always valid).""" + return BBox(xmin=min(a.xmin, b.xmin), + ymin=min(a.ymin, b.ymin), + xmax=max(a.xmax, b.xmax), + ymax=max(a.ymax, b.ymax)) + + @staticmethod + def iou(a, b): + """Returns intersection-over-union value.""" + intersection = BBox.intersect(a, b) + if not intersection.valid: + return 0.0 + area = intersection.area + return area / (a.area + b.area - area) + + +def input_size(interpreter): + """Returns input image size as (width, height) tuple.""" + _, height, width, _ = interpreter.get_input_details()[0]['shape'] + return width, height + + +def input_tensor(interpreter): + """Returns input tensor view as numpy array of shape (height, width, 3).""" + tensor_index = interpreter.get_input_details()[0]['index'] + return interpreter.tensor(tensor_index)()[0] + + +def set_input(interpreter, size, resize): + """Copies a resized and properly zero-padded image to the input tensor. + + Args: + interpreter: Interpreter object. + size: original image size as (width, height) tuple. + resize: a function that takes a (width, height) tuple, and returns an RGB + image resized to those dimensions. + Returns: + Actual resize ratio, which should be passed to `get_output` function. + """ + width, height = input_size(interpreter) + w, h = size + scale = min(width / w, height / h) + w, h = int(w * scale), int(h * scale) + tensor = input_tensor(interpreter) + tensor.fill(0) # padding + _, _, channel = tensor.shape + tensor[:h, :w] = np.reshape(resize((w, h)), (h, w, channel)) + return scale, scale + + +def output_tensor(interpreter, i): + """Returns output tensor view.""" + tensor = interpreter.tensor(interpreter.get_output_details()[i]['index'])() + return np.squeeze(tensor) + + +def get_output(interpreter, score_threshold, image_scale=(1.0, 1.0)): + """Returns list of detected objects.""" + boxes = output_tensor(interpreter, 0) + class_ids = output_tensor(interpreter, 1) + scores = output_tensor(interpreter, 2) + count = int(output_tensor(interpreter, 3)) + + width, height = input_size(interpreter) + image_scale_x, image_scale_y = image_scale + sx, sy = width / image_scale_x, height / image_scale_y + + def make(i): + ymin, xmin, ymax, xmax = boxes[i] + return Object( + id=int(class_ids[i]), + score=float(scores[i]), + bbox=BBox(xmin=xmin, + ymin=ymin, + xmax=xmax, + ymax=ymax).scale(sx, sy).map(int)) + + return [make(i) for i in range(count) if scores[i] >= score_threshold] diff --git a/plugins/tensorflow-coral/detect_image.py b/plugins/tensorflow-coral/detect_image.py new file mode 100644 index 00000000..bd250a6b --- /dev/null +++ b/plugins/tensorflow-coral/detect_image.py @@ -0,0 +1,119 @@ +# Lint as: python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Example using TF Lite to detect objects in a given image.""" + +import argparse +import time +import sys +from PIL import Image +from PIL import ImageDraw +from io import BytesIO, StringIO +import time +import base64 +import json +import detect +import tflite_runtime.interpreter as tflite +import platform + +EDGETPU_SHARED_LIB = { + 'Linux': 'libedgetpu.so.1', + 'Darwin': 'libedgetpu.1.dylib', + 'Windows': 'edgetpu.dll' +}[platform.system()] + + +def load_labels(path, encoding='utf-8'): + """Loads labels from file (with or without index numbers). + + Args: + path: path to label file. + encoding: label file encoding. + Returns: + Dictionary mapping indices to labels. + """ + with open(path, 'r', encoding=encoding) as f: + lines = f.readlines() + if not lines: + return {} + + if lines[0].split(' ', maxsplit=1)[0].isdigit(): + pairs = [line.split(' ', maxsplit=1) for line in lines] + return {int(index): label.strip() for index, label in pairs} + else: + return {index: line.strip() for index, line in enumerate(lines)} + + +def make_interpreter(model_file): + model_file, *device = model_file.split('@') + return tflite.Interpreter( + model_path=model_file, + experimental_delegates=[ + tflite.load_delegate(EDGETPU_SHARED_LIB, + {'device': device[0]} if device else {}) + ]) + + +def draw_objects(draw, objs, labels): + """Draws the bounding box and label for each object.""" + for obj in objs: + bbox = obj.bbox + draw.rectangle([(bbox.xmin, bbox.ymin), (bbox.xmax, bbox.ymax)], + outline='red') + draw.text((bbox.xmin + 10, bbox.ymin + 10), + '%s\n%.2f' % (labels.get(obj.id, obj.id), obj.score), + fill='red') + + +def printInfo(text): + print(json.dumps({"type": "info", "data": text})) + +def printError(text): + print(json.dumps({"type": "error", "data": text})) + +def printData(array, time): + print(json.dumps({"type": "data", "data": array, "time": time})) + +def main(): + labels = load_labels("models/coco_labels.txt") + interpreter = make_interpreter("models/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite") + interpreter.allocate_tensors() + threshold = 0.4 + printInfo("ready") + while True: + line = sys.stdin.readline().rstrip("\n") + try: + rawImage = BytesIO(base64.b64decode(line)) + image = Image.open(rawImage) + scale = detect.set_input(interpreter, image.size, + lambda size: image.resize(size, Image.ANTIALIAS)) + + start = time.perf_counter() + interpreter.invoke() + inference_time = time.perf_counter() - start + objs = detect.get_output(interpreter, threshold, scale) + output = [] + for obj in objs: + label = labels.get(obj.id, obj.id) + labelID = obj[0] + score = obj[1] + bbox = obj[2] + output.append({"bbox": bbox, "class": label, "score": score}) + printData(output, (inference_time * 1000)) + except Exception as e: + printError(str(e)) + + +if __name__ == '__main__': + main() diff --git a/plugins/tensorflow-coral/package.json b/plugins/tensorflow-coral/package.json new file mode 100644 index 00000000..e79dbef6 --- /dev/null +++ b/plugins/tensorflow-coral/package.json @@ -0,0 +1,33 @@ +{ + "name": "shinobi-tensorflow-coral", + "author": "Shinob Systems, Moinul Alam | dermodmaster, Levent Koch", + "version": "1.0.0", + "description": "Object Detection plugin based on tensorflow using Google Coral USB Accelerator", + "main": "shinobi-tensorflow.js", + "dependencies": { + "dotenv": "^8.2.0", + "express": "^4.16.2", + "moment": "^2.19.2", + "socket.io": "^2.0.4", + "socket.io-client": "^1.7.4" + }, + "devDependencies": {}, + "bin": "shinobi-tensorflow.js", + "scripts": { + "package": "pkg package.json -t linux,macos,win --out-path dist", + "package-x64": "pkg package.json -t linux-x64,macos-x64,win-x64 --out-path dist/x64", + "package-x86": "pkg package.json -t linux-x86,macos-x86,win-x86 --out-path dist/x86", + "package-armv6": "pkg package.json -t linux-armv6,macos-armv6,win-armv6 --out-path dist/armv6", + "package-armv7": "pkg package.json -t linux-armv7,macos-armv7,win-armv7 --out-path dist/armv7", + "package-all": "npm run package && npm run package-x64 && npm run package-x86 && npm run package-armv6 && npm run package-armv7" + }, + "pkg": { + "targets": [ + "node12" + ], + "scripts": [ + "../pluginBase.js" + ], + "assets": [] + } +} diff --git a/plugins/tensorflow-coral/shinobi-tensorflow.js b/plugins/tensorflow-coral/shinobi-tensorflow.js new file mode 100644 index 00000000..12c57ef2 --- /dev/null +++ b/plugins/tensorflow-coral/shinobi-tensorflow.js @@ -0,0 +1,122 @@ +// +// Shinobi - Tensorflow Plugin +// Copyright (C) 2016-2025 Moe Alam, moeiscool +// Copyright (C) 2020 Levent Koch, dermodmaster +// +// # Donate +// +// If you like what I am doing here and want me to continue please consider donating :) +// PayPal : paypal@m03.ca +// +// Base Init >> +var fs = require('fs'); +var config = require('./conf.json') +var dotenv = require('dotenv').config() +var s +try{ + s = require('../pluginBase.js')(__dirname,config) +}catch(err){ + console.log(err) + try{ + s = require('./pluginBase.js')(__dirname,config) + }catch(err){ + console.log(err) + return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + } +} +// Base Init />> +var spawn = require('child_process').spawn; +var child = spawn('python3', ['-u', 'detect_image.py']); +var ready = false; + +child.stdout.on('data', function (data) { + var rawString = data.toString('utf8'); + console.log(rawString); + var messages = rawString.split('\n') + messages.forEach((message) => { + if(message === "") return; + var obj = JSON.parse(message) + if (obj.type === "error") { + throw message.data; + } + + if (obj.type === "info" && obj.data === "ready") { + console.log("set ready true") + ready = true; + } else { + if(obj.type !== "data" && obj.type !== "info"){ + throw "Unexpected message: "+rawString; + } + } + }) +}) + +const emptyDataObject = {data: [], type: undefined, time: 0}; + +async function process(buffer, type) { + const startTime = new Date(); + if(!ready){ + return emptyDataObject; + } + ready = false; + child.stdin.write(buffer.toString('base64')+'\n'); + + var message = null; + await new Promise(resolve => { + child.stdout.once('data', (data) => { + var rawString = data.toString('utf8').split("\n")[0]; + try { + message = JSON.parse(rawString) + } + catch (e) { + message = {data: []}; + } + resolve(); + }); + }) + const data = message.data; + ready = true; + return { + data: data, + type: type, + time: new Date() - startTime + } +} + + +s.detectObject = function(buffer,d,tx,frameLocation,callback){ + process( buffer).then((resp)=>{ + var results = resp.data + if(results[0]){ + var mats = [] + results.forEach(function(v){ + mats.push({ + x: v.bbox[0], + y: v.bbox[1], + width: v.bbox[2], + height: v.bbox[3], + tag: v.class, + confidence: v.score, + }) + }) + var isObjectDetectionSeparate = d.mon.detector_pam === '1' && d.mon.detector_use_detect_object === '1' + var width = parseFloat(isObjectDetectionSeparate && d.mon.detector_scale_y_object ? d.mon.detector_scale_y_object : d.mon.detector_scale_y) + var height = parseFloat(isObjectDetectionSeparate && d.mon.detector_scale_x_object ? d.mon.detector_scale_x_object : d.mon.detector_scale_x) + tx({ + f:'trigger', + id:d.id, + ke:d.ke, + details:{ + plug:config.plug, + name:'Tensorflow', + reason:'object', + matrices:mats, + imgHeight:width, + imgWidth:height, + time: resp.time + } + }) + } + callback() + }) +} From dfb1559ecd1530def708371eb1854296441f7da6 Mon Sep 17 00:00:00 2001 From: Levent Koch Date: Sat, 19 Sep 2020 17:07:09 +0200 Subject: [PATCH 2/8] update readme and install script --- plugins/tensorflow-coral/INSTALL.sh | 1 + plugins/tensorflow-coral/README.md | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/plugins/tensorflow-coral/INSTALL.sh b/plugins/tensorflow-coral/INSTALL.sh index 40ebc13b..8d16d475 100755 --- a/plugins/tensorflow-coral/INSTALL.sh +++ b/plugins/tensorflow-coral/INSTALL.sh @@ -5,6 +5,7 @@ echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | sud curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - sudo apt-get update sudo apt-get install libedgetpu1-max +sudo apt-get install libatlas-base-dev echo "Coral dependencies installed." echo "Getting coral object detection models..." mkdir -p models diff --git a/plugins/tensorflow-coral/README.md b/plugins/tensorflow-coral/README.md index 56471ba6..d16e0662 100644 --- a/plugins/tensorflow-coral/README.md +++ b/plugins/tensorflow-coral/README.md @@ -8,6 +8,16 @@ Go to the Shinobi directory. **/home/Shinobi** is the default directory. cd /home/Shinobi/plugins/tensorflow ``` +Install TensorFlows python version first: +https://www.tensorflow.org/lite/guide/python +Make sure that you are downloading the correct file for your system architecture and python version. + +Install other python dependencies +``` +pip install pillow +pip install numpy +``` + Copy the config file. ``` From 672c6926c45b35955e0ea68eb1a2fec79518d936 Mon Sep 17 00:00:00 2001 From: Levent Koch Date: Sat, 19 Sep 2020 17:10:25 +0200 Subject: [PATCH 3/8] update install script --- plugins/tensorflow-coral/INSTALL.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/tensorflow-coral/INSTALL.sh b/plugins/tensorflow-coral/INSTALL.sh index 8d16d475..8bbd36f8 100755 --- a/plugins/tensorflow-coral/INSTALL.sh +++ b/plugins/tensorflow-coral/INSTALL.sh @@ -10,7 +10,7 @@ echo "Coral dependencies installed." echo "Getting coral object detection models..." mkdir -p models wget "https://github.com/google-coral/edgetpu/raw/master/test_data/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite" -mv ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite models/ +mv mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite models/ wget "https://dl.google.com/coral/canned_models/coco_labels.txt" mv coco_labels.txt models/ echo "Models downloaded." From 8093e42461a6b6608e407b525ecaade1b98ab159 Mon Sep 17 00:00:00 2001 From: Levent K Date: Sat, 10 Oct 2020 22:32:53 +0000 Subject: [PATCH 4/8] Some fixes in shinobi-tensorflow.js --- plugins/tensorflow-coral/shinobi-tensorflow.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/tensorflow-coral/shinobi-tensorflow.js b/plugins/tensorflow-coral/shinobi-tensorflow.js index 12c57ef2..3840cc80 100644 --- a/plugins/tensorflow-coral/shinobi-tensorflow.js +++ b/plugins/tensorflow-coral/shinobi-tensorflow.js @@ -28,15 +28,24 @@ try{ var spawn = require('child_process').spawn; var child = spawn('python3', ['-u', 'detect_image.py']); var ready = false; +var lastStatusLog = new Date(); + +child.on('close', function() { + child = spawn('python3', ['-u', 'detect_image.py']); +}); child.stdout.on('data', function (data) { var rawString = data.toString('utf8'); - console.log(rawString); + if(new Date() - lastStatusLog > 5000){ + lastStatusLog = new Date(); + console.log(rawString, new Date()); + } var messages = rawString.split('\n') messages.forEach((message) => { if(message === "") return; var obj = JSON.parse(message) if (obj.type === "error") { + console.log("Script got error: "+message.data, new Date()); throw message.data; } @@ -87,7 +96,8 @@ async function process(buffer, type) { s.detectObject = function(buffer,d,tx,frameLocation,callback){ process( buffer).then((resp)=>{ var results = resp.data - if(results[0]){ + //console.log(resp.time) + if(Array.isArray(results) && results[0]){ var mats = [] results.forEach(function(v){ mats.push({ From 4ce70008516a0d2cb858aa856bf9fad3d0386d24 Mon Sep 17 00:00:00 2001 From: Levent K Date: Sun, 11 Oct 2020 08:01:54 +0000 Subject: [PATCH 5/8] Update detect_image.py --- plugins/tensorflow-coral/detect_image.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/tensorflow-coral/detect_image.py b/plugins/tensorflow-coral/detect_image.py index bd250a6b..62878e39 100644 --- a/plugins/tensorflow-coral/detect_image.py +++ b/plugins/tensorflow-coral/detect_image.py @@ -33,7 +33,6 @@ EDGETPU_SHARED_LIB = { 'Windows': 'edgetpu.dll' }[platform.system()] - def load_labels(path, encoding='utf-8'): """Loads labels from file (with or without index numbers). From e2fb577878159e57e0bbf812338822c1a70016c0 Mon Sep 17 00:00:00 2001 From: Levent K Date: Sun, 11 Oct 2020 08:09:36 +0000 Subject: [PATCH 6/8] Adjust model file name --- plugins/tensorflow-coral/detect_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/tensorflow-coral/detect_image.py b/plugins/tensorflow-coral/detect_image.py index 62878e39..e9322328 100644 --- a/plugins/tensorflow-coral/detect_image.py +++ b/plugins/tensorflow-coral/detect_image.py @@ -86,7 +86,7 @@ def printData(array, time): def main(): labels = load_labels("models/coco_labels.txt") - interpreter = make_interpreter("models/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite") + interpreter = make_interpreter("models/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite") interpreter.allocate_tensors() threshold = 0.4 printInfo("ready") From 18b3ae1c45e8d03671b55e98a6fbbab58a73534f Mon Sep 17 00:00:00 2001 From: Levent K Date: Mon, 12 Oct 2020 16:48:22 +0000 Subject: [PATCH 7/8] Update INSTALL.sh --- plugins/tensorflow-coral/INSTALL.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/tensorflow-coral/INSTALL.sh b/plugins/tensorflow-coral/INSTALL.sh index 8bbd36f8..8d16d475 100755 --- a/plugins/tensorflow-coral/INSTALL.sh +++ b/plugins/tensorflow-coral/INSTALL.sh @@ -10,7 +10,7 @@ echo "Coral dependencies installed." echo "Getting coral object detection models..." mkdir -p models wget "https://github.com/google-coral/edgetpu/raw/master/test_data/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite" -mv mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite models/ +mv ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite models/ wget "https://dl.google.com/coral/canned_models/coco_labels.txt" mv coco_labels.txt models/ echo "Models downloaded." From db13c162606001dd9ee057e43b53476d61ac7f2b Mon Sep 17 00:00:00 2001 From: Levent K Date: Wed, 28 Oct 2020 15:02:53 +0000 Subject: [PATCH 8/8] Refactored shinobi-tensorflow.js (by moe and thellamafarm) --- .../tensorflow-coral/shinobi-tensorflow.js | 140 ++++++++++-------- 1 file changed, 77 insertions(+), 63 deletions(-) diff --git a/plugins/tensorflow-coral/shinobi-tensorflow.js b/plugins/tensorflow-coral/shinobi-tensorflow.js index 3840cc80..f7d5d8b9 100644 --- a/plugins/tensorflow-coral/shinobi-tensorflow.js +++ b/plugins/tensorflow-coral/shinobi-tensorflow.js @@ -13,63 +13,77 @@ var fs = require('fs'); var config = require('./conf.json') var dotenv = require('dotenv').config() var s -try{ - s = require('../pluginBase.js')(__dirname,config) -}catch(err){ +try { + s = require('../pluginBase.js')(__dirname, config) +} catch (err) { console.log(err) - try{ - s = require('./pluginBase.js')(__dirname,config) - }catch(err){ + try { + s = require('./pluginBase.js')(__dirname, config) + } catch (err) { console.log(err) - return console.log(config.plug,'Plugin start has failed. pluginBase.js was not found.') + return console.log(config.plug, 'Plugin start has failed. pluginBase.js was not found.') } } -// Base Init />> -var spawn = require('child_process').spawn; -var child = spawn('python3', ['-u', 'detect_image.py']); + + var ready = false; -var lastStatusLog = new Date(); - -child.on('close', function() { - child = spawn('python3', ['-u', 'detect_image.py']); -}); - -child.stdout.on('data', function (data) { - var rawString = data.toString('utf8'); - if(new Date() - lastStatusLog > 5000){ - lastStatusLog = new Date(); - console.log(rawString, new Date()); - } - var messages = rawString.split('\n') - messages.forEach((message) => { - if(message === "") return; - var obj = JSON.parse(message) - if (obj.type === "error") { - console.log("Script got error: "+message.data, new Date()); - throw message.data; +const spawn = require('child_process').spawn; +var child = null +function respawn() { + + console.log("respawned python",(new Date())) + const theChild = spawn('python3', ['-u', 'detect_image.py']); + + var lastStatusLog = new Date(); + + theChild.on('exit', () => { + child = respawn(); + }); + + theChild.stdout.on('data', function (data) { + var rawString = data.toString('utf8'); + if (new Date() - lastStatusLog > 5000) { + lastStatusLog = new Date(); + console.log(rawString, new Date()); } - - if (obj.type === "info" && obj.data === "ready") { - console.log("set ready true") - ready = true; - } else { - if(obj.type !== "data" && obj.type !== "info"){ - throw "Unexpected message: "+rawString; + var messages = rawString.split('\n') + messages.forEach((message) => { + if (message === "") return; + var obj = JSON.parse(message) + if (obj.type === "error") { + console.log("Script got error: " + message.data, new Date()); + throw message.data; } - } + + if (obj.type === "info" && obj.data === "ready") { + console.log("set ready true") + ready = true; + } else { + if (obj.type !== "data" && obj.type !== "info") { + throw "Unexpected message: " + rawString; + } + } + }) }) -}) - -const emptyDataObject = {data: [], type: undefined, time: 0}; - + return theChild +} + + + + +// Base Init />> +child = respawn(); + +const emptyDataObject = { data: [], type: undefined, time: 0 }; + async function process(buffer, type) { const startTime = new Date(); - if(!ready){ + if (!ready) { return emptyDataObject; } ready = false; - child.stdin.write(buffer.toString('base64')+'\n'); - + child.stdin.write(buffer.toString('base64') + '\n'); + var message = null; await new Promise(resolve => { child.stdout.once('data', (data) => { @@ -78,7 +92,7 @@ async function process(buffer, type) { message = JSON.parse(rawString) } catch (e) { - message = {data: []}; + message = { data: [] }; } resolve(); }); @@ -91,15 +105,15 @@ async function process(buffer, type) { time: new Date() - startTime } } - - -s.detectObject = function(buffer,d,tx,frameLocation,callback){ - process( buffer).then((resp)=>{ + + +s.detectObject = function (buffer, d, tx, frameLocation, callback) { + process(buffer).then((resp) => { var results = resp.data //console.log(resp.time) - if(Array.isArray(results) && results[0]){ + if (Array.isArray(results) && results[0]) { var mats = [] - results.forEach(function(v){ + results.forEach(function (v) { mats.push({ x: v.bbox[0], y: v.bbox[1], @@ -110,19 +124,19 @@ s.detectObject = function(buffer,d,tx,frameLocation,callback){ }) }) var isObjectDetectionSeparate = d.mon.detector_pam === '1' && d.mon.detector_use_detect_object === '1' - var width = parseFloat(isObjectDetectionSeparate && d.mon.detector_scale_y_object ? d.mon.detector_scale_y_object : d.mon.detector_scale_y) - var height = parseFloat(isObjectDetectionSeparate && d.mon.detector_scale_x_object ? d.mon.detector_scale_x_object : d.mon.detector_scale_x) + var width = parseFloat(isObjectDetectionSeparate && d.mon.detector_scale_y_object ? d.mon.detector_scale_y_object : d.mon.detector_scale_y) + var height = parseFloat(isObjectDetectionSeparate && d.mon.detector_scale_x_object ? d.mon.detector_scale_x_object : d.mon.detector_scale_x) tx({ - f:'trigger', - id:d.id, - ke:d.ke, - details:{ - plug:config.plug, - name:'Tensorflow', - reason:'object', - matrices:mats, - imgHeight:width, - imgWidth:height, + f: 'trigger', + id: d.id, + ke: d.ke, + details: { + plug: config.plug, + name: 'Tensorflow', + reason: 'object', + matrices: mats, + imgHeight: width, + imgWidth: height, time: resp.time } })