feat(mqtt & api): total count for person, match, miss, and unknown (#223)

jakowenko · jakowenko · commit 2bf4406aa7e9 · 2022-10-20T23:35:21.000-04:00
also refactored how the values are calculated
diff --git a/api/src/controllers/recognize.controller.js b/api/src/controllers/recognize.controller.js
@@ -148,7 +148,7 @@ module.exports.start = async (req, res) => {
       );
     }
 
-    const { best, misses, unknown, results, attempts } = recognize.normalize(
+    const { best, misses, unknowns, results, attempts, counts } = recognize.normalize(
       await Promise.all(promises)
     );
 
@@ -160,28 +160,28 @@ module.exports.start = async (req, res) => {
       attempts,
       camera,
       zones,
+      counts,
       matches: best,
       misses,
+      unknowns,
     };
-    if (unknown && Object.keys(unknown).length) output.unknown = unknown;
     if (AUTH) output.token = jwt.sign({ route: 'storage', expiresIn: TOKEN.IMAGE });
 
     if (resultsOutput === 'all') output.results = results;
 
     console.log(`done processing ${camera}: ${id} in ${duration} sec`);
 
     const loggedOutput = JSON.parse(JSON.stringify(output));
-    ['matches', 'misses'].forEach((type) =>
+    ['matches', 'misses', 'unknowns'].forEach((type) =>
       loggedOutput[type].forEach((result) => delete result.base64)
     );
-    if (loggedOutput.unknown) delete loggedOutput.unknown.base64;
     console.log(loggedOutput);
 
     PROCESSING = false;
 
     res.send(output);
 
-    recognize.save.latest(camera, best, misses, unknown);
+    recognize.save.latest(camera, best, misses, unknowns[0]);
     mqtt.recognize(output);
     notify.publish(output, camera, results);
     if (event.type === 'frigate') frigate.subLabel(event.topic, id, best);
diff --git a/api/src/util/mqtt.util.js b/api/src/util/mqtt.util.js
@@ -3,7 +3,6 @@ const { v4: uuidv4 } = require('uuid');
 const axios = require('axios');
 const mqtt = require('mqtt');
 const fs = require('./fs.util');
-const { contains } = require('./helpers.util');
 const { jwt } = require('./auth.util');
 const { AUTH, SERVER, MQTT, FRIGATE, CAMERAS, STORAGE, UI } = require('../constants')();
 const config = require('../constants/config');
@@ -156,47 +155,51 @@ module.exports.subscribe = () => {
 module.exports.recognize = (data) => {
   try {
     if (!MQTT || !MQTT.HOST) return;
-    const { matches, misses, unknown } = data;
-    const camera = data.camera.toLowerCase();
-    const hasUnknown = unknown && Object.keys(unknown).length;
-
-    const configData = JSON.parse(JSON.stringify(data));
-    delete configData.matches;
-    delete configData.unknown;
-    delete configData.results;
+    const baseData = JSON.parse(JSON.stringify(data));
+    const { id, duration, timestamp, attempts, zones, matches, misses, unknowns, counts } =
+      baseData;
+    const camera = baseData.camera.toLowerCase();
+
+    const payload = {
+      base: {
+        id,
+        duration,
+        timestamp,
+        attempts,
+        camera,
+        zones,
+      },
+    };
+    payload.unknown = { ...payload.base, unknown: unknowns[0], unknowns };
+    payload.match = { ...payload.base };
+    payload.camera = {
+      ...payload.base,
+      matches,
+      misses,
+      unknowns,
+      counts,
+    };
+    payload.cameraReset = {
+      ...payload.camera,
+      counts: {
+        person: 0,
+        match: 0,
+        miss: 0,
+        unknown: 0,
+      },
+    };
 
     const messages = [];
-    const persons = [...new Set([...matches, ...misses].map(({ name }) => name))];
-    let personCount = persons.length ? persons.length : hasUnknown ? 1 : 0;
-    // check to see if unknown bounding box is contained within or contains any of the match bounding boxes
-    // if false, then add 1 to the person count
-    if (persons.length && hasUnknown) {
-      let unknownFoundInMatch = false;
-      matches.forEach((match) => {
-        if (contains(match.box, unknown.box) || contains(unknown.box, match.box))
-          unknownFoundInMatch = true;
-      });
-
-      let unknownFoundInMiss = false;
-      misses.forEach((miss) => {
-        if (contains(miss.box, unknown.box) || contains(unknown.box, miss.box))
-          unknownFoundInMiss = true;
-      });
-      if (!unknownFoundInMatch && !unknownFoundInMiss) personCount += 1;
-    }
 
     messages.push({
       topic: `${MQTT.TOPICS.CAMERAS}/${camera}/person`,
-      message: personCount.toString(),
+      message: counts.person.toString(),
     });
 
-    if (hasUnknown) {
+    if (unknowns.length) {
       messages.push({
         topic: `${MQTT.TOPICS.MATCHES}/unknown`,
-        message: JSON.stringify({
-          ...configData,
-          unknown,
-        }),
+        message: JSON.stringify(payload.unknown),
       });
 
       if (MQTT.TOPICS.HOMEASSISTANT) {
@@ -215,10 +218,7 @@ module.exports.recognize = (data) => {
 
         messages.push({
           topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/unknown/state`,
-          message: JSON.stringify({
-            ...configData,
-            unknown,
-          }),
+          message: JSON.stringify(payload.unknown),
         });
       }
     }
@@ -230,7 +230,7 @@ module.exports.recognize = (data) => {
       messages.push({
         topic: `${MQTT.TOPICS.MATCHES}/${topic}`,
         message: JSON.stringify({
-          ...configData,
+          ...payload.match,
           match,
         }),
       });
@@ -252,22 +252,17 @@ module.exports.recognize = (data) => {
         messages.push({
           topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${topic}/state`,
           message: JSON.stringify({
-            ...configData,
+            ...payload.match,
             match,
           }),
         });
       }
     });
 
-    if (matches.length || misses.length || hasUnknown) {
+    if (matches.length || misses.length || unknowns.length) {
       messages.push({
         topic: `${MQTT.TOPICS.CAMERAS}/${camera}`,
-        message: JSON.stringify({
-          ...configData,
-          matches,
-          misses,
-          unknown,
-        }),
+        message: JSON.stringify(payload.camera),
       });
 
       if (MQTT.TOPICS.HOMEASSISTANT) {
@@ -286,13 +281,7 @@ module.exports.recognize = (data) => {
 
         messages.push({
           topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${camera}/state`,
-          message: JSON.stringify({
-            ...configData,
-            matches,
-            misses,
-            unknown,
-            personCount,
-          }),
+          message: JSON.stringify(payload.camera),
         });
       }
     }
@@ -305,12 +294,7 @@ module.exports.recognize = (data) => {
       if (MQTT.TOPICS.HOMEASSISTANT) {
         this.publish({
           topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${camera}/state`,
-          message: JSON.stringify({
-            ...configData,
-            matches,
-            unknown,
-            personCount: 0,
-          }),
+          message: JSON.stringify(payload.cameraReset),
         });
       }
     }, 30000);
diff --git a/api/src/util/recognize.util.js b/api/src/util/recognize.util.js
@@ -39,28 +39,33 @@ module.exports.save = {
 
 module.exports.normalize = (results = []) => {
   const best = { matches: [], misses: [] };
-  const tmp = { matches: {}, misses: {} };
-  let unknown = {};
+  const tmp = { matches: {}, misses: {}, counts: {} };
+  const unknowns = [];
 
   let attempts = 0;
   results.forEach((group) => {
     attempts += group.attempts;
     group.results.forEach((attempt) => {
+      const face = tmp.counts[attempt.detector];
+      if (face)
+        tmp.counts[attempt.detector] = {
+          count: face.count + attempt.results.length,
+          attempts: (face.attempts += 1),
+        };
+      else tmp.counts[attempt.detector] = { count: attempt.results.length, attempts: 1 };
+
       const matches = attempt.results.filter((obj) => obj.match);
       const misses = attempt.results.filter((obj) => !obj.match && obj.name !== 'unknown');
-      const unknowns = attempt.results.filter((obj) => obj.name === 'unknown');
-
-      unknowns.forEach((obj) => {
-        if (unknown.confidence === undefined || unknown.confidence < obj.confidence) {
-          unknown = {
-            ...obj,
-            type: group.type,
-            duration: attempt.duration,
-            detector: attempt.detector,
-            filename: attempt.filename,
-            base64: attempt.base64 || null,
-          };
-        }
+      const tmpUnknowns = attempt.results.filter((obj) => obj.name === 'unknown');
+      tmpUnknowns.forEach((obj) => {
+        unknowns.push({
+          ...obj,
+          type: group.type,
+          duration: attempt.duration,
+          detector: attempt.detector,
+          filename: attempt.filename,
+          base64: attempt.base64 || null,
+        });
       });
 
       matches.forEach((match) => {
@@ -100,5 +105,23 @@ module.exports.normalize = (results = []) => {
   for (const value of Object.values(tmp.matches)) best.matches.push(value);
   for (const value of Object.values(tmp.misses)) best.misses.push(value);
 
-  return { best: best.matches, misses: best.misses, results, attempts, unknown };
+  let personCount = 0;
+  for (const [, value] of Object.entries(tmp.counts))
+    personCount += Math.round(value.count / value.attempts);
+
+  const counts = {
+    person: personCount ? Math.round(personCount / Object.keys(tmp.counts).length) : 0,
+    match: best.matches.length,
+    miss: best.misses.length,
+  };
+  counts.unknown = counts.person - counts.match - counts.miss;
+
+  return {
+    best: best.matches,
+    misses: best.misses,
+    results,
+    attempts,
+    unknowns: unknowns.sort((a, b) => b.confidence - a.confidence),
+    counts,
+  };
 };
