Added face-api usage

Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>

package.json

@@ -17,14 +17,18 @@
     "face-recognition": "^0.9.4"
   },
   "dependencies": {
+    "@tensorflow/tfjs-core": "^1.5.1",
+    "@tensorflow/tfjs-node": "^1.5.1",
     "bluebird": "^3.7.2",
     "body-parser": "^1.19.0",
+    "canvas": "^2.6.1",
     "config": "^1.31.0",
     "connect-sqlite3": "^0.9.11",
     "cookie-parser": "^1.4.4",
     "exif-reader": "github:paras20xx/exif-reader",
     "express": "^4.17.1",
     "express-session": "^1.17.0",
+    "face-api.js": "^0.22.0",
     "handlebars": "^4.5.3",
     "ldapauth-fork": "^4.2.0",
     "ldapjs": "^1.0.2",
@@ -33,6 +37,7 @@
     "moment-holiday": "^1.5.1",
     "morgan": "^1.9.1",
     "mustache": "^3.2.1",
+    "node-fetch": "^2.6.0",
     "nodemailer": "^4.7.0",
     "qs": "^6.9.1",
     "sequelize": "^4.44.3",

server/face-recognizer.js

@@ -19,21 +19,40 @@ process.env.TZ = "Etc/GMT";
 
 console.log("Loading face-recognizer");
 
+require('@tensorflow/tfjs-node');
+
 const config = require("config"),
   Promise = require("bluebird"),
-  { mkdir, unlink } = require("./lib/util"),
+  { exists, mkdir, unlink } = require("./lib/util"),
-  fr = require("face-recognition");
+  faceapi = require("face-api.js"),
+  fs = require("fs"),
+  canvas = require("canvas");
+
+const { Canvas, Image, ImageData } = canvas;
+
+faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
+
+const maxConcurrency = require("os").cpus().length;
 
 require("./console-line.js"); /* Monkey-patch console.log with line numbers */
 
 const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
   faceData = picturesPath + "face-data/";
 
-let photoDB = null, faceDataExists = false;
+let photoDB = null;
 
 console.log("Loading pictures out of: " + picturesPath);
 
-require("./db/photos").then(function(db) {
+faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
+.then(() => {
+  console.log("ssdMobileNetv1 loaded.");
+  return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
+}).then(() => {
+  console.log("landmark68 loaded.");
+  return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
+}).then(() => {
+  console.log("faceRecognitionNet loaded.");
+  return require("./db/photos").then(function(db) {
     photoDB = db;
   }).then(() => {
     console.log("DB connected.");
@@ -50,73 +69,78 @@ require("./db/photos").then(function(db) {
       return Promise.map(results, (photo) => {
         const filePath = photo.path + photo.filename;
         console.log(`Processing ${filePath}...`);
-
-      return photoDB.sequelize.transaction(function(transaction) {
           /* Remove any existing face data for this photo */
           return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
-          transaction: transaction,
             replacements: photo,
           }).then((faces) => {
             /* For each face-id, remove any face-data files, and then remove all the entries
             * from the DB */
-          return Promise.map(faces, (id) => {
+            return Promise.map(faces, (face) => {
-            return Promise.mapSeries(["-normalized.png", "-data.json" ], (fileSuffix) => {
+              const id = face.id,
-              const filePath = faceData + "/" + (id % 100) + "/" + id + fileSuffix;
+                filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
               return exists(filePath).then((result) => {
+                if (result) {
                   console.log(`...removing ${filePath}`);
                   return unlink(filePath);
-              });
+                }
               });
             }).then(() => {
               return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
-              transaction: transaction,
                 replacements: photo,
               });
-          }).then(() => {
+            }).then(async () => {
-            const image = fr.loadImage(filePath),
+              console.log("...loading image.");
-              detector = fr.FaceDetector();
+              const image = await canvas.loadImage(picturesPath + filePath);
 
               console.log("...detecting faces.");
-            const faceRectangles = detector.locateFaces(image)
+              const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors();
-            if (faceRectangles.length == 0) {
-              console.log("...no faces found in image.");
-              return;
-            }
 
-            /* Create a face entry in photos for each face found. */
+              console.log(`...${detections.length} faces identified.`);
-            const faceImages = detector.detectFaces(image, 200)
+              return Promise.map(detections, (face, index) => {
-            
+                const width = face.detection._box._width,
-            console.log(`...saving ${faceImages.length} faces.`);
+                  height = face.detection._box._height,
-
+                  replacements = {
-            return Promise.map(faceRectangles, (face, index) => {
+                    id: photo.id,
+                    top: face.detection._box._y / photo.height,
+                    left: face.detection._box._x / photo.width,
+                    bottom: (face.detection._box._x + height) / photo.height,
+                    right: (face.detection._box._x + width) / photo.width,
+                    faceConfidence: face.detection._score
+                  };
                 return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
                   "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
-                replacements: {
+                  replacements: replacements
-                  id: photo.id,
+                }).catch(() => {
-                  top: face.top / photo.height,
+                  console.error(filePath, index);
-                  left: face.left / photo.width,
+                  console.error(JSON.stringify(face, null, 2));
-                  bottom: face.top / photo.height,
+                  console.error(JSON.stringify(replacements, null, 2));
-                  right: face.right / photo.width,
+                  process.exit(-1);
-                  faceConfidence: face.confidence
-                },
-                transaction: transaction
                 }).spread((results, metadata) => {
                   return metadata.lastID;
                 }).then((id) => {
-                console.log(`...DB id ${id}. Writing data and images...`);
+                  console.log(`...DB id ${id}. Writing descriptor data...`);
-                const filePathPrefix = faceData + "/" + (id % 100) + "/" + id;
+                  const path = faceData + "/" + (id % 100);
-                /* https://medium.com/@ageitgey/machine-learning-is-fun-part-4-modern-face-recognition-with-deep-learning-c3cffc121d78 */
+                  return mkdir(path).then(() => {
-                const data = [];
+                    const filePath = path + "/" + id + "-data.json",
+                      data = [];
                     for (let i = 0; i < 128; i++) {
-                  data.push(Math.random() - 0.5);
+                      data.push(face.descriptor[i]);
                     }
-                fs.writeFileSync(filePathPrefix + "-data.json", JSON.stringify(data));
+                    fs.writeFileSync(filePath, JSON.stringify(data));
-                fr.saveImage(filePathPrefix + "-normalized.png", faceImages[index]);
+                  });
-              });
+                });
+              }).then(() => {
+                return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
+                  replacements: {
+                    id: photo.id,
+                    faces: detections.length
+                  },
                 });
               });
             });
         });
+      }, {
+        concurrency: maxConcurrency
       });
     });
   }).then(() => {
@@ -125,3 +149,23 @@ require("./db/photos").then(function(db) {
     console.error(error);
     process.exit(-1);
   });
+});
+
+/* TODO:
+1. For each path / person, look up highest face confidence and tag
+2. Use highest face and identity confidence for input into 
+https://github.com/justadudewhohacks/face-api.js#face-recognition-by-matching-descriptors
+
+const labeledDescriptors = [
+  new faceapi.LabeledFaceDescriptors(
+    'obama',
+    [descriptorObama1, descriptorObama2]
+  ),
+  new faceapi.LabeledFaceDescriptors(
+    'trump',
+    [descriptorTrump]
+  )
+]
+
+const faceMatcher = new faceapi.FaceMatcher(labeledDescriptors)
+*/