Add face-crop creation

Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>

server/face-recognizer.js

@@ -28,7 +28,7 @@ const config = require("config"),
   fs = require("fs"),
   canvas = require("canvas");
 
-const { Canvas, Image, ImageData } = canvas;
+const { createCanvas, Canvas, Image, ImageData } = canvas;
 
 faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
 
@@ -43,19 +43,20 @@ let photoDB = null;
 
 console.log("Loading pictures out of: " + picturesPath);
 
-faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
+process.stdout.write("Loading DB.");
-.then(() => {
+require("./db/photos").then(function(db) {
-  console.log("ssdMobileNetv1 loaded.");
+  process.stdout.write("done\n");
-  return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
-}).then(() => {
-  console.log("landmark68 loaded.");
-  return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
-}).then(() => {
-  console.log("faceRecognitionNet loaded.");
-  return require("./db/photos").then(function(db) {
   photoDB = db;
 }).then(() => {
   console.log("DB connected.");
+  process.stdout.write("Loading models.");
+  return faceapi.nets.ssdMobilenetv1.loadFromDisk('./models');
+}).then(() => {
+  process.stdout.write(".");
+  return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
+}).then(() => {
+  process.stdout.write(".");
+  return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
 }).then(() => {
   console.log("Beginning face detection scanning.");
   return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
@@ -64,14 +65,16 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
     "WHERE faces=-1 AND photos.duplicate=0 AND photos.deleted=0 ORDER BY albums.path,photos.filename", {
     type: photoDB.sequelize.QueryTypes.SELECT,
     raw: true
-      }
+  });
-    ).then((results) => {
+}).then((needToScan) => {
-      const total = results.length;
+  const total = needToScan.length;
   let remaining = total,
     processed = 0,
     lastStatus = Date.now();
-      console.log(`${results.length} photos have not had faces scanned.`);
+  
-      return Promise.map(results, (photo) => {
+    console.log(`${needToScan.length} photos have not had faces scanned.`);
+
+  return Promise.map(needToScan, (photo) => {
     const photoPath = photo.path + photo.filename;
 
     console.log(`Processing ${photoPath}...`);
@@ -83,22 +86,24 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
       /* For each face-id, remove any face-data files, and then remove all the entries
         * from the DB */
       return Promise.map(faces, (face) => {
+        return Promise.map([ "-data.json", "-original.png" ], (suffix) => {
           const id = face.id,
-              dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
+            dataPath = faceData + "/" + (id % 100) + "/" + id + suffix;
           return exists(dataPath).then((result) => {
             if (result) {
               console.log(`...removing ${dataPath}`);
               return unlink(dataPath);
             }
           });
+        });
       }).then(() => {
         return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
           replacements: photo,
         });
       });
     }).then(async () => {
+      /* Process image for faces data */
       const image = await canvas.loadImage(picturesPath + photoPath);
-
       const detections = await faceapi.detectAllFaces(image, 
           new faceapi.SsdMobilenetv1Options({
             minConfidence: 0.8
@@ -106,7 +111,7 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
       ).withFaceLandmarks().withFaceDescriptors();
 
       if (detections.length > 0) {
-            console.log(`...${detections.length} faces identified.`);
+        console.log(`...${detections.length} faces identified in ${photoPath}.`);
       }
 
       return Promise.map(detections, (face) => {
@@ -128,15 +133,35 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
         }).spread((results, metadata) => {
           return metadata.lastID;
         }).then((id) => {
-              console.log(`...DB id ${id}. Writing descriptor data...`);
+          const path = faceData + (id % 100);
-              const path = faceData + "/" + (id % 100);
           return mkdir(path).then(() => {
-                const dataPath = path + "/" + id + "-data.json",
+            const dataPath = `${path}/${id}-data.json`, data = [];
-                  data = [];
+            console.log(`...writing descriptor data to ${dataPath}...`);
             for (let i = 0; i < 128; i++) {
               data.push(face.descriptor[i]);
             }
             fs.writeFileSync(dataPath, JSON.stringify(data));
+          }).then(() => {
+            const canvas = createCanvas(200, 200),
+              target = `${path}/${id}-original.png`,
+              ctx = canvas.getContext('2d'),
+              box = face.detection._box,
+              aspect = box._width / box._height,
+              dx = (aspect > 1.0) ? 200 : (200 * aspect),
+              dy = (aspect < 1.0) ? 200 : (200 / aspect);
+            ctx.fillStyle = "rgba(0, 0, 0, 0)";
+            ctx.fillRect(0, 0, 200, 200);
+            ctx.drawImage(image, box._x, box._y, box._width, box._height, 
+              Math.floor((200 - dx) * 0.5),
+              Math.floor((200 - dy) * 0.5), dx, dy);
+            console.log(`...writing face crop to ${target}.`);
+            fs.writeFileSync(target, canvas.toBuffer("image/png", {
+              quality: 0.95,
+              chromaSubsampling: false
+            }));
+          }).catch((error) => {
+            console.error(error);
+            process.exit(-1);
           });
         });
       }).then(() => {
@@ -169,10 +194,9 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
   }, {
     concurrency: maxConcurrency
   });
-    });
 }).then(() => {
   console.log("Looking for face distances that need to be updated...");
-    const descriptors = {};
+  let maxId;
 
   return photoDB.sequelize.query("SELECT faces.id FROM faces ORDER BY faces.id DESC LIMIT 1", {
     type: photoDB.sequelize.QueryTypes.SELECT,
@@ -180,28 +204,31 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
   }).then((results) => {
     if (!results.length) {
       console.log("...no faces exist yet to generate distances.");
-        return;
+      maxId = 0;
+      return [];
     }
-      const maxId = results[0].id;
+    maxId = results[0].id;
     return photoDB.sequelize.query(
       "SELECT faces.id,faces.lastComparedId " +
       "FROM faces INNER JOIN photos ON photos.duplicate=0 AND photos.deleted=0 AND photos.id=faces.photoId " +
-        "WHERE faces.lastComparedId<:maxId OR faces.lastComparedId IS NULL", {
+      "WHERE faces.lastComparedId<:maxId OR faces.lastComparedId IS NULL " +
+      "ORDER BY faces.id ASC", {
       replacements: {
         maxId: maxId
       },
       type: photoDB.sequelize.QueryTypes.SELECT,
       raw: true
+    });
   }).then((facesToUpdate) => {
-        console.log(`...${facesToUpdate.length} faces need to be updated.`);
+    console.log(`...${facesToUpdate.length} faces need distances updated.`);
     if (facesToUpdate.length == 0) {
       return facesToUpdate;
     }
 
-        console.log("...removing old assets.");
+    const descriptors = {};
 
     return photoDB.sequelize.query(
-          "SELECT id FROM faces", {
+      "SELECT id FROM faces ORDER BY id ASC", {
       type: photoDB.sequelize.QueryTypes.SELECT,
       raw: true
     }).then((allFaces) => {
@@ -222,13 +249,8 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
 
           descriptors[id] = JSON.parse(fs.readFileSync(dataPath));
         });
-          }, {
-            concurrency: maxConcurrency
       });
     }).then(() => {
-          return facesToUpdate;
-        });
-      }).then((facesToUpdate) => {
       const total = facesToUpdate.length;
       let remaining = total,
         processed = 0,
@@ -248,6 +270,18 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
         const faceDescriptor = descriptors[face.id];
 
         return photoDB.sequelize.transaction((transaction) => {
+          return photoDB.sequelize.query(
+            "SELECT distance,face1Id,face2Id " +
+            "FROM facedistances " +
+            "WHERE face1Id=:id OR face2Id=:id " +
+            "ORDER BY face1Id ASC", {
+            replacements: {
+              id: face.id
+            },
+            type: photoDB.sequelize.QueryTypes.SELECT,
+            raw: true,
+            transaction: transaction
+          }).then((distances) => {
             return Promise.map(targets, (target) => {
               /* Skip comparing to self */
               if (target.id == face.id) {
@@ -259,23 +293,22 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
                 return;
               }
 
-              return photoDB.sequelize.query(
+              const index = distances.findIndex((distance) => {
-                "SELECT distance " +
+                return distance.face1Id == target.id || distance.face2Id == target.id
-                "FROM facedistances " +
+              });
-                "WHERE face1Id=:first AND face2Id=:second", {
+
-                replacements: {
+              if (index != -1) {
-                  first: Math.min(face.id, target.id),
+                /* A distance has already been calculated between face and target */
-                  second: Math.max(face.id, target.id)
-                },
-                type: photoDB.sequelize.QueryTypes.SELECT,
-                raw: true,
-                transaction: transaction
-              }).then((results) => {
-                if (results.length) {
                 return;
               }
+
               const distance = faceapi.euclideanDistance(faceDescriptor, target.descriptor);
 
+              /* If the distance > 0.6, we don't want to store this in the DB */
+              if (distance > 0.6) {
+                return;
+              }
+
               if (distance < 0.4) {
                 console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
               }
@@ -290,9 +323,9 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
                 },
                 transaction: transaction
               });
-              });
             }, {
               concurrency: maxConcurrency
+            });
           }).then(() => {
             return photoDB.sequelize.query(
               "UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
@@ -323,23 +356,3 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
   console.error(error);
   process.exit(-1);
 });
-});
-
-/* TODO:
-1. For each path / person, look up highest face confidence and tag
-2. Use highest face and identity confidence for input into 
-https://github.com/justadudewhohacks/face-api.js#face-recognition-by-matching-descriptors
-
-const labeledDescriptors = [
-  new faceapi.LabeledFaceDescriptors(
-    'obama',
-    [descriptorObama1, descriptorObama2]
-  ),
-  new faceapi.LabeledFaceDescriptors(
-    'trump',
-    [descriptorTrump]
-  )
-]
-
-const faceMatcher = new faceapi.FaceMatcher(labeledDescriptors)
-*/

server/face.js

@@ -4,6 +4,8 @@ process.env.TZ = "Etc/GMT";
 
 require('@tensorflow/tfjs-node');
 
+let photoDB = null;
+
 const config = require("config"),
   Promise = require("bluebird"),
   { exists, mkdir, unlink } = require("./lib/util"),
@@ -11,7 +13,7 @@ const config = require("config"),
   fs = require("fs"),
   canvas = require("canvas");
 
-const { Canvas, Image, ImageData } = canvas;
+const { createCanvas, Canvas, Image, ImageData } = canvas;
 
 faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
 
@@ -19,45 +21,179 @@ const maxConcurrency = require("os").cpus().length;
 
 require("./console-line.js"); /* Monkey-patch console.log with line numbers */
 
-faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
+const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
-.then(() => {
+  faceData = picturesPath + "face-data/";
-  console.log("ssdMobileNetv1 loaded.");
+
+process.stdout.write("Loading DB.");
+require("./db/photos").then(function(db) {
+  process.stdout.write("done\n");
+  photoDB = db;
+}).then(() => {
+  console.log("DB connected.");
+  process.stdout.write("Loading models.");
+  return faceapi.nets.ssdMobilenetv1.loadFromDisk('./models');
+}).then(() => {
+  process.stdout.write(".");
   return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
 }).then(() => {
-  console.log("landmark68 loaded.");
+  process.stdout.write(".");
   return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
 }).then(async () => {
-  console.log("faceRecognitionNet loaded.");
+  process.stdout.write(".done\n");
-  let faces = [];
+
-  for (let a = 2; a < process.argv.length; a++) {
+  if (process.argv[0].match(/node/)) {
-    const file = process.argv[a];
+    process.argv.shift(); /* node */
-    process.stdout.write(`Loading ${file}...`);
+  }
-    const image = await canvas.loadImage(file),
+  process.argv.shift(); /* script name */
-      detectors = await faceapi.detectAllFaces(image, 
+
+  return Promise.resolve().then(() => {
+    console.log(process.argv.length);
+
+    if (process.argv.length != 0) {
+      return process.argv;
+    }
+
+    /* If no parameters provided, scan all faces to create image crops */
+    return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id ASC", {
+      type: photoDB.sequelize.QueryTypes.SELECT,
+      raw: true
+    }).then((results) => {
+      return results.map(result => result.id);
+    });
+  });
+}).then((args) => {
+  const faces = [];
+
+  return Promise.map(args, (arg) => {
+    const file = arg,
+      id = parseInt(arg);
+
+    let loader;
+
+    if (id == file) {
+      /* This is a face id */
+      console.log(`Looking up face-id ${id}...`);
+      loader = photoDB.sequelize.query(
+        "SELECT albums.path,photos.filename,photos.width,photos.height,faces.* " +
+        "FROM faces,photos,albums " +
+        "WHERE photos.id=faces.photoId " +
+        "AND albums.id=photos.albumId " +
+        "AND faces.id=:id", {
+          replacements: {
+            id: id
+          },
+          type: photoDB.sequelize.QueryTypes.SELECT,
+          raw: true
+      }).then((results) => {
+        if (results.length != 1) {
+          console.error(`...error. No face-id found: ${id}.\n`);
+          process.exit(-1);
+        }
+        const photo = results[0];
+        console.log(`...loading ${photo.filename}`);
+
+        const file = photo.path + photo.filename;
+        return canvas.loadImage(picturesPath + file).then(async (image) => {
+          const detectors = [ {
+            detection: {
+              _box: {
+                _x: photo.left * photo.width,
+                _y: photo.top * photo.height,
+                _width: (photo.right - photo.left) * photo.width,
+                _height: (photo.bottom - photo.top) * photo.height,
+              }
+            },
+            descriptor: JSON.parse(fs.readFileSync(faceData + (id % 100) + "/" + id + "-data.json"))
+          } ];
+          return [ file, image, detectors ];
+        });
+      });
+    } else {
+      /* This is a file */
+      console.log(`Loading ${file}...`);
+      id = undefined;
+      loader = canvas.loadImage(picturesPath + file).then(async (image) => {
+        const detectors = await faceapi.detectAllFaces(image, 
             new faceapi.SsdMobilenetv1Options({
               minConfidence: 0.8
             })
           ).withFaceLandmarks().withFaceDescriptors();
-    process.stdout.write(`${detectors.length} faces.\n`);
+        detectors.forEach((detector) => {
-    
+          const data = [];
-    detectors.forEach((face, index) => {
+          /* Confert from sparse object to dense array */
-      faces.push({
+          for (let i = 0; i < 128; i++) {
-        file: file,
+            data.push(detector.descriptor[i]);
-        index: index,
+          }
-        descriptor: face.descriptor
+          detector.descriptor = data;
-      })
+        });
+        return [ file, image, detectors ];
       });
     }
 
-  for (let i = 0; i < faces.length; i++) {
+    return loader.spread((filepath, image, detectors) => {
-    for (let j = 0; j < faces.length; j++) {
+      process.stdout.write(`${detectors.length} faces.\n`);
-      const a = faces[i], b = faces[j];
+        
-      const distance = faceapi.euclideanDistance(a.descriptor, b.descriptor);
+      return Promise.map(detectors, (face, index) => {
-      console.log(`${a.file}.${a.index} to ${b.file}.${b.index} = ${distance}`);
+        faces.push({
+          filepath: filepath,
+          index: index,
+          descriptor: face.descriptor
+        })
+
+        /* If this is a face-id, output the -original.png 
+         * meta-data file */
+        if (!id) {
+          return;
         }
+
+        const path = "face-data/" + (id % 100),
+          target = `${path}/${id}-original.png`,
+          box = face.detection._box,
+          aspect = box._width / box._height,
+          dx = (aspect > 1.0) ? 200 : (200 * aspect),
+          dy = (aspect < 1.0) ? 200 : (200 / aspect);
+          
+        return exists(target).then((doesExist) => {
+          if (doesExist) {
+            console.log(`...${target} already exists.`);
+            return;
           }
+          const canvas = createCanvas(200, 200),
+            ctx = canvas.getContext('2d');
+
+          ctx.fillStyle = "rgba(0, 0, 0, 0)";
+          ctx.fillRect(0, 0, 200, 200);
+          ctx.drawImage(image, box._x, box._y, box._width, box._height, 
+            Math.floor((200 - dx) * 0.5),
+            Math.floor((200 - dy) * 0.5), dx, dy);
+
+          console.log(`...writing to ${target}.`);
+
+          return mkdir(path).then(() => {
+            fs.writeFileSync(picturesPath + target, canvas.toBuffer("image/png", {
+              quality: 0.95,
+              chromaSubsampling: false
+            }));
+          });
+        });
+      });
+    });
+  }, {
+    concurrency: maxConcurrency
   }).then(() => {
     console.log("Face detection scanning completed.");
+    faces.forEach((a, i) => {
+      faces.forEach((b, j) => {
+        if (i == j) {
+          return;
+        }
+        const distance = faceapi.euclideanDistance(a.descriptor, b.descriptor);
+        if (distance < 0.4) {
+          console.log(`${a.filepath}.${a.index} is similar to ${b.filepath}.${b.index}: ${distance}`);
+        }
+      })
+    });
+  });
 }).catch((error) => {
   console.error(error);
   process.exit(-1);