Face scanning making progress. Working on face-box.

Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>

frontend/slideshow.html

@@ -87,6 +87,45 @@ const days = [ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday",
       months = [ "January", "February", "March", "April", "May", "June",
             "July", "August", "September", "October", "November", "December" ];
 
+let activeFaces = [];
+
+function makeFaceBoxes() {
+  const el = document.getElementById("photo");
+
+  let width, height, offsetLeft = 0, offsetTop = 0;
+
+  /* If photo is wider than viewport, it will be 100% width and < 100% height */
+  if (photo.width / photo.height > el.offsetWidth / el.offsetHeight) {
+    console.log("A");
+    width = 100;
+    height = 100 * photo.height / photo.width * el.offsetWidth / el.offsetHeight;
+    offsetLeft = 0;
+    offsetTop = (100 - height) * 0.5;
+  } else {
+    console.log("B");
+    width = 100 * photo.width / photo.height * el.offsetHeight / el.offsetWidth;
+    height = 100;
+    offsetLeft = (100 - width) * 0.5;
+    offsetTop = 0;
+  }
+
+  
+  activeFaces.forEach((face) => {
+    const box = document.createElement("div");
+    box.classList.add("face");
+    document.body.appendChild(box);
+    box.style.position = "absolute";
+    box.style.display = "inline-block";
+    box.style.border = "1px solid red";
+    box.style.background = "rgba(255, 0, 0, 0.5)";
+    box.style.opacity = 0.5;
+    box.style.left = offsetLeft + Math.floor(face.left * width)  + "%";
+    box.style.top = offsetTop + Math.floor(face.top * height)  + "%";
+    box.style.width = Math.floor((face.right - face.left) * width)  + "%";
+    box.style.height = Math.floor((face.bottom - face.top) * height)  + "%";
+  });
+}
+
 function loadPhoto(index) {
   const photo = photos[index],
     xml = new XMLHttpRequest(),
@@ -114,6 +153,16 @@ function loadPhoto(index) {
     document.getElementById("photo").style.backgroundImage = "url(" + encodeURI(url) + ")";
     countdown = 15;
     tick();
+    Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => {
+      el.parentElement.removeChild(el);
+    });
+    window.fetch("api/v1/photos/faces/" + photo.id).then(res => res.json()).then((faces) => {
+      activeFaces = faces;
+      makeFaceBoxes();
+    }).catch(function(error) {
+      console.error(error);
+      info.textContent += "Unable to obtain face information :(";
+    });
   }
 
   xml.onerror = function(event) {

server/db/photos.js

@@ -151,19 +151,19 @@ function init() {
     });
 
     const FaceDistances = db.sequelize.define('facedistance', {
-      photoId: {
+      face1Id: {
         type: Sequelize.INTEGER,
         allowNull: false,
         references: {
-          model: Photo,
+          model: Face,
           key: 'id',
         }
       },
-      targetId: {
+      face2Id: {
         type: Sequelize.INTEGER,
         allowNull: false,
         references: {
-          model: Photo,
+          model: Face,
           key: 'id',
         }
       },

server/face-recognizer.js

@@ -62,13 +62,18 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
       "FROM photos " +
       "LEFT JOIN albums ON (albums.id=photos.albumId) " +
       "WHERE faces=-1 ORDER BY albums.path,photos.filename", {
-        type: photoDB.sequelize.QueryTypes.SELECT
+        type: photoDB.sequelize.QueryTypes.SELECT,
+        raw: true
       }
     ).then((results) => {
+      const remaining = results.length,
+        lastStatus = Date.now();
       console.log(`${results.length} photos have not had faces scanned.`);
       return Promise.map(results, (photo) => {
-        const filePath = photo.path + photo.filename;
+        const photoPath = photo.path + photo.filename;
-        console.log(`Processing ${filePath}...`);
+
+        console.log(`Processing ${photoPath}...`);
+
         /* Remove any existing face data for this photo */
         return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
           replacements: photo,
@@ -77,11 +82,11 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
             * from the DB */
           return Promise.map(faces, (face) => {
             const id = face.id,
-                filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
+              dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
-              return exists(filePath).then((result) => {
+            return exists(dataPath).then((result) => {
               if (result) {
-                  console.log(`...removing ${filePath}`);
+                console.log(`...removing ${dataPath}`);
-                  return unlink(filePath);
+                return unlink(dataPath);
               }
             });
           }).then(() => {
@@ -89,13 +94,17 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
               replacements: photo,
             });
           }).then(async () => {
-              console.log("...loading image.");
+            const image = await canvas.loadImage(picturesPath + photoPath);
-              const image = await canvas.loadImage(picturesPath + filePath);
 
-              console.log("...detecting faces.");
+            const detections = await faceapi.detectAllFaces(image, 
-              const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors();
+                new faceapi.SsdMobilenetv1Options({
+                  minConfidence: 0.8
+                })
+              ).withFaceLandmarks().withFaceDescriptors();
 
+            if (detections.length > 0) {
               console.log(`...${detections.length} faces identified.`);
+            }
             return Promise.map(detections, (face, index) => {
               const width = face.detection._box._width,
                 height = face.detection._box._height,
@@ -103,7 +112,7 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
                   id: photo.id,
                   top: face.detection._box._y / photo.height,
                   left: face.detection._box._x / photo.width,
-                    bottom: (face.detection._box._x + height) / photo.height,
+                  bottom: (face.detection._box._y + height) / photo.height,
                   right: (face.detection._box._x + width) / photo.width,
                   faceConfidence: face.detection._score
                 };
@@ -111,7 +120,7 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
                 "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
                 replacements: replacements
               }).catch(() => {
-                  console.error(filePath, index);
+                console.error(photoPath, index);
                 console.error(JSON.stringify(face, null, 2));
                 console.error(JSON.stringify(replacements, null, 2));
                 process.exit(-1);
@@ -121,12 +130,12 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
                 console.log(`...DB id ${id}. Writing descriptor data...`);
                 const path = faceData + "/" + (id % 100);
                 return mkdir(path).then(() => {
-                    const filePath = path + "/" + id + "-data.json",
+                  const dataPath = path + "/" + id + "-data.json",
                     data = [];
                   for (let i = 0; i < 128; i++) {
                     data.push(face.descriptor[i]);
                   }
-                    fs.writeFileSync(filePath, JSON.stringify(data));
+                  fs.writeFileSync(dataPath, JSON.stringify(data));
                 });
               });
             }).then(() => {
@@ -144,24 +153,131 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
       });
     });
   }).then(() => {
-    console.log("Looking for face distances that need to be updated.");
+    console.log("Looking for face distances that need to be updated...");
+    const descriptors = {};
 
     return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id DESC LIMIT 1", {
-        type: photoDB.sequelize.QueryTypes.SELECT
+      type: photoDB.sequelize.QueryTypes.SELECT,
-      }
+      raw: true
     }).then((results) => {
       if (!results.length) {
-        console.log("No faces exist yet to generate distances.");
+        console.log("...no faces exist yet to generate distances.");
         return;
       }
       const maxId = results[0].id;
-      return photoDB.sequelize.query("SELECT id FROM faces WHERE lastComparedId<:maxId OR lastComparedId IS NULL", {
+      return photoDB.sequelize.query(
+        "SELECT id,lastComparedId " +
+        "FROM faces " +
+        "WHERE lastComparedId<:maxId OR lastComparedId IS NULL", {
         replacements: {
           maxId: maxId
         },
-        type: photoDB.sequelize.QueryTypes.SELECT
+        type: photoDB.sequelize.QueryTypes.SELECT,
+        raw: true
+      }).then((facesToUpdate) => {
+        console.log(`...${facesToUpdate.length} faces need to be updated.`);
+        if (facesToUpdate.length == 0) {
+          return facesToUpdate;
+        }
+
+        return photoDB.sequelize.query(
+          "SELECT id FROM faces", {
+          type: photoDB.sequelize.QueryTypes.SELECT,
+          raw: true
+        }).then((allFaces) => {
+          console.log(`...reading ${allFaces.length} descriptors...`);
+          return Promise.map(allFaces, (face) => {
+            const id = face.id,
+              dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
+
+            if (id in descriptors) {
+              return;
+            }
+
+            return exists(dataPath).then((doesExist) => {
+              if (!doesExist) {
+                console.warn(`${dataPath} is missing!`);
+                return;
+              }
+
+              descriptors[id] = JSON.parse(fs.readFileSync(dataPath));
+            });
+          }, {
+            concurrency: maxConcurrency
+          });
+        }).then(() => {
+          return facesToUpdate;
+        });
+      }).then((facesToUpdate) => {
+        return Promise.mapSeries(facesToUpdate, (face) => {
+          if (!face.id in descriptors) {
+            console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
+            return;
+          }
+
+          const WHERE = "WHERE id" + (face.lastComparedId ? ">:lastId" : "!=:id");
+          return photoDB.sequelize.query(
+            "SELECT id " +
+            "FROM faces " + 
+            WHERE, {
+            replacements: {
+              id: face.id,
+              lastId: face.lastComparedId
+            },
+            type: photoDB.sequelize.QueryTypes.SELECT,
+            raw: true
+          }).then((facesToCompare) => {
+            return photoDB.sequelize.transaction((transaction) => {
+              return Promise.mapSeries(facesToCompare, (target) => {
+                if (!target.id in descriptors) {
+                  console.warn(`...attempt to compare distance with no descriptor for ${target.id}`)
+                  return;
+                }
+
+                return photoDB.sequelize.query(
+                  "SELECT distance " +
+                  "FROM facedistances " +
+                  "WHERE (face1Id=:first AND face2Id=:second) OR (face1Id=:second AND face2Id=:first)", {
+                  replacements: {
+                    first: face.id,
+                    second: target.id
+                  },
+                  type: photoDB.sequelize.QueryTypes.SELECT,
+                  raw: true
                 }).then((results) => {
-        console.log(`${results.length} faces need to be updated.`);
+                  if (results.length) {
+                    return;
+                  }
+                  const distance = faceapi.euclideanDistance(descriptors[face.id], descriptors[target.id]);
+
+                  if (distance < 0.4) {
+                    console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
+                  }
+
+                  return photoDB.sequelize.query(
+                    "INSERT INTO facedistances (face1Id, face2Id, distance) " +
+                    "VALUES (:first, :second, :distance)", {
+                    replacements: {
+                      first: face.id,
+                      second: target.id,
+                      distance: distance
+                    },
+                    transaction: transaction
+                  });
+                });
+              }).then(() => {
+                return photoDB.sequelize.query(
+                  "UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
+                  replacements: {
+                    lastId: maxId,
+                    id: face.id
+                  },
+                  transaction: transaction
+                });
+              });
+            });
+          });
+        });
       });
     });
   }).then(() => {

server/routes/photos.js

@@ -781,6 +781,22 @@ router.get("/trash", function(req, res/*, next*/) {
   });
 });
 
+router.get("/faces/:id", (req, res) => {
+  const id = parseInt(req.params.id);
+  if (id != req.params.id) {
+    return res.status(400).send({ message: "Usage faces/:id"});
+  }
+  return photoDB.sequelize.query("SELECT * FROM faces WHERE photoId=:id", {
+    replacements: {
+      id: id
+    },
+    type: photoDB.Sequelize.QueryTypes.SELECT,
+    raw: true
+  }).then((faces) => {
+    return res.status(200).json(faces);
+  });
+});
+
 router.get("/mvimg/*", function(req, res/*, next*/) {
   let limit = parseInt(req.query.limit) || 50,
     id, cursor, index;