ketr.photos/server/face-recognizer.js

/*
* Face recognition:
* 1. For each photo, extract all faces. Store face rectangles.
*    face_id unique
*    photo_id foreign key
*    top left bottom right
*    identity_id
*    distance (0 == truth; manually assigned identity)
* 2. For each face_id, create:
*    /${picturesPath}face-data/${face_id % 100}/
*      ${face_id}-normalized
*      ${face_id}-original
*      ${face_id}-data
*/

"use strict";

process.env.TZ = "Etc/GMT";

console.log("Loading face-recognizer");

require('@tensorflow/tfjs-node');

const config = require("config"),
  Promise = require("bluebird"),
  { exists, mkdir, unlink } = require("./lib/util"),
  faceapi = require("face-api.js"),
  fs = require("fs"),
  canvas = require("canvas");

const { Canvas, Image, ImageData } = canvas;

faceapi.env.monkeyPatch({ Canvas, Image, ImageData });

const maxConcurrency = require("os").cpus().length;

require("./console-line.js"); /* Monkey-patch console.log with line numbers */

const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
  faceData = picturesPath + "face-data/";

let photoDB = null;

console.log("Loading pictures out of: " + picturesPath);

faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
.then(() => {
  console.log("ssdMobileNetv1 loaded.");
  return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
}).then(() => {
  console.log("landmark68 loaded.");
  return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
}).then(() => {
  console.log("faceRecognitionNet loaded.");
  return require("./db/photos").then(function(db) {
    photoDB = db;
  }).then(() => {
    console.log("DB connected.");
  }).then(() => {
    console.log("Beginning face detection scanning.");
    return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
      "FROM photos " +
      "LEFT JOIN albums ON (albums.id=photos.albumId) " +
      "WHERE faces=-1 ORDER BY albums.path,photos.filename", {
        type: photoDB.sequelize.QueryTypes.SELECT
      }
    ).then((results) => {
      console.log(`${results.length} photos have not had faces scanned.`);
      return Promise.map(results, (photo) => {
        const filePath = photo.path + photo.filename;
        console.log(`Processing ${filePath}...`);
          /* Remove any existing face data for this photo */
          return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
            replacements: photo,
          }).then((faces) => {
            /* For each face-id, remove any face-data files, and then remove all the entries
            * from the DB */
            return Promise.map(faces, (face) => {
              const id = face.id,
                filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
              return exists(filePath).then((result) => {
                if (result) {
                  console.log(`...removing ${filePath}`);
                  return unlink(filePath);
                }
              });
            }).then(() => {
              return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
                replacements: photo,
              });
            }).then(async () => {
              console.log("...loading image.");
              const image = await canvas.loadImage(picturesPath + filePath);

              console.log("...detecting faces.");
              const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors();

              console.log(`...${detections.length} faces identified.`);
              return Promise.map(detections, (face, index) => {
                const width = face.detection._box._width,
                  height = face.detection._box._height,
                  replacements = {
                    id: photo.id,
                    top: face.detection._box._y / photo.height,
                    left: face.detection._box._x / photo.width,
                    bottom: (face.detection._box._x + height) / photo.height,
                    right: (face.detection._box._x + width) / photo.width,
                    faceConfidence: face.detection._score
                  };
                return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
                  "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
                  replacements: replacements
                }).catch(() => {
                  console.error(filePath, index);
                  console.error(JSON.stringify(face, null, 2));
                  console.error(JSON.stringify(replacements, null, 2));
                  process.exit(-1);
                }).spread((results, metadata) => {
                  return metadata.lastID;
                }).then((id) => {
                  console.log(`...DB id ${id}. Writing descriptor data...`);
                  const path = faceData + "/" + (id % 100);
                  return mkdir(path).then(() => {
                    const filePath = path + "/" + id + "-data.json",
                      data = [];
                    for (let i = 0; i < 128; i++) {
                      data.push(face.descriptor[i]);
                    }
                    fs.writeFileSync(filePath, JSON.stringify(data));
                  });
                });
              }).then(() => {
                return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
                  replacements: {
                    id: photo.id,
                    faces: detections.length
                  },
                });
              });
            });
        });
      }, {
        concurrency: maxConcurrency
      });
    });
  }).then(() => {
    console.log("Looking for face distances that need to be updated.");

    return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id DESC LIMIT 1", {
        type: photoDB.sequelize.QueryTypes.SELECT
      }
    }).then((results) => {
      if (!results.length) {
        console.log("No faces exist yet to generate distances.");
        return;
      }
      const maxId = results[0].id;
      return photoDB.sequelize.query("SELECT id FROM faces WHERE lastComparedId<:maxId OR lastComparedId IS NULL", {
        replacements: {
          maxId: maxId
        },
        type: photoDB.sequelize.QueryTypes.SELECT
      }).then((results) => {
        console.log(`${results.length} faces need to be updated.`);
      });
    });
  }).then(() => {
    console.log("Face detection scanning completed.");
  }).catch((error) => {
    console.error(error);
    process.exit(-1);
  });
});

/* TODO:
1. For each path / person, look up highest face confidence and tag
2. Use highest face and identity confidence for input into 
https://github.com/justadudewhohacks/face-api.js#face-recognition-by-matching-descriptors

const labeledDescriptors = [
  new faceapi.LabeledFaceDescriptors(
    'obama',
    [descriptorObama1, descriptorObama2]
  ),
  new faceapi.LabeledFaceDescriptors(
    'trump',
    [descriptorTrump]
  )
]

const faceMatcher = new faceapi.FaceMatcher(labeledDescriptors)
*/