From 046fd0fd1e479e8ec3c96ba5fdae6370ad11ff3a Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Sat, 4 Jan 2020 15:25:46 -0800 Subject: [PATCH] Added face-api usage Signed-off-by: James Ketrenos --- package.json | 5 + server/face-recognizer.js | 212 +++++++++++++++++++++++--------------- 2 files changed, 133 insertions(+), 84 deletions(-) diff --git a/package.json b/package.json index e40041a..9c223a1 100644 --- a/package.json +++ b/package.json @@ -17,14 +17,18 @@ "face-recognition": "^0.9.4" }, "dependencies": { + "@tensorflow/tfjs-core": "^1.5.1", + "@tensorflow/tfjs-node": "^1.5.1", "bluebird": "^3.7.2", "body-parser": "^1.19.0", + "canvas": "^2.6.1", "config": "^1.31.0", "connect-sqlite3": "^0.9.11", "cookie-parser": "^1.4.4", "exif-reader": "github:paras20xx/exif-reader", "express": "^4.17.1", "express-session": "^1.17.0", + "face-api.js": "^0.22.0", "handlebars": "^4.5.3", "ldapauth-fork": "^4.2.0", "ldapjs": "^1.0.2", @@ -33,6 +37,7 @@ "moment-holiday": "^1.5.1", "morgan": "^1.9.1", "mustache": "^3.2.1", + "node-fetch": "^2.6.0", "nodemailer": "^4.7.0", "qs": "^6.9.1", "sequelize": "^4.44.3", diff --git a/server/face-recognizer.js b/server/face-recognizer.js index 6035eab..5409430 100644 --- a/server/face-recognizer.js +++ b/server/face-recognizer.js @@ -19,109 +19,153 @@ process.env.TZ = "Etc/GMT"; console.log("Loading face-recognizer"); +require('@tensorflow/tfjs-node'); + const config = require("config"), Promise = require("bluebird"), - { mkdir, unlink } = require("./lib/util"), - fr = require("face-recognition"); + { exists, mkdir, unlink } = require("./lib/util"), + faceapi = require("face-api.js"), + fs = require("fs"), + canvas = require("canvas"); + +const { Canvas, Image, ImageData } = canvas; + +faceapi.env.monkeyPatch({ Canvas, Image, ImageData }); + +const maxConcurrency = require("os").cpus().length; require("./console-line.js"); /* Monkey-patch console.log with line numbers */ const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/", faceData = picturesPath + "face-data/"; -let photoDB = null, faceDataExists = false; +let photoDB = null; console.log("Loading pictures out of: " + picturesPath); -require("./db/photos").then(function(db) { - photoDB = db; +faceapi.nets.ssdMobilenetv1.loadFromDisk('./models') +.then(() => { + console.log("ssdMobileNetv1 loaded."); + return faceapi.nets.faceLandmark68Net.loadFromDisk('./models'); }).then(() => { - console.log("DB connected."); + console.log("landmark68 loaded."); + return faceapi.nets.faceRecognitionNet.loadFromDisk('./models'); }).then(() => { - console.log("Beginning face detection scanning."); - return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " + - "FROM photos " + - "LEFT JOIN albums ON (albums.id=photos.albumId) " + - "WHERE faces=-1 ORDER BY albums.path,photos.filename", { - type: photoDB.sequelize.QueryTypes.SELECT - } - ).then((results) => { - console.log(`${results.length} photos have not had faces scanned.`); - return Promise.map(results, (photo) => { - const filePath = photo.path + photo.filename; - console.log(`Processing ${filePath}...`); - - return photoDB.sequelize.transaction(function(transaction) { - /* Remove any existing face data for this photo */ - return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", { - transaction: transaction, - replacements: photo, - }).then((faces) => { - /* For each face-id, remove any face-data files, and then remove all the entries - * from the DB */ - return Promise.map(faces, (id) => { - return Promise.mapSeries(["-normalized.png", "-data.json" ], (fileSuffix) => { - const filePath = faceData + "/" + (id % 100) + "/" + id + fileSuffix; + console.log("faceRecognitionNet loaded."); + return require("./db/photos").then(function(db) { + photoDB = db; + }).then(() => { + console.log("DB connected."); + }).then(() => { + console.log("Beginning face detection scanning."); + return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " + + "FROM photos " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE faces=-1 ORDER BY albums.path,photos.filename", { + type: photoDB.sequelize.QueryTypes.SELECT + } + ).then((results) => { + console.log(`${results.length} photos have not had faces scanned.`); + return Promise.map(results, (photo) => { + const filePath = photo.path + photo.filename; + console.log(`Processing ${filePath}...`); + /* Remove any existing face data for this photo */ + return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", { + replacements: photo, + }).then((faces) => { + /* For each face-id, remove any face-data files, and then remove all the entries + * from the DB */ + return Promise.map(faces, (face) => { + const id = face.id, + filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json"; return exists(filePath).then((result) => { - console.log(`...removing ${filePath}`); - return unlink(filePath); - }); - }); - }).then(() => { - return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", { - transaction: transaction, - replacements: photo, - }); - }).then(() => { - const image = fr.loadImage(filePath), - detector = fr.FaceDetector(); - - console.log("...detecting faces."); - const faceRectangles = detector.locateFaces(image) - if (faceRectangles.length == 0) { - console.log("...no faces found in image."); - return; - } - - /* Create a face entry in photos for each face found. */ - const faceImages = detector.detectFaces(image, 200) - - console.log(`...saving ${faceImages.length} faces.`); - - return Promise.map(faceRectangles, (face, index) => { - return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " + - "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", { - replacements: { - id: photo.id, - top: face.top / photo.height, - left: face.left / photo.width, - bottom: face.top / photo.height, - right: face.right / photo.width, - faceConfidence: face.confidence - }, - transaction: transaction - }).spread((results, metadata) => { - return metadata.lastID; - }).then((id) => { - console.log(`...DB id ${id}. Writing data and images...`); - const filePathPrefix = faceData + "/" + (id % 100) + "/" + id; - /* https://medium.com/@ageitgey/machine-learning-is-fun-part-4-modern-face-recognition-with-deep-learning-c3cffc121d78 */ - const data = []; - for (let i = 0; i < 128; i++) { - data.push(Math.random() - 0.5); + if (result) { + console.log(`...removing ${filePath}`); + return unlink(filePath); } - fs.writeFileSync(filePathPrefix + "-data.json", JSON.stringify(data)); - fr.saveImage(filePathPrefix + "-normalized.png", faceImages[index]); + }); + }).then(() => { + return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", { + replacements: photo, + }); + }).then(async () => { + console.log("...loading image."); + const image = await canvas.loadImage(picturesPath + filePath); + + console.log("...detecting faces."); + const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors(); + + console.log(`...${detections.length} faces identified.`); + return Promise.map(detections, (face, index) => { + const width = face.detection._box._width, + height = face.detection._box._height, + replacements = { + id: photo.id, + top: face.detection._box._y / photo.height, + left: face.detection._box._x / photo.width, + bottom: (face.detection._box._x + height) / photo.height, + right: (face.detection._box._x + width) / photo.width, + faceConfidence: face.detection._score + }; + return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " + + "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", { + replacements: replacements + }).catch(() => { + console.error(filePath, index); + console.error(JSON.stringify(face, null, 2)); + console.error(JSON.stringify(replacements, null, 2)); + process.exit(-1); + }).spread((results, metadata) => { + return metadata.lastID; + }).then((id) => { + console.log(`...DB id ${id}. Writing descriptor data...`); + const path = faceData + "/" + (id % 100); + return mkdir(path).then(() => { + const filePath = path + "/" + id + "-data.json", + data = []; + for (let i = 0; i < 128; i++) { + data.push(face.descriptor[i]); + } + fs.writeFileSync(filePath, JSON.stringify(data)); + }); + }); + }).then(() => { + return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", { + replacements: { + id: photo.id, + faces: detections.length + }, + }); }); }); - }); }); + }, { + concurrency: maxConcurrency }); }); + }).then(() => { + console.log("Face detection scanning completed."); + }).catch((error) => { + console.error(error); + process.exit(-1); }); -}).then(() => { - console.log("Face detection scanning completed."); -}).catch((error) => { - console.error(error); - process.exit(-1); }); + +/* TODO: +1. For each path / person, look up highest face confidence and tag +2. Use highest face and identity confidence for input into +https://github.com/justadudewhohacks/face-api.js#face-recognition-by-matching-descriptors + +const labeledDescriptors = [ + new faceapi.LabeledFaceDescriptors( + 'obama', + [descriptorObama1, descriptorObama2] + ), + new faceapi.LabeledFaceDescriptors( + 'trump', + [descriptorTrump] + ) +] + +const faceMatcher = new faceapi.FaceMatcher(labeledDescriptors) +*/