From 72efd92b99fc24ee6ff551434456c6b331c9606e Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Sat, 4 Jan 2020 22:51:07 -0800 Subject: [PATCH] Face scanning making progress. Working on face-box. Signed-off-by: James Ketrenos --- frontend/slideshow.html | 49 +++++++ server/db/photos.js | 8 +- server/face-recognizer.js | 266 +++++++++++++++++++++++++++----------- server/routes/photos.js | 16 +++ 4 files changed, 260 insertions(+), 79 deletions(-) diff --git a/frontend/slideshow.html b/frontend/slideshow.html index d29c38c..7892114 100755 --- a/frontend/slideshow.html +++ b/frontend/slideshow.html @@ -87,6 +87,45 @@ const days = [ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", months = [ "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December" ]; +let activeFaces = []; + +function makeFaceBoxes() { + const el = document.getElementById("photo"); + + let width, height, offsetLeft = 0, offsetTop = 0; + + /* If photo is wider than viewport, it will be 100% width and < 100% height */ + if (photo.width / photo.height > el.offsetWidth / el.offsetHeight) { + console.log("A"); + width = 100; + height = 100 * photo.height / photo.width * el.offsetWidth / el.offsetHeight; + offsetLeft = 0; + offsetTop = (100 - height) * 0.5; + } else { + console.log("B"); + width = 100 * photo.width / photo.height * el.offsetHeight / el.offsetWidth; + height = 100; + offsetLeft = (100 - width) * 0.5; + offsetTop = 0; + } + + + activeFaces.forEach((face) => { + const box = document.createElement("div"); + box.classList.add("face"); + document.body.appendChild(box); + box.style.position = "absolute"; + box.style.display = "inline-block"; + box.style.border = "1px solid red"; + box.style.background = "rgba(255, 0, 0, 0.5)"; + box.style.opacity = 0.5; + box.style.left = offsetLeft + Math.floor(face.left * width) + "%"; + box.style.top = offsetTop + Math.floor(face.top * height) + "%"; + box.style.width = Math.floor((face.right - face.left) * width) + "%"; + box.style.height = Math.floor((face.bottom - face.top) * height) + "%"; + }); +} + function loadPhoto(index) { const photo = photos[index], xml = new XMLHttpRequest(), @@ -114,6 +153,16 @@ function loadPhoto(index) { document.getElementById("photo").style.backgroundImage = "url(" + encodeURI(url) + ")"; countdown = 15; tick(); + Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => { + el.parentElement.removeChild(el); + }); + window.fetch("api/v1/photos/faces/" + photo.id).then(res => res.json()).then((faces) => { + activeFaces = faces; + makeFaceBoxes(); + }).catch(function(error) { + console.error(error); + info.textContent += "Unable to obtain face information :("; + }); } xml.onerror = function(event) { diff --git a/server/db/photos.js b/server/db/photos.js index 342c648..a652f32 100755 --- a/server/db/photos.js +++ b/server/db/photos.js @@ -151,19 +151,19 @@ function init() { }); const FaceDistances = db.sequelize.define('facedistance', { - photoId: { + face1Id: { type: Sequelize.INTEGER, allowNull: false, references: { - model: Photo, + model: Face, key: 'id', } }, - targetId: { + face2Id: { type: Sequelize.INTEGER, allowNull: false, references: { - model: Photo, + model: Face, key: 'id', } }, diff --git a/server/face-recognizer.js b/server/face-recognizer.js index 2b167d4..50a7221 100644 --- a/server/face-recognizer.js +++ b/server/face-recognizer.js @@ -62,106 +62,222 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models') "FROM photos " + "LEFT JOIN albums ON (albums.id=photos.albumId) " + "WHERE faces=-1 ORDER BY albums.path,photos.filename", { - type: photoDB.sequelize.QueryTypes.SELECT + type: photoDB.sequelize.QueryTypes.SELECT, + raw: true } ).then((results) => { + const remaining = results.length, + lastStatus = Date.now(); console.log(`${results.length} photos have not had faces scanned.`); return Promise.map(results, (photo) => { - const filePath = photo.path + photo.filename; - console.log(`Processing ${filePath}...`); - /* Remove any existing face data for this photo */ - return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", { - replacements: photo, - }).then((faces) => { - /* For each face-id, remove any face-data files, and then remove all the entries + const photoPath = photo.path + photo.filename; + + console.log(`Processing ${photoPath}...`); + + /* Remove any existing face data for this photo */ + return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", { + replacements: photo, + }).then((faces) => { + /* For each face-id, remove any face-data files, and then remove all the entries * from the DB */ - return Promise.map(faces, (face) => { - const id = face.id, - filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json"; - return exists(filePath).then((result) => { - if (result) { - console.log(`...removing ${filePath}`); - return unlink(filePath); - } + return Promise.map(faces, (face) => { + const id = face.id, + dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json"; + return exists(dataPath).then((result) => { + if (result) { + console.log(`...removing ${dataPath}`); + return unlink(dataPath); + } + }); + }).then(() => { + return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", { + replacements: photo, + }); + }).then(async () => { + const image = await canvas.loadImage(picturesPath + photoPath); + + const detections = await faceapi.detectAllFaces(image, + new faceapi.SsdMobilenetv1Options({ + minConfidence: 0.8 + }) + ).withFaceLandmarks().withFaceDescriptors(); + + if (detections.length > 0) { + console.log(`...${detections.length} faces identified.`); + } + return Promise.map(detections, (face, index) => { + const width = face.detection._box._width, + height = face.detection._box._height, + replacements = { + id: photo.id, + top: face.detection._box._y / photo.height, + left: face.detection._box._x / photo.width, + bottom: (face.detection._box._y + height) / photo.height, + right: (face.detection._box._x + width) / photo.width, + faceConfidence: face.detection._score + }; + return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " + + "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", { + replacements: replacements + }).catch(() => { + console.error(photoPath, index); + console.error(JSON.stringify(face, null, 2)); + console.error(JSON.stringify(replacements, null, 2)); + process.exit(-1); + }).spread((results, metadata) => { + return metadata.lastID; + }).then((id) => { + console.log(`...DB id ${id}. Writing descriptor data...`); + const path = faceData + "/" + (id % 100); + return mkdir(path).then(() => { + const dataPath = path + "/" + id + "-data.json", + data = []; + for (let i = 0; i < 128; i++) { + data.push(face.descriptor[i]); + } + fs.writeFileSync(dataPath, JSON.stringify(data)); + }); }); }).then(() => { - return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", { - replacements: photo, - }); - }).then(async () => { - console.log("...loading image."); - const image = await canvas.loadImage(picturesPath + filePath); - - console.log("...detecting faces."); - const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors(); - - console.log(`...${detections.length} faces identified.`); - return Promise.map(detections, (face, index) => { - const width = face.detection._box._width, - height = face.detection._box._height, - replacements = { - id: photo.id, - top: face.detection._box._y / photo.height, - left: face.detection._box._x / photo.width, - bottom: (face.detection._box._x + height) / photo.height, - right: (face.detection._box._x + width) / photo.width, - faceConfidence: face.detection._score - }; - return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " + - "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", { - replacements: replacements - }).catch(() => { - console.error(filePath, index); - console.error(JSON.stringify(face, null, 2)); - console.error(JSON.stringify(replacements, null, 2)); - process.exit(-1); - }).spread((results, metadata) => { - return metadata.lastID; - }).then((id) => { - console.log(`...DB id ${id}. Writing descriptor data...`); - const path = faceData + "/" + (id % 100); - return mkdir(path).then(() => { - const filePath = path + "/" + id + "-data.json", - data = []; - for (let i = 0; i < 128; i++) { - data.push(face.descriptor[i]); - } - fs.writeFileSync(filePath, JSON.stringify(data)); - }); - }); - }).then(() => { - return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", { - replacements: { - id: photo.id, - faces: detections.length - }, - }); + return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", { + replacements: { + id: photo.id, + faces: detections.length + }, }); }); + }); }); }, { concurrency: maxConcurrency }); }); }).then(() => { - console.log("Looking for face distances that need to be updated."); + console.log("Looking for face distances that need to be updated..."); + const descriptors = {}; return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id DESC LIMIT 1", { - type: photoDB.sequelize.QueryTypes.SELECT - } + type: photoDB.sequelize.QueryTypes.SELECT, + raw: true }).then((results) => { if (!results.length) { - console.log("No faces exist yet to generate distances."); + console.log("...no faces exist yet to generate distances."); return; } const maxId = results[0].id; - return photoDB.sequelize.query("SELECT id FROM faces WHERE lastComparedId<:maxId OR lastComparedId IS NULL", { + return photoDB.sequelize.query( + "SELECT id,lastComparedId " + + "FROM faces " + + "WHERE lastComparedId<:maxId OR lastComparedId IS NULL", { replacements: { maxId: maxId }, - type: photoDB.sequelize.QueryTypes.SELECT - }).then((results) => { - console.log(`${results.length} faces need to be updated.`); + type: photoDB.sequelize.QueryTypes.SELECT, + raw: true + }).then((facesToUpdate) => { + console.log(`...${facesToUpdate.length} faces need to be updated.`); + if (facesToUpdate.length == 0) { + return facesToUpdate; + } + + return photoDB.sequelize.query( + "SELECT id FROM faces", { + type: photoDB.sequelize.QueryTypes.SELECT, + raw: true + }).then((allFaces) => { + console.log(`...reading ${allFaces.length} descriptors...`); + return Promise.map(allFaces, (face) => { + const id = face.id, + dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json"; + + if (id in descriptors) { + return; + } + + return exists(dataPath).then((doesExist) => { + if (!doesExist) { + console.warn(`${dataPath} is missing!`); + return; + } + + descriptors[id] = JSON.parse(fs.readFileSync(dataPath)); + }); + }, { + concurrency: maxConcurrency + }); + }).then(() => { + return facesToUpdate; + }); + }).then((facesToUpdate) => { + return Promise.mapSeries(facesToUpdate, (face) => { + if (!face.id in descriptors) { + console.warn(`...attempt to compare distance with no descriptor for ${face.id}`); + return; + } + + const WHERE = "WHERE id" + (face.lastComparedId ? ">:lastId" : "!=:id"); + return photoDB.sequelize.query( + "SELECT id " + + "FROM faces " + + WHERE, { + replacements: { + id: face.id, + lastId: face.lastComparedId + }, + type: photoDB.sequelize.QueryTypes.SELECT, + raw: true + }).then((facesToCompare) => { + return photoDB.sequelize.transaction((transaction) => { + return Promise.mapSeries(facesToCompare, (target) => { + if (!target.id in descriptors) { + console.warn(`...attempt to compare distance with no descriptor for ${target.id}`) + return; + } + + return photoDB.sequelize.query( + "SELECT distance " + + "FROM facedistances " + + "WHERE (face1Id=:first AND face2Id=:second) OR (face1Id=:second AND face2Id=:first)", { + replacements: { + first: face.id, + second: target.id + }, + type: photoDB.sequelize.QueryTypes.SELECT, + raw: true + }).then((results) => { + if (results.length) { + return; + } + const distance = faceapi.euclideanDistance(descriptors[face.id], descriptors[target.id]); + + if (distance < 0.4) { + console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`); + } + + return photoDB.sequelize.query( + "INSERT INTO facedistances (face1Id, face2Id, distance) " + + "VALUES (:first, :second, :distance)", { + replacements: { + first: face.id, + second: target.id, + distance: distance + }, + transaction: transaction + }); + }); + }).then(() => { + return photoDB.sequelize.query( + "UPDATE faces SET lastComparedId=:lastId WHERE id=:id", { + replacements: { + lastId: maxId, + id: face.id + }, + transaction: transaction + }); + }); + }); + }); + }); }); }); }).then(() => { diff --git a/server/routes/photos.js b/server/routes/photos.js index bd5227d..76f81d9 100755 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -781,6 +781,22 @@ router.get("/trash", function(req, res/*, next*/) { }); }); +router.get("/faces/:id", (req, res) => { + const id = parseInt(req.params.id); + if (id != req.params.id) { + return res.status(400).send({ message: "Usage faces/:id"}); + } + return photoDB.sequelize.query("SELECT * FROM faces WHERE photoId=:id", { + replacements: { + id: id + }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }).then((faces) => { + return res.status(200).json(faces); + }); +}); + router.get("/mvimg/*", function(req, res/*, next*/) { let limit = parseInt(req.query.limit) || 50, id, cursor, index;