diff --git a/frontend/slideshow.html b/frontend/slideshow.html
index 7892114..85040fd 100755
--- a/frontend/slideshow.html
+++ b/frontend/slideshow.html
@@ -7,6 +7,20 @@ body {
padding: 0;
}
+.face {
+ position: absolute;
+ display: inline-block;
+ border: 1px solid rgb(128,0,0);
+ border-radius: 0.5em;
+ opacity: 0.5;
+ cursor: pointer;
+}
+
+.face:hover {
+ border-color: #ff0000;
+ background-color: rgba(128,0,0,0.5);
+}
+
#photo {
position: fixed;
display: inline-block;
@@ -90,19 +104,22 @@ const days = [ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday",
let activeFaces = [];
function makeFaceBoxes() {
- const el = document.getElementById("photo");
+ Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => {
+ el.parentElement.removeChild(el);
+ });
+
+ const el = document.getElementById("photo"),
+ photo = photos[photoIndex];
let width, height, offsetLeft = 0, offsetTop = 0;
/* If photo is wider than viewport, it will be 100% width and < 100% height */
if (photo.width / photo.height > el.offsetWidth / el.offsetHeight) {
- console.log("A");
width = 100;
height = 100 * photo.height / photo.width * el.offsetWidth / el.offsetHeight;
offsetLeft = 0;
offsetTop = (100 - height) * 0.5;
} else {
- console.log("B");
width = 100 * photo.width / photo.height * el.offsetHeight / el.offsetWidth;
height = 100;
offsetLeft = (100 - width) * 0.5;
@@ -114,15 +131,20 @@ function makeFaceBoxes() {
const box = document.createElement("div");
box.classList.add("face");
document.body.appendChild(box);
- box.style.position = "absolute";
- box.style.display = "inline-block";
- box.style.border = "1px solid red";
- box.style.background = "rgba(255, 0, 0, 0.5)";
- box.style.opacity = 0.5;
box.style.left = offsetLeft + Math.floor(face.left * width) + "%";
box.style.top = offsetTop + Math.floor(face.top * height) + "%";
box.style.width = Math.floor((face.right - face.left) * width) + "%";
box.style.height = Math.floor((face.bottom - face.top) * height) + "%";
+ box.addEventListener("click", (event) => {
+ console.log(face);
+ face.relatedPhotos.forEach((path) => {
+ window.open(base + path);
+ });
+ event.preventDefault = true;
+ event.stopImmediatePropagation();
+ event.stopPropagation();
+ return false;
+ });
});
}
@@ -153,12 +175,9 @@ function loadPhoto(index) {
document.getElementById("photo").style.backgroundImage = "url(" + encodeURI(url) + ")";
countdown = 15;
tick();
- Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => {
- el.parentElement.removeChild(el);
- });
window.fetch("api/v1/photos/faces/" + photo.id).then(res => res.json()).then((faces) => {
activeFaces = faces;
- makeFaceBoxes();
+ makeFaceBoxes(photo);
}).catch(function(error) {
console.error(error);
info.textContent += "Unable to obtain face information :(";
@@ -227,6 +246,14 @@ document.addEventListener("DOMContentLoaded", function() {
base = "/";
}
+ var timeout = 0;
+ window.addEventListener("resize", (event) => {
+ if (timeout) {
+ window.clearTimeout(timeout);
+ }
+ timeout = window.setTimeout(makeFaceBoxes, 250);
+ });
+
document.addEventListener("click", function(event) {
toggleFullscreen();
var now = new Date().getTime();
diff --git a/server/face-recognizer.js b/server/face-recognizer.js
index 50a7221..0d7bd29 100644
--- a/server/face-recognizer.js
+++ b/server/face-recognizer.js
@@ -66,7 +66,9 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
raw: true
}
).then((results) => {
- const remaining = results.length,
+ const total = results.length;
+ let remaining = total,
+ processed = 0,
lastStatus = Date.now();
console.log(`${results.length} photos have not had faces scanned.`);
return Promise.map(results, (photo) => {
@@ -93,60 +95,68 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
replacements: photo,
});
- }).then(async () => {
- const image = await canvas.loadImage(picturesPath + photoPath);
+ });
+ }).then(async () => {
+ const image = await canvas.loadImage(picturesPath + photoPath);
- const detections = await faceapi.detectAllFaces(image,
- new faceapi.SsdMobilenetv1Options({
- minConfidence: 0.8
- })
- ).withFaceLandmarks().withFaceDescriptors();
+ const detections = await faceapi.detectAllFaces(image,
+ new faceapi.SsdMobilenetv1Options({
+ minConfidence: 0.8
+ })
+ ).withFaceLandmarks().withFaceDescriptors();
- if (detections.length > 0) {
- console.log(`...${detections.length} faces identified.`);
- }
- return Promise.map(detections, (face, index) => {
- const width = face.detection._box._width,
- height = face.detection._box._height,
- replacements = {
- id: photo.id,
- top: face.detection._box._y / photo.height,
- left: face.detection._box._x / photo.width,
- bottom: (face.detection._box._y + height) / photo.height,
- right: (face.detection._box._x + width) / photo.width,
- faceConfidence: face.detection._score
- };
- return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
- "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
- replacements: replacements
- }).catch(() => {
- console.error(photoPath, index);
- console.error(JSON.stringify(face, null, 2));
- console.error(JSON.stringify(replacements, null, 2));
- process.exit(-1);
- }).spread((results, metadata) => {
- return metadata.lastID;
- }).then((id) => {
- console.log(`...DB id ${id}. Writing descriptor data...`);
- const path = faceData + "/" + (id % 100);
- return mkdir(path).then(() => {
- const dataPath = path + "/" + id + "-data.json",
- data = [];
- for (let i = 0; i < 128; i++) {
- data.push(face.descriptor[i]);
- }
- fs.writeFileSync(dataPath, JSON.stringify(data));
- });
- });
- }).then(() => {
- return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
- replacements: {
- id: photo.id,
- faces: detections.length
- },
+ if (detections.length > 0) {
+ console.log(`...${detections.length} faces identified.`);
+ }
+
+ return Promise.map(detections, (face) => {
+ const detection = face.detection;
+ const width = detection._box._width,
+ height = detection._box._height,
+ replacements = {
+ id: photo.id,
+ top: detection._box._y / detection._imageDims.height,
+ left: detection._box._x / detection._imageDims.width,
+ bottom: (detection._box._y + height) / detection._imageDims.height,
+ right: (detection._box._x + width) / detection._imageDims.width,
+ faceConfidence: detection._score
+ };
+
+ return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
+ "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
+ replacements: replacements
+ }).spread((results, metadata) => {
+ return metadata.lastID;
+ }).then((id) => {
+ console.log(`...DB id ${id}. Writing descriptor data...`);
+ const path = faceData + "/" + (id % 100);
+ return mkdir(path).then(() => {
+ const dataPath = path + "/" + id + "-data.json",
+ data = [];
+ for (let i = 0; i < 128; i++) {
+ data.push(face.descriptor[i]);
+ }
+ fs.writeFileSync(dataPath, JSON.stringify(data));
});
});
+ }).then(() => {
+ return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
+ replacements: {
+ id: photo.id,
+ faces: detections.length
+ },
+ });
});
+ }).then(() => {
+ processed++;
+ const now = Date.now();
+ if (now - lastStatus > 5000) {
+ const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
+ eta = Math.round((total - processed) / rate);
+ lastStatus = now;
+ remaining = total - processed;
+ console.log(`Processing ${rate} images per second. ${remaining} images to be processed. ETA: ${eta}s`);
+ }
});
}, {
concurrency: maxConcurrency
@@ -180,6 +190,8 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
return facesToUpdate;
}
+ console.log("...removing old assets.");
+
return photoDB.sequelize.query(
"SELECT id FROM faces", {
type: photoDB.sequelize.QueryTypes.SELECT,
@@ -209,73 +221,90 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
return facesToUpdate;
});
}).then((facesToUpdate) => {
+ const total = facesToUpdate.length;
+ let remaining = total,
+ processed = 0,
+ lastStatus = Date.now(),
+ targets = [];
+
+ for (let target in descriptors) {
+ targets.push({ id: target, descriptor: descriptors[target] });
+ }
+
return Promise.mapSeries(facesToUpdate, (face) => {
- if (!face.id in descriptors) {
+ if (!(face.id in descriptors)) {
console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
return;
}
- const WHERE = "WHERE id" + (face.lastComparedId ? ">:lastId" : "!=:id");
- return photoDB.sequelize.query(
- "SELECT id " +
- "FROM faces " +
- WHERE, {
- replacements: {
- id: face.id,
- lastId: face.lastComparedId
- },
- type: photoDB.sequelize.QueryTypes.SELECT,
- raw: true
- }).then((facesToCompare) => {
- return photoDB.sequelize.transaction((transaction) => {
- return Promise.mapSeries(facesToCompare, (target) => {
- if (!target.id in descriptors) {
- console.warn(`...attempt to compare distance with no descriptor for ${target.id}`)
+ const faceDescriptor = descriptors[face.id];
+
+ return photoDB.sequelize.transaction((transaction) => {
+ return Promise.map(targets, (target) => {
+ /* Skip comparing to self */
+ if (target.id == face.id) {
+ return;
+ }
+
+ /* Only compare against newer faces */
+ if (face.lastComparedId && target.id <= face.lastComparedId) {
+ return;
+ }
+
+ return photoDB.sequelize.query(
+ "SELECT distance " +
+ "FROM facedistances " +
+ "WHERE face1Id=:first AND face2Id=:second", {
+ replacements: {
+ first: Math.min(face.id, target.id),
+ second: Math.max(face.id, target.id)
+ },
+ type: photoDB.sequelize.QueryTypes.SELECT,
+ raw: true,
+ transaction: transaction
+ }).then((results) => {
+ if (results.length) {
return;
}
+ const distance = faceapi.euclideanDistance(faceDescriptor, target.descriptor);
+
+ if (distance < 0.4) {
+ console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
+ }
return photoDB.sequelize.query(
- "SELECT distance " +
- "FROM facedistances " +
- "WHERE (face1Id=:first AND face2Id=:second) OR (face1Id=:second AND face2Id=:first)", {
+ "INSERT INTO facedistances (face1Id, face2Id, distance) " +
+ "VALUES (:first, :second, :distance)", {
replacements: {
- first: face.id,
- second: target.id
- },
- type: photoDB.sequelize.QueryTypes.SELECT,
- raw: true
- }).then((results) => {
- if (results.length) {
- return;
- }
- const distance = faceapi.euclideanDistance(descriptors[face.id], descriptors[target.id]);
-
- if (distance < 0.4) {
- console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
- }
-
- return photoDB.sequelize.query(
- "INSERT INTO facedistances (face1Id, face2Id, distance) " +
- "VALUES (:first, :second, :distance)", {
- replacements: {
- first: face.id,
- second: target.id,
- distance: distance
- },
- transaction: transaction
- });
- });
- }).then(() => {
- return photoDB.sequelize.query(
- "UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
- replacements: {
- lastId: maxId,
- id: face.id
+ first: Math.min(face.id, target.id),
+ second: Math.max(face.id, target.id),
+ distance: distance
},
transaction: transaction
});
});
+ }, {
+ concurrency: maxConcurrency
+ }).then(() => {
+ return photoDB.sequelize.query(
+ "UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
+ replacements: {
+ lastId: maxId,
+ id: face.id
+ },
+ transaction: transaction
+ });
});
+ }).then(() => {
+ processed++;
+ const now = Date.now();
+ if (now - lastStatus > 5000) {
+ const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
+ eta = Math.round((total - processed) / rate);
+ lastStatus = now;
+ remaining = total - processed;
+ console.log(`Processing ${rate} faces per second. ${remaining} faces to be processed. ETA: ${eta}s`);
+ }
});
});
});
diff --git a/server/face.js b/server/face.js
new file mode 100644
index 0000000..e159541
--- /dev/null
+++ b/server/face.js
@@ -0,0 +1,64 @@
+"use strict";
+
+process.env.TZ = "Etc/GMT";
+
+require('@tensorflow/tfjs-node');
+
+const config = require("config"),
+ Promise = require("bluebird"),
+ { exists, mkdir, unlink } = require("./lib/util"),
+ faceapi = require("face-api.js"),
+ fs = require("fs"),
+ canvas = require("canvas");
+
+const { Canvas, Image, ImageData } = canvas;
+
+faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
+
+const maxConcurrency = require("os").cpus().length;
+
+require("./console-line.js"); /* Monkey-patch console.log with line numbers */
+
+faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
+.then(() => {
+ console.log("ssdMobileNetv1 loaded.");
+ return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
+}).then(() => {
+ console.log("landmark68 loaded.");
+ return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
+}).then(async () => {
+ console.log("faceRecognitionNet loaded.");
+ let faces = [];
+ for (let a = 2; a < process.argv.length; a++) {
+ const file = process.argv[a];
+ process.stdout.write(`Loading ${file}...`);
+ const image = await canvas.loadImage(file),
+ detectors = await faceapi.detectAllFaces(image,
+ new faceapi.SsdMobilenetv1Options({
+ minConfidence: 0.8
+ })
+ ).withFaceLandmarks().withFaceDescriptors();
+ process.stdout.write(`${detectors.length} faces.\n`);
+
+ detectors.forEach((face, index) => {
+ faces.push({
+ file: file,
+ index: index,
+ descriptor: face.descriptor
+ })
+ });
+ }
+
+ for (let i = 0; i < faces.length; i++) {
+ for (let j = 0; j < faces.length; j++) {
+ const a = faces[i], b = faces[j];
+ const distance = faceapi.euclideanDistance(a.descriptor, b.descriptor);
+ console.log(`${a.file}.${a.index} to ${b.file}.${b.index} = ${distance}`);
+ }
+ }
+}).then(() => {
+ console.log("Face detection scanning completed.");
+}).catch((error) => {
+ console.error(error);
+ process.exit(-1);
+});
diff --git a/server/routes/photos.js b/server/routes/photos.js
index 76f81d9..e0d7695 100755
--- a/server/routes/photos.js
+++ b/server/routes/photos.js
@@ -793,7 +793,40 @@ router.get("/faces/:id", (req, res) => {
type: photoDB.Sequelize.QueryTypes.SELECT,
raw: true
}).then((faces) => {
- return res.status(200).json(faces);
+ return Promise.map(faces, (face) => {
+ return photoDB.sequelize.query(
+ "SELECT face1ID,face2ID " +
+ "FROM facedistances " +
+ "WHERE distance<0.45 AND (face1ID=:id OR face2ID=:id) " +
+ "ORDER BY distance ASC", {
+ replacements: {
+ id: face.id
+ },
+ type: photoDB.Sequelize.QueryTypes.SELECT,
+ raw: true
+ }).then((faceIds) => {
+ return photoDB.sequelize.query(
+ "SELECT photos.id,albums.path,photos.filename " +
+ "FROM faces " +
+ "LEFT JOIN photos ON photos.id=faces.photoId " +
+ "LEFT JOIN albums ON albums.id=photos.albumId " +
+ "WHERE faces.id IN (:ids)", {
+ replacements: {
+ ids: faceIds.map((face) => {
+ return (face.face1Id == face.id) ? face.face2Id : face.face1Id;
+ })
+ },
+ type: photoDB.Sequelize.QueryTypes.SELECT,
+ raw: true
+ });
+ }).then((photos) => {
+ face.relatedPhotos = photos.filter((photo) => { return photo.id != id }).map((photo) => {
+ return photo.path + photo.filename;
+ });
+ });
+ }).then(() => {
+ return res.status(200).json(faces);
+ });
});
});