diff --git a/frontend/slideshow.html b/frontend/slideshow.html
index d29c38c..7892114 100755
--- a/frontend/slideshow.html
+++ b/frontend/slideshow.html
@@ -87,6 +87,45 @@ const days = [ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday",
months = [ "January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December" ];
+let activeFaces = [];
+
+function makeFaceBoxes() {
+ const el = document.getElementById("photo");
+
+ let width, height, offsetLeft = 0, offsetTop = 0;
+
+ /* If photo is wider than viewport, it will be 100% width and < 100% height */
+ if (photo.width / photo.height > el.offsetWidth / el.offsetHeight) {
+ console.log("A");
+ width = 100;
+ height = 100 * photo.height / photo.width * el.offsetWidth / el.offsetHeight;
+ offsetLeft = 0;
+ offsetTop = (100 - height) * 0.5;
+ } else {
+ console.log("B");
+ width = 100 * photo.width / photo.height * el.offsetHeight / el.offsetWidth;
+ height = 100;
+ offsetLeft = (100 - width) * 0.5;
+ offsetTop = 0;
+ }
+
+
+ activeFaces.forEach((face) => {
+ const box = document.createElement("div");
+ box.classList.add("face");
+ document.body.appendChild(box);
+ box.style.position = "absolute";
+ box.style.display = "inline-block";
+ box.style.border = "1px solid red";
+ box.style.background = "rgba(255, 0, 0, 0.5)";
+ box.style.opacity = 0.5;
+ box.style.left = offsetLeft + Math.floor(face.left * width) + "%";
+ box.style.top = offsetTop + Math.floor(face.top * height) + "%";
+ box.style.width = Math.floor((face.right - face.left) * width) + "%";
+ box.style.height = Math.floor((face.bottom - face.top) * height) + "%";
+ });
+}
+
function loadPhoto(index) {
const photo = photos[index],
xml = new XMLHttpRequest(),
@@ -114,6 +153,16 @@ function loadPhoto(index) {
document.getElementById("photo").style.backgroundImage = "url(" + encodeURI(url) + ")";
countdown = 15;
tick();
+ Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => {
+ el.parentElement.removeChild(el);
+ });
+ window.fetch("api/v1/photos/faces/" + photo.id).then(res => res.json()).then((faces) => {
+ activeFaces = faces;
+ makeFaceBoxes();
+ }).catch(function(error) {
+ console.error(error);
+ info.textContent += "Unable to obtain face information :(";
+ });
}
xml.onerror = function(event) {
diff --git a/server/db/photos.js b/server/db/photos.js
index 342c648..a652f32 100755
--- a/server/db/photos.js
+++ b/server/db/photos.js
@@ -151,19 +151,19 @@ function init() {
});
const FaceDistances = db.sequelize.define('facedistance', {
- photoId: {
+ face1Id: {
type: Sequelize.INTEGER,
allowNull: false,
references: {
- model: Photo,
+ model: Face,
key: 'id',
}
},
- targetId: {
+ face2Id: {
type: Sequelize.INTEGER,
allowNull: false,
references: {
- model: Photo,
+ model: Face,
key: 'id',
}
},
diff --git a/server/face-recognizer.js b/server/face-recognizer.js
index 2b167d4..50a7221 100644
--- a/server/face-recognizer.js
+++ b/server/face-recognizer.js
@@ -62,106 +62,222 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
"FROM photos " +
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
"WHERE faces=-1 ORDER BY albums.path,photos.filename", {
- type: photoDB.sequelize.QueryTypes.SELECT
+ type: photoDB.sequelize.QueryTypes.SELECT,
+ raw: true
}
).then((results) => {
+ const remaining = results.length,
+ lastStatus = Date.now();
console.log(`${results.length} photos have not had faces scanned.`);
return Promise.map(results, (photo) => {
- const filePath = photo.path + photo.filename;
- console.log(`Processing ${filePath}...`);
- /* Remove any existing face data for this photo */
- return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
- replacements: photo,
- }).then((faces) => {
- /* For each face-id, remove any face-data files, and then remove all the entries
+ const photoPath = photo.path + photo.filename;
+
+ console.log(`Processing ${photoPath}...`);
+
+ /* Remove any existing face data for this photo */
+ return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
+ replacements: photo,
+ }).then((faces) => {
+ /* For each face-id, remove any face-data files, and then remove all the entries
* from the DB */
- return Promise.map(faces, (face) => {
- const id = face.id,
- filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
- return exists(filePath).then((result) => {
- if (result) {
- console.log(`...removing ${filePath}`);
- return unlink(filePath);
- }
+ return Promise.map(faces, (face) => {
+ const id = face.id,
+ dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
+ return exists(dataPath).then((result) => {
+ if (result) {
+ console.log(`...removing ${dataPath}`);
+ return unlink(dataPath);
+ }
+ });
+ }).then(() => {
+ return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
+ replacements: photo,
+ });
+ }).then(async () => {
+ const image = await canvas.loadImage(picturesPath + photoPath);
+
+ const detections = await faceapi.detectAllFaces(image,
+ new faceapi.SsdMobilenetv1Options({
+ minConfidence: 0.8
+ })
+ ).withFaceLandmarks().withFaceDescriptors();
+
+ if (detections.length > 0) {
+ console.log(`...${detections.length} faces identified.`);
+ }
+ return Promise.map(detections, (face, index) => {
+ const width = face.detection._box._width,
+ height = face.detection._box._height,
+ replacements = {
+ id: photo.id,
+ top: face.detection._box._y / photo.height,
+ left: face.detection._box._x / photo.width,
+ bottom: (face.detection._box._y + height) / photo.height,
+ right: (face.detection._box._x + width) / photo.width,
+ faceConfidence: face.detection._score
+ };
+ return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
+ "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
+ replacements: replacements
+ }).catch(() => {
+ console.error(photoPath, index);
+ console.error(JSON.stringify(face, null, 2));
+ console.error(JSON.stringify(replacements, null, 2));
+ process.exit(-1);
+ }).spread((results, metadata) => {
+ return metadata.lastID;
+ }).then((id) => {
+ console.log(`...DB id ${id}. Writing descriptor data...`);
+ const path = faceData + "/" + (id % 100);
+ return mkdir(path).then(() => {
+ const dataPath = path + "/" + id + "-data.json",
+ data = [];
+ for (let i = 0; i < 128; i++) {
+ data.push(face.descriptor[i]);
+ }
+ fs.writeFileSync(dataPath, JSON.stringify(data));
+ });
});
}).then(() => {
- return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
- replacements: photo,
- });
- }).then(async () => {
- console.log("...loading image.");
- const image = await canvas.loadImage(picturesPath + filePath);
-
- console.log("...detecting faces.");
- const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors();
-
- console.log(`...${detections.length} faces identified.`);
- return Promise.map(detections, (face, index) => {
- const width = face.detection._box._width,
- height = face.detection._box._height,
- replacements = {
- id: photo.id,
- top: face.detection._box._y / photo.height,
- left: face.detection._box._x / photo.width,
- bottom: (face.detection._box._x + height) / photo.height,
- right: (face.detection._box._x + width) / photo.width,
- faceConfidence: face.detection._score
- };
- return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
- "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
- replacements: replacements
- }).catch(() => {
- console.error(filePath, index);
- console.error(JSON.stringify(face, null, 2));
- console.error(JSON.stringify(replacements, null, 2));
- process.exit(-1);
- }).spread((results, metadata) => {
- return metadata.lastID;
- }).then((id) => {
- console.log(`...DB id ${id}. Writing descriptor data...`);
- const path = faceData + "/" + (id % 100);
- return mkdir(path).then(() => {
- const filePath = path + "/" + id + "-data.json",
- data = [];
- for (let i = 0; i < 128; i++) {
- data.push(face.descriptor[i]);
- }
- fs.writeFileSync(filePath, JSON.stringify(data));
- });
- });
- }).then(() => {
- return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
- replacements: {
- id: photo.id,
- faces: detections.length
- },
- });
+ return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
+ replacements: {
+ id: photo.id,
+ faces: detections.length
+ },
});
});
+ });
});
}, {
concurrency: maxConcurrency
});
});
}).then(() => {
- console.log("Looking for face distances that need to be updated.");
+ console.log("Looking for face distances that need to be updated...");
+ const descriptors = {};
return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id DESC LIMIT 1", {
- type: photoDB.sequelize.QueryTypes.SELECT
- }
+ type: photoDB.sequelize.QueryTypes.SELECT,
+ raw: true
}).then((results) => {
if (!results.length) {
- console.log("No faces exist yet to generate distances.");
+ console.log("...no faces exist yet to generate distances.");
return;
}
const maxId = results[0].id;
- return photoDB.sequelize.query("SELECT id FROM faces WHERE lastComparedId<:maxId OR lastComparedId IS NULL", {
+ return photoDB.sequelize.query(
+ "SELECT id,lastComparedId " +
+ "FROM faces " +
+ "WHERE lastComparedId<:maxId OR lastComparedId IS NULL", {
replacements: {
maxId: maxId
},
- type: photoDB.sequelize.QueryTypes.SELECT
- }).then((results) => {
- console.log(`${results.length} faces need to be updated.`);
+ type: photoDB.sequelize.QueryTypes.SELECT,
+ raw: true
+ }).then((facesToUpdate) => {
+ console.log(`...${facesToUpdate.length} faces need to be updated.`);
+ if (facesToUpdate.length == 0) {
+ return facesToUpdate;
+ }
+
+ return photoDB.sequelize.query(
+ "SELECT id FROM faces", {
+ type: photoDB.sequelize.QueryTypes.SELECT,
+ raw: true
+ }).then((allFaces) => {
+ console.log(`...reading ${allFaces.length} descriptors...`);
+ return Promise.map(allFaces, (face) => {
+ const id = face.id,
+ dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
+
+ if (id in descriptors) {
+ return;
+ }
+
+ return exists(dataPath).then((doesExist) => {
+ if (!doesExist) {
+ console.warn(`${dataPath} is missing!`);
+ return;
+ }
+
+ descriptors[id] = JSON.parse(fs.readFileSync(dataPath));
+ });
+ }, {
+ concurrency: maxConcurrency
+ });
+ }).then(() => {
+ return facesToUpdate;
+ });
+ }).then((facesToUpdate) => {
+ return Promise.mapSeries(facesToUpdate, (face) => {
+ if (!face.id in descriptors) {
+ console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
+ return;
+ }
+
+ const WHERE = "WHERE id" + (face.lastComparedId ? ">:lastId" : "!=:id");
+ return photoDB.sequelize.query(
+ "SELECT id " +
+ "FROM faces " +
+ WHERE, {
+ replacements: {
+ id: face.id,
+ lastId: face.lastComparedId
+ },
+ type: photoDB.sequelize.QueryTypes.SELECT,
+ raw: true
+ }).then((facesToCompare) => {
+ return photoDB.sequelize.transaction((transaction) => {
+ return Promise.mapSeries(facesToCompare, (target) => {
+ if (!target.id in descriptors) {
+ console.warn(`...attempt to compare distance with no descriptor for ${target.id}`)
+ return;
+ }
+
+ return photoDB.sequelize.query(
+ "SELECT distance " +
+ "FROM facedistances " +
+ "WHERE (face1Id=:first AND face2Id=:second) OR (face1Id=:second AND face2Id=:first)", {
+ replacements: {
+ first: face.id,
+ second: target.id
+ },
+ type: photoDB.sequelize.QueryTypes.SELECT,
+ raw: true
+ }).then((results) => {
+ if (results.length) {
+ return;
+ }
+ const distance = faceapi.euclideanDistance(descriptors[face.id], descriptors[target.id]);
+
+ if (distance < 0.4) {
+ console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
+ }
+
+ return photoDB.sequelize.query(
+ "INSERT INTO facedistances (face1Id, face2Id, distance) " +
+ "VALUES (:first, :second, :distance)", {
+ replacements: {
+ first: face.id,
+ second: target.id,
+ distance: distance
+ },
+ transaction: transaction
+ });
+ });
+ }).then(() => {
+ return photoDB.sequelize.query(
+ "UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
+ replacements: {
+ lastId: maxId,
+ id: face.id
+ },
+ transaction: transaction
+ });
+ });
+ });
+ });
+ });
});
});
}).then(() => {
diff --git a/server/routes/photos.js b/server/routes/photos.js
index bd5227d..76f81d9 100755
--- a/server/routes/photos.js
+++ b/server/routes/photos.js
@@ -781,6 +781,22 @@ router.get("/trash", function(req, res/*, next*/) {
});
});
+router.get("/faces/:id", (req, res) => {
+ const id = parseInt(req.params.id);
+ if (id != req.params.id) {
+ return res.status(400).send({ message: "Usage faces/:id"});
+ }
+ return photoDB.sequelize.query("SELECT * FROM faces WHERE photoId=:id", {
+ replacements: {
+ id: id
+ },
+ type: photoDB.Sequelize.QueryTypes.SELECT,
+ raw: true
+ }).then((faces) => {
+ return res.status(200).json(faces);
+ });
+});
+
router.get("/mvimg/*", function(req, res/*, next*/) {
let limit = parseInt(req.query.limit) || 50,
id, cursor, index;