jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Face recognition is working

Browse Source 
Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>
This commit is contained in:
James Ketrenos 2020-01-05 01:18:31 -08:00
parent 72efd92b99
commit 694a6e0a39
4 changed files with 268 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
frontend/slideshow.html
View File

@ -7,6 +7,20 @@ body {
padding: 0;
}
.face {
position: absolute;
display: inline-block;
border: 1px solid rgb(128,0,0);
border-radius: 0.5em;
opacity: 0.5;
cursor: pointer;
}
.face:hover {
border-color: #ff0000;
background-color: rgba(128,0,0,0.5);
}
#photo {
position: fixed;
display: inline-block;
@ -90,19 +104,22 @@ const days = [ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday",
let activeFaces = [];
function makeFaceBoxes() {
const el = document.getElementById("photo");
Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => {
el.parentElement.removeChild(el);
});
const el = document.getElementById("photo"),
photo = photos[photoIndex];
let width, height, offsetLeft = 0, offsetTop = 0;
/* If photo is wider than viewport, it will be 100% width and < 100% height */
if (photo.width / photo.height > el.offsetWidth / el.offsetHeight) {
console.log("A");
width = 100;
height = 100 * photo.height / photo.width * el.offsetWidth / el.offsetHeight;
offsetLeft = 0;
offsetTop = (100 - height) * 0.5;
} else {
console.log("B");
width = 100 * photo.width / photo.height * el.offsetHeight / el.offsetWidth;
height = 100;
offsetLeft = (100 - width) * 0.5;
@ -114,15 +131,20 @@ function makeFaceBoxes() {
const box = document.createElement("div");
box.classList.add("face");
document.body.appendChild(box);
box.style.position = "absolute";
box.style.display = "inline-block";
box.style.border = "1px solid red";
box.style.background = "rgba(255, 0, 0, 0.5)";
box.style.opacity = 0.5;
box.style.left = offsetLeft + Math.floor(face.left * width) + "%";
box.style.top = offsetTop + Math.floor(face.top * height) + "%";
box.style.width = Math.floor((face.right - face.left) * width) + "%";
box.style.height = Math.floor((face.bottom - face.top) * height) + "%";
box.addEventListener("click", (event) => {
console.log(face);
face.relatedPhotos.forEach((path) => {
window.open(base + path);
});
event.preventDefault = true;
event.stopImmediatePropagation();
event.stopPropagation();
return false;
});
});
}
@ -153,12 +175,9 @@ function loadPhoto(index) {
document.getElementById("photo").style.backgroundImage = "url(" + encodeURI(url) + ")";
countdown = 15;
tick();
Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => {
el.parentElement.removeChild(el);
});
window.fetch("api/v1/photos/faces/" + photo.id).then(res => res.json()).then((faces) => {
activeFaces = faces;
makeFaceBoxes();
makeFaceBoxes(photo);
}).catch(function(error) {
console.error(error);
info.textContent += "Unable to obtain face information :(";
@ -227,6 +246,14 @@ document.addEventListener("DOMContentLoaded", function() {
base = "/";
}
var timeout = 0;
window.addEventListener("resize", (event) => {
if (timeout) {
window.clearTimeout(timeout);
}
timeout = window.setTimeout(makeFaceBoxes, 250);
});
document.addEventListener("click", function(event) {
toggleFullscreen();
var now = new Date().getTime();

233
server/face-recognizer.js
View File

@ -66,7 +66,9 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
raw: true
}
).then((results) => {
const remaining = results.length,
const total = results.length;
let remaining = total,
processed = 0,
lastStatus = Date.now();
console.log(`${results.length} photos have not had faces scanned.`);
return Promise.map(results, (photo) => {
@ -93,60 +95,68 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
replacements: photo,
});
}).then(async () => {
const image = await canvas.loadImage(picturesPath + photoPath);
});
}).then(async () => {
const image = await canvas.loadImage(picturesPath + photoPath);
const detections = await faceapi.detectAllFaces(image,
new faceapi.SsdMobilenetv1Options({
minConfidence: 0.8
})
).withFaceLandmarks().withFaceDescriptors();
const detections = await faceapi.detectAllFaces(image,
new faceapi.SsdMobilenetv1Options({
minConfidence: 0.8
})
).withFaceLandmarks().withFaceDescriptors();
if (detections.length > 0) {
console.log(`...${detections.length} faces identified.`);
}
return Promise.map(detections, (face, index) => {
const width = face.detection._box._width,
height = face.detection._box._height,
replacements = {
id: photo.id,
top: face.detection._box._y / photo.height,
left: face.detection._box._x / photo.width,
bottom: (face.detection._box._y + height) / photo.height,
right: (face.detection._box._x + width) / photo.width,
faceConfidence: face.detection._score
};
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
replacements: replacements
}).catch(() => {
console.error(photoPath, index);
console.error(JSON.stringify(face, null, 2));
console.error(JSON.stringify(replacements, null, 2));
process.exit(-1);
}).spread((results, metadata) => {
return metadata.lastID;
}).then((id) => {
console.log(`...DB id ${id}. Writing descriptor data...`);
const path = faceData + "/" + (id % 100);
return mkdir(path).then(() => {
const dataPath = path + "/" + id + "-data.json",
data = [];
for (let i = 0; i < 128; i++) {
data.push(face.descriptor[i]);
}
fs.writeFileSync(dataPath, JSON.stringify(data));
});
});
}).then(() => {
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
replacements: {
id: photo.id,
faces: detections.length
},
if (detections.length > 0) {
console.log(`...${detections.length} faces identified.`);
}
return Promise.map(detections, (face) => {
const detection = face.detection;
const width = detection._box._width,
height = detection._box._height,
replacements = {
id: photo.id,
top: detection._box._y / detection._imageDims.height,
left: detection._box._x / detection._imageDims.width,
bottom: (detection._box._y + height) / detection._imageDims.height,
right: (detection._box._x + width) / detection._imageDims.width,
faceConfidence: detection._score
};
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
replacements: replacements
}).spread((results, metadata) => {
return metadata.lastID;
}).then((id) => {
console.log(`...DB id ${id}. Writing descriptor data...`);
const path = faceData + "/" + (id % 100);
return mkdir(path).then(() => {
const dataPath = path + "/" + id + "-data.json",
data = [];
for (let i = 0; i < 128; i++) {
data.push(face.descriptor[i]);
}
fs.writeFileSync(dataPath, JSON.stringify(data));
});
});
}).then(() => {
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
replacements: {
id: photo.id,
faces: detections.length
},
});
});
}).then(() => {
processed++;
const now = Date.now();
if (now - lastStatus > 5000) {
const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
eta = Math.round((total - processed) / rate);
lastStatus = now;
remaining = total - processed;
console.log(`Processing ${rate} images per second. ${remaining} images to be processed. ETA: ${eta}s`);
}
});
}, {
concurrency: maxConcurrency
@ -180,6 +190,8 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
return facesToUpdate;
}
console.log("...removing old assets.");
return photoDB.sequelize.query(
"SELECT id FROM faces", {
type: photoDB.sequelize.QueryTypes.SELECT,
@ -209,73 +221,90 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
return facesToUpdate;
});
}).then((facesToUpdate) => {
const total = facesToUpdate.length;
let remaining = total,
processed = 0,
lastStatus = Date.now(),
targets = [];
for (let target in descriptors) {
targets.push({ id: target, descriptor: descriptors[target] });
}
return Promise.mapSeries(facesToUpdate, (face) => {
if (!face.id in descriptors) {
if (!(face.id in descriptors)) {
console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
return;
}
const WHERE = "WHERE id" + (face.lastComparedId ? ">:lastId" : "!=:id");
return photoDB.sequelize.query(
"SELECT id " +
"FROM faces " +
WHERE, {
replacements: {
id: face.id,
lastId: face.lastComparedId
},
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true
}).then((facesToCompare) => {
return photoDB.sequelize.transaction((transaction) => {
return Promise.mapSeries(facesToCompare, (target) => {
if (!target.id in descriptors) {
console.warn(`...attempt to compare distance with no descriptor for ${target.id}`)
const faceDescriptor = descriptors[face.id];
return photoDB.sequelize.transaction((transaction) => {
return Promise.map(targets, (target) => {
/* Skip comparing to self */
if (target.id == face.id) {
return;
}
/* Only compare against newer faces */
if (face.lastComparedId && target.id <= face.lastComparedId) {
return;
}
return photoDB.sequelize.query(
"SELECT distance " +
"FROM facedistances " +
"WHERE face1Id=:first AND face2Id=:second", {
replacements: {
first: Math.min(face.id, target.id),
second: Math.max(face.id, target.id)
},
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true,
transaction: transaction
}).then((results) => {
if (results.length) {
return;
}
const distance = faceapi.euclideanDistance(faceDescriptor, target.descriptor);
if (distance < 0.4) {
console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
}
return photoDB.sequelize.query(
"SELECT distance " +
"FROM facedistances " +
"WHERE (face1Id=:first AND face2Id=:second) OR (face1Id=:second AND face2Id=:first)", {
"INSERT INTO facedistances (face1Id, face2Id, distance) " +
"VALUES (:first, :second, :distance)", {
replacements: {
first: face.id,
second: target.id
},
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true
}).then((results) => {
if (results.length) {
return;
}
const distance = faceapi.euclideanDistance(descriptors[face.id], descriptors[target.id]);
if (distance < 0.4) {
console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
}
return photoDB.sequelize.query(
"INSERT INTO facedistances (face1Id, face2Id, distance) " +
"VALUES (:first, :second, :distance)", {
replacements: {
first: face.id,
second: target.id,
distance: distance
},
transaction: transaction
});
});
}).then(() => {
return photoDB.sequelize.query(
"UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
replacements: {
lastId: maxId,
id: face.id
first: Math.min(face.id, target.id),
second: Math.max(face.id, target.id),
distance: distance
},
transaction: transaction
});
});
}, {
concurrency: maxConcurrency
}).then(() => {
return photoDB.sequelize.query(
"UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
replacements: {
lastId: maxId,
id: face.id
},
transaction: transaction
});
});
}).then(() => {
processed++;
const now = Date.now();
if (now - lastStatus > 5000) {
const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
eta = Math.round((total - processed) / rate);
lastStatus = now;
remaining = total - processed;
console.log(`Processing ${rate} faces per second. ${remaining} faces to be processed. ETA: ${eta}s`);
}
});
});
});

64
server/face.js Normal file
View File

@ -0,0 +1,64 @@
"use strict";
process.env.TZ = "Etc/GMT";
require('@tensorflow/tfjs-node');
const config = require("config"),
Promise = require("bluebird"),
{ exists, mkdir, unlink } = require("./lib/util"),
faceapi = require("face-api.js"),
fs = require("fs"),
canvas = require("canvas");
const { Canvas, Image, ImageData } = canvas;
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
const maxConcurrency = require("os").cpus().length;
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
.then(() => {
console.log("ssdMobileNetv1 loaded.");
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
}).then(() => {
console.log("landmark68 loaded.");
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
}).then(async () => {
console.log("faceRecognitionNet loaded.");
let faces = [];
for (let a = 2; a < process.argv.length; a++) {
const file = process.argv[a];
process.stdout.write(`Loading ${file}...`);
const image = await canvas.loadImage(file),
detectors = await faceapi.detectAllFaces(image,
new faceapi.SsdMobilenetv1Options({
minConfidence: 0.8
})
).withFaceLandmarks().withFaceDescriptors();
process.stdout.write(`${detectors.length} faces.\n`);
detectors.forEach((face, index) => {
faces.push({
file: file,
index: index,
descriptor: face.descriptor
})
});
}
for (let i = 0; i < faces.length; i++) {
for (let j = 0; j < faces.length; j++) {
const a = faces[i], b = faces[j];
const distance = faceapi.euclideanDistance(a.descriptor, b.descriptor);
console.log(`${a.file}.${a.index} to ${b.file}.${b.index} = ${distance}`);
}
}
}).then(() => {
console.log("Face detection scanning completed.");
}).catch((error) => {
console.error(error);
process.exit(-1);
});

35
server/routes/photos.js
View File

@ -793,7 +793,40 @@ router.get("/faces/:id", (req, res) => {
type: photoDB.Sequelize.QueryTypes.SELECT,
raw: true
}).then((faces) => {
return res.status(200).json(faces);
return Promise.map(faces, (face) => {
return photoDB.sequelize.query(
"SELECT face1ID,face2ID " +
"FROM facedistances " +
"WHERE distance<0.45 AND (face1ID=:id OR face2ID=:id) " +
"ORDER BY distance ASC", {
replacements: {
id: face.id
},
type: photoDB.Sequelize.QueryTypes.SELECT,
raw: true
}).then((faceIds) => {
return photoDB.sequelize.query(
"SELECT photos.id,albums.path,photos.filename " +
"FROM faces " +
"LEFT JOIN photos ON photos.id=faces.photoId " +
"LEFT JOIN albums ON albums.id=photos.albumId " +
"WHERE faces.id IN (:ids)", {
replacements: {
ids: faceIds.map((face) => {
return (face.face1Id == face.id) ? face.face2Id : face.face1Id;
})
},
type: photoDB.Sequelize.QueryTypes.SELECT,
raw: true
});
}).then((photos) => {
face.relatedPhotos = photos.filter((photo) => { return photo.id != id }).map((photo) => {
return photo.path + photo.filename;
});
});
}).then(() => {
return res.status(200).json(faces);
});
});
});