jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Face scanning making progress. Working on face-box.

Browse Source 
Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>
This commit is contained in:
James Ketrenos 2020-01-04 22:51:07 -08:00
parent 840178b6ae
commit 72efd92b99
4 changed files with 260 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
frontend/slideshow.html
View File

@ -87,6 +87,45 @@ const days = [ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday",
months = [ "January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December" ];
let activeFaces = [];
function makeFaceBoxes() {
const el = document.getElementById("photo");
let width, height, offsetLeft = 0, offsetTop = 0;
/* If photo is wider than viewport, it will be 100% width and < 100% height */
if (photo.width / photo.height > el.offsetWidth / el.offsetHeight) {
console.log("A");
width = 100;
height = 100 * photo.height / photo.width * el.offsetWidth / el.offsetHeight;
offsetLeft = 0;
offsetTop = (100 - height) * 0.5;
} else {
console.log("B");
width = 100 * photo.width / photo.height * el.offsetHeight / el.offsetWidth;
height = 100;
offsetLeft = (100 - width) * 0.5;
offsetTop = 0;
}
activeFaces.forEach((face) => {
const box = document.createElement("div");
box.classList.add("face");
document.body.appendChild(box);
box.style.position = "absolute";
box.style.display = "inline-block";
box.style.border = "1px solid red";
box.style.background = "rgba(255, 0, 0, 0.5)";
box.style.opacity = 0.5;
box.style.left = offsetLeft + Math.floor(face.left * width) + "%";
box.style.top = offsetTop + Math.floor(face.top * height) + "%";
box.style.width = Math.floor((face.right - face.left) * width) + "%";
box.style.height = Math.floor((face.bottom - face.top) * height) + "%";
});
}
function loadPhoto(index) {
const photo = photos[index],
xml = new XMLHttpRequest(),
@ -114,6 +153,16 @@ function loadPhoto(index) {
document.getElementById("photo").style.backgroundImage = "url(" + encodeURI(url) + ")";
countdown = 15;
tick();
Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => {
el.parentElement.removeChild(el);
});
window.fetch("api/v1/photos/faces/" + photo.id).then(res => res.json()).then((faces) => {
activeFaces = faces;
makeFaceBoxes();
}).catch(function(error) {
console.error(error);
info.textContent += "Unable to obtain face information :(";
});
}
xml.onerror = function(event) {

8
server/db/photos.js
View File

@ -151,19 +151,19 @@ function init() {
});
const FaceDistances = db.sequelize.define('facedistance', {
photoId: {
face1Id: {
type: Sequelize.INTEGER,
allowNull: false,
references: {
model: Photo,
model: Face,
key: 'id',
}
},
targetId: {
face2Id: {
type: Sequelize.INTEGER,
allowNull: false,
references: {
model: Photo,
model: Face,
key: 'id',
}
},

266
server/face-recognizer.js
View File

@ -62,106 +62,222 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
"FROM photos " +
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
"WHERE faces=-1 ORDER BY albums.path,photos.filename", {
type: photoDB.sequelize.QueryTypes.SELECT
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true
}
).then((results) => {
const remaining = results.length,
lastStatus = Date.now();
console.log(`${results.length} photos have not had faces scanned.`);
return Promise.map(results, (photo) => {
const filePath = photo.path + photo.filename;
console.log(`Processing ${filePath}...`);
/* Remove any existing face data for this photo */
return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
replacements: photo,
}).then((faces) => {
/* For each face-id, remove any face-data files, and then remove all the entries
const photoPath = photo.path + photo.filename;
console.log(`Processing ${photoPath}...`);
/* Remove any existing face data for this photo */
return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
replacements: photo,
}).then((faces) => {
/* For each face-id, remove any face-data files, and then remove all the entries
* from the DB */
return Promise.map(faces, (face) => {
const id = face.id,
filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
return exists(filePath).then((result) => {
if (result) {
console.log(`...removing ${filePath}`);
return unlink(filePath);
}
return Promise.map(faces, (face) => {
const id = face.id,
dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
return exists(dataPath).then((result) => {
if (result) {
console.log(`...removing ${dataPath}`);
return unlink(dataPath);
}
});
}).then(() => {
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
replacements: photo,
});
}).then(async () => {
const image = await canvas.loadImage(picturesPath + photoPath);
const detections = await faceapi.detectAllFaces(image,
new faceapi.SsdMobilenetv1Options({
minConfidence: 0.8
})
).withFaceLandmarks().withFaceDescriptors();
if (detections.length > 0) {
console.log(`...${detections.length} faces identified.`);
}
return Promise.map(detections, (face, index) => {
const width = face.detection._box._width,
height = face.detection._box._height,
replacements = {
id: photo.id,
top: face.detection._box._y / photo.height,
left: face.detection._box._x / photo.width,
bottom: (face.detection._box._y + height) / photo.height,
right: (face.detection._box._x + width) / photo.width,
faceConfidence: face.detection._score
};
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
replacements: replacements
}).catch(() => {
console.error(photoPath, index);
console.error(JSON.stringify(face, null, 2));
console.error(JSON.stringify(replacements, null, 2));
process.exit(-1);
}).spread((results, metadata) => {
return metadata.lastID;
}).then((id) => {
console.log(`...DB id ${id}. Writing descriptor data...`);
const path = faceData + "/" + (id % 100);
return mkdir(path).then(() => {
const dataPath = path + "/" + id + "-data.json",
data = [];
for (let i = 0; i < 128; i++) {
data.push(face.descriptor[i]);
}
fs.writeFileSync(dataPath, JSON.stringify(data));
});
});
}).then(() => {
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
replacements: photo,
});
}).then(async () => {
console.log("...loading image.");
const image = await canvas.loadImage(picturesPath + filePath);
console.log("...detecting faces.");
const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors();
console.log(`...${detections.length} faces identified.`);
return Promise.map(detections, (face, index) => {
const width = face.detection._box._width,
height = face.detection._box._height,
replacements = {
id: photo.id,
top: face.detection._box._y / photo.height,
left: face.detection._box._x / photo.width,
bottom: (face.detection._box._x + height) / photo.height,
right: (face.detection._box._x + width) / photo.width,
faceConfidence: face.detection._score
};
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
replacements: replacements
}).catch(() => {
console.error(filePath, index);
console.error(JSON.stringify(face, null, 2));
console.error(JSON.stringify(replacements, null, 2));
process.exit(-1);
}).spread((results, metadata) => {
return metadata.lastID;
}).then((id) => {
console.log(`...DB id ${id}. Writing descriptor data...`);
const path = faceData + "/" + (id % 100);
return mkdir(path).then(() => {
const filePath = path + "/" + id + "-data.json",
data = [];
for (let i = 0; i < 128; i++) {
data.push(face.descriptor[i]);
}
fs.writeFileSync(filePath, JSON.stringify(data));
});
});
}).then(() => {
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
replacements: {
id: photo.id,
faces: detections.length
},
});
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
replacements: {
id: photo.id,
faces: detections.length
},
});
});
});
});
}, {
concurrency: maxConcurrency
});
});
}).then(() => {
console.log("Looking for face distances that need to be updated.");
console.log("Looking for face distances that need to be updated...");
const descriptors = {};
return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id DESC LIMIT 1", {
type: photoDB.sequelize.QueryTypes.SELECT
}
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true
}).then((results) => {
if (!results.length) {
console.log("No faces exist yet to generate distances.");
console.log("...no faces exist yet to generate distances.");
return;
}
const maxId = results[0].id;
return photoDB.sequelize.query("SELECT id FROM faces WHERE lastComparedId<:maxId OR lastComparedId IS NULL", {
return photoDB.sequelize.query(
"SELECT id,lastComparedId " +
"FROM faces " +
"WHERE lastComparedId<:maxId OR lastComparedId IS NULL", {
replacements: {
maxId: maxId
},
type: photoDB.sequelize.QueryTypes.SELECT
}).then((results) => {
console.log(`${results.length} faces need to be updated.`);
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true
}).then((facesToUpdate) => {
console.log(`...${facesToUpdate.length} faces need to be updated.`);
if (facesToUpdate.length == 0) {
return facesToUpdate;
}
return photoDB.sequelize.query(
"SELECT id FROM faces", {
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true
}).then((allFaces) => {
console.log(`...reading ${allFaces.length} descriptors...`);
return Promise.map(allFaces, (face) => {
const id = face.id,
dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
if (id in descriptors) {
return;
}
return exists(dataPath).then((doesExist) => {
if (!doesExist) {
console.warn(`${dataPath} is missing!`);
return;
}
descriptors[id] = JSON.parse(fs.readFileSync(dataPath));
});
}, {
concurrency: maxConcurrency
});
}).then(() => {
return facesToUpdate;
});
}).then((facesToUpdate) => {
return Promise.mapSeries(facesToUpdate, (face) => {
if (!face.id in descriptors) {
console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
return;
}
const WHERE = "WHERE id" + (face.lastComparedId ? ">:lastId" : "!=:id");
return photoDB.sequelize.query(
"SELECT id " +
"FROM faces " +
WHERE, {
replacements: {
id: face.id,
lastId: face.lastComparedId
},
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true
}).then((facesToCompare) => {
return photoDB.sequelize.transaction((transaction) => {
return Promise.mapSeries(facesToCompare, (target) => {
if (!target.id in descriptors) {
console.warn(`...attempt to compare distance with no descriptor for ${target.id}`)
return;
}
return photoDB.sequelize.query(
"SELECT distance " +
"FROM facedistances " +
"WHERE (face1Id=:first AND face2Id=:second) OR (face1Id=:second AND face2Id=:first)", {
replacements: {
first: face.id,
second: target.id
},
type: photoDB.sequelize.QueryTypes.SELECT,
raw: true
}).then((results) => {
if (results.length) {
return;
}
const distance = faceapi.euclideanDistance(descriptors[face.id], descriptors[target.id]);
if (distance < 0.4) {
console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
}
return photoDB.sequelize.query(
"INSERT INTO facedistances (face1Id, face2Id, distance) " +
"VALUES (:first, :second, :distance)", {
replacements: {
first: face.id,
second: target.id,
distance: distance
},
transaction: transaction
});
});
}).then(() => {
return photoDB.sequelize.query(
"UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
replacements: {
lastId: maxId,
id: face.id
},
transaction: transaction
});
});
});
});
});
});
});
}).then(() => {

16
server/routes/photos.js
View File

@ -781,6 +781,22 @@ router.get("/trash", function(req, res/*, next*/) {
});
});
router.get("/faces/:id", (req, res) => {
const id = parseInt(req.params.id);
if (id != req.params.id) {
return res.status(400).send({ message: "Usage faces/:id"});
}
return photoDB.sequelize.query("SELECT * FROM faces WHERE photoId=:id", {
replacements: {
id: id
},
type: photoDB.Sequelize.QueryTypes.SELECT,
raw: true
}).then((faces) => {
return res.status(200).json(faces);
});
});
router.get("/mvimg/*", function(req, res/*, next*/) {
let limit = parseInt(req.query.limit) || 50,
id, cursor, index;