309 lines
11 KiB
JavaScript
309 lines
11 KiB
JavaScript
/*
|
|
* Face recognition:
|
|
* 1. For each photo, extract all faces. Store face rectangles.
|
|
* face_id unique
|
|
* photo_id foreign key
|
|
* top left bottom right
|
|
* identity_id
|
|
* distance (0 == truth; manually assigned identity)
|
|
* 2. For each face_id, create:
|
|
* /${picturesPath}face-data/${face_id % 100}/
|
|
* ${face_id}-normalized
|
|
* ${face_id}-original
|
|
* ${face_id}-data
|
|
*/
|
|
|
|
"use strict";
|
|
|
|
process.env.TZ = "Etc/GMT";
|
|
|
|
console.log("Loading face-recognizer");
|
|
|
|
require('@tensorflow/tfjs-node');
|
|
|
|
const config = require("config"),
|
|
Promise = require("bluebird"),
|
|
{ exists, mkdir, unlink } = require("./lib/util"),
|
|
faceapi = require("face-api.js"),
|
|
fs = require("fs"),
|
|
canvas = require("canvas");
|
|
|
|
const { Canvas, Image, ImageData } = canvas;
|
|
|
|
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
|
|
|
const maxConcurrency = require("os").cpus().length;
|
|
|
|
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
|
|
|
|
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
|
|
faceData = picturesPath + "face-data/";
|
|
|
|
let photoDB = null;
|
|
|
|
console.log("Loading pictures out of: " + picturesPath);
|
|
|
|
faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
|
|
.then(() => {
|
|
console.log("ssdMobileNetv1 loaded.");
|
|
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
|
|
}).then(() => {
|
|
console.log("landmark68 loaded.");
|
|
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
|
|
}).then(() => {
|
|
console.log("faceRecognitionNet loaded.");
|
|
return require("./db/photos").then(function(db) {
|
|
photoDB = db;
|
|
}).then(() => {
|
|
console.log("DB connected.");
|
|
}).then(() => {
|
|
console.log("Beginning face detection scanning.");
|
|
return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
|
|
"FROM photos " +
|
|
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
|
"WHERE faces=-1 ORDER BY albums.path,photos.filename", {
|
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
}
|
|
).then((results) => {
|
|
const remaining = results.length,
|
|
lastStatus = Date.now();
|
|
console.log(`${results.length} photos have not had faces scanned.`);
|
|
return Promise.map(results, (photo) => {
|
|
const photoPath = photo.path + photo.filename;
|
|
|
|
console.log(`Processing ${photoPath}...`);
|
|
|
|
/* Remove any existing face data for this photo */
|
|
return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
|
|
replacements: photo,
|
|
}).then((faces) => {
|
|
/* For each face-id, remove any face-data files, and then remove all the entries
|
|
* from the DB */
|
|
return Promise.map(faces, (face) => {
|
|
const id = face.id,
|
|
dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
|
|
return exists(dataPath).then((result) => {
|
|
if (result) {
|
|
console.log(`...removing ${dataPath}`);
|
|
return unlink(dataPath);
|
|
}
|
|
});
|
|
}).then(() => {
|
|
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
|
|
replacements: photo,
|
|
});
|
|
}).then(async () => {
|
|
const image = await canvas.loadImage(picturesPath + photoPath);
|
|
|
|
const detections = await faceapi.detectAllFaces(image,
|
|
new faceapi.SsdMobilenetv1Options({
|
|
minConfidence: 0.8
|
|
})
|
|
).withFaceLandmarks().withFaceDescriptors();
|
|
|
|
if (detections.length > 0) {
|
|
console.log(`...${detections.length} faces identified.`);
|
|
}
|
|
return Promise.map(detections, (face, index) => {
|
|
const width = face.detection._box._width,
|
|
height = face.detection._box._height,
|
|
replacements = {
|
|
id: photo.id,
|
|
top: face.detection._box._y / photo.height,
|
|
left: face.detection._box._x / photo.width,
|
|
bottom: (face.detection._box._y + height) / photo.height,
|
|
right: (face.detection._box._x + width) / photo.width,
|
|
faceConfidence: face.detection._score
|
|
};
|
|
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
|
|
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
|
|
replacements: replacements
|
|
}).catch(() => {
|
|
console.error(photoPath, index);
|
|
console.error(JSON.stringify(face, null, 2));
|
|
console.error(JSON.stringify(replacements, null, 2));
|
|
process.exit(-1);
|
|
}).spread((results, metadata) => {
|
|
return metadata.lastID;
|
|
}).then((id) => {
|
|
console.log(`...DB id ${id}. Writing descriptor data...`);
|
|
const path = faceData + "/" + (id % 100);
|
|
return mkdir(path).then(() => {
|
|
const dataPath = path + "/" + id + "-data.json",
|
|
data = [];
|
|
for (let i = 0; i < 128; i++) {
|
|
data.push(face.descriptor[i]);
|
|
}
|
|
fs.writeFileSync(dataPath, JSON.stringify(data));
|
|
});
|
|
});
|
|
}).then(() => {
|
|
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
|
|
replacements: {
|
|
id: photo.id,
|
|
faces: detections.length
|
|
},
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}, {
|
|
concurrency: maxConcurrency
|
|
});
|
|
});
|
|
}).then(() => {
|
|
console.log("Looking for face distances that need to be updated...");
|
|
const descriptors = {};
|
|
|
|
return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id DESC LIMIT 1", {
|
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
}).then((results) => {
|
|
if (!results.length) {
|
|
console.log("...no faces exist yet to generate distances.");
|
|
return;
|
|
}
|
|
const maxId = results[0].id;
|
|
return photoDB.sequelize.query(
|
|
"SELECT id,lastComparedId " +
|
|
"FROM faces " +
|
|
"WHERE lastComparedId<:maxId OR lastComparedId IS NULL", {
|
|
replacements: {
|
|
maxId: maxId
|
|
},
|
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
}).then((facesToUpdate) => {
|
|
console.log(`...${facesToUpdate.length} faces need to be updated.`);
|
|
if (facesToUpdate.length == 0) {
|
|
return facesToUpdate;
|
|
}
|
|
|
|
return photoDB.sequelize.query(
|
|
"SELECT id FROM faces", {
|
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
}).then((allFaces) => {
|
|
console.log(`...reading ${allFaces.length} descriptors...`);
|
|
return Promise.map(allFaces, (face) => {
|
|
const id = face.id,
|
|
dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
|
|
|
|
if (id in descriptors) {
|
|
return;
|
|
}
|
|
|
|
return exists(dataPath).then((doesExist) => {
|
|
if (!doesExist) {
|
|
console.warn(`${dataPath} is missing!`);
|
|
return;
|
|
}
|
|
|
|
descriptors[id] = JSON.parse(fs.readFileSync(dataPath));
|
|
});
|
|
}, {
|
|
concurrency: maxConcurrency
|
|
});
|
|
}).then(() => {
|
|
return facesToUpdate;
|
|
});
|
|
}).then((facesToUpdate) => {
|
|
return Promise.mapSeries(facesToUpdate, (face) => {
|
|
if (!face.id in descriptors) {
|
|
console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
|
|
return;
|
|
}
|
|
|
|
const WHERE = "WHERE id" + (face.lastComparedId ? ">:lastId" : "!=:id");
|
|
return photoDB.sequelize.query(
|
|
"SELECT id " +
|
|
"FROM faces " +
|
|
WHERE, {
|
|
replacements: {
|
|
id: face.id,
|
|
lastId: face.lastComparedId
|
|
},
|
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
}).then((facesToCompare) => {
|
|
return photoDB.sequelize.transaction((transaction) => {
|
|
return Promise.mapSeries(facesToCompare, (target) => {
|
|
if (!target.id in descriptors) {
|
|
console.warn(`...attempt to compare distance with no descriptor for ${target.id}`)
|
|
return;
|
|
}
|
|
|
|
return photoDB.sequelize.query(
|
|
"SELECT distance " +
|
|
"FROM facedistances " +
|
|
"WHERE (face1Id=:first AND face2Id=:second) OR (face1Id=:second AND face2Id=:first)", {
|
|
replacements: {
|
|
first: face.id,
|
|
second: target.id
|
|
},
|
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
}).then((results) => {
|
|
if (results.length) {
|
|
return;
|
|
}
|
|
const distance = faceapi.euclideanDistance(descriptors[face.id], descriptors[target.id]);
|
|
|
|
if (distance < 0.4) {
|
|
console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
|
|
}
|
|
|
|
return photoDB.sequelize.query(
|
|
"INSERT INTO facedistances (face1Id, face2Id, distance) " +
|
|
"VALUES (:first, :second, :distance)", {
|
|
replacements: {
|
|
first: face.id,
|
|
second: target.id,
|
|
distance: distance
|
|
},
|
|
transaction: transaction
|
|
});
|
|
});
|
|
}).then(() => {
|
|
return photoDB.sequelize.query(
|
|
"UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
|
|
replacements: {
|
|
lastId: maxId,
|
|
id: face.id
|
|
},
|
|
transaction: transaction
|
|
});
|
|
});
|
|
});
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}).then(() => {
|
|
console.log("Face detection scanning completed.");
|
|
}).catch((error) => {
|
|
console.error(error);
|
|
process.exit(-1);
|
|
});
|
|
});
|
|
|
|
/* TODO:
|
|
1. For each path / person, look up highest face confidence and tag
|
|
2. Use highest face and identity confidence for input into
|
|
https://github.com/justadudewhohacks/face-api.js#face-recognition-by-matching-descriptors
|
|
|
|
const labeledDescriptors = [
|
|
new faceapi.LabeledFaceDescriptors(
|
|
'obama',
|
|
[descriptorObama1, descriptorObama2]
|
|
),
|
|
new faceapi.LabeledFaceDescriptors(
|
|
'trump',
|
|
[descriptorTrump]
|
|
)
|
|
]
|
|
|
|
const faceMatcher = new faceapi.FaceMatcher(labeledDescriptors)
|
|
*/
|