Added face-api usage
Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>
This commit is contained in:
James Ketrenos
parent
44b157ee8b
commit
046fd0fd1e
@ -17,14 +17,18 @@
|
||||
"face-recognition": "^0.9.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tensorflow/tfjs-core": "^1.5.1",
|
||||
"@tensorflow/tfjs-node": "^1.5.1",
|
||||
"bluebird": "^3.7.2",
|
||||
"body-parser": "^1.19.0",
|
||||
"canvas": "^2.6.1",
|
||||
"config": "^1.31.0",
|
||||
"connect-sqlite3": "^0.9.11",
|
||||
"cookie-parser": "^1.4.4",
|
||||
"exif-reader": "github:paras20xx/exif-reader",
|
||||
"express": "^4.17.1",
|
||||
"express-session": "^1.17.0",
|
||||
"face-api.js": "^0.22.0",
|
||||
"handlebars": "^4.5.3",
|
||||
"ldapauth-fork": "^4.2.0",
|
||||
"ldapjs": "^1.0.2",
|
||||
@ -33,6 +37,7 @@
|
||||
"moment-holiday": "^1.5.1",
|
||||
"morgan": "^1.9.1",
|
||||
"mustache": "^3.2.1",
|
||||
"node-fetch": "^2.6.0",
|
||||
"nodemailer": "^4.7.0",
|
||||
"qs": "^6.9.1",
|
||||
"sequelize": "^4.44.3",
|
||||
|
||||
@ -19,109 +19,153 @@ process.env.TZ = "Etc/GMT";
|
||||
|
||||
console.log("Loading face-recognizer");
|
||||
|
||||
require('@tensorflow/tfjs-node');
|
||||
|
||||
const config = require("config"),
|
||||
Promise = require("bluebird"),
|
||||
{ mkdir, unlink } = require("./lib/util"),
|
||||
fr = require("face-recognition");
|
||||
{ exists, mkdir, unlink } = require("./lib/util"),
|
||||
faceapi = require("face-api.js"),
|
||||
fs = require("fs"),
|
||||
canvas = require("canvas");
|
||||
|
||||
const { Canvas, Image, ImageData } = canvas;
|
||||
|
||||
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
||||
|
||||
const maxConcurrency = require("os").cpus().length;
|
||||
|
||||
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
|
||||
|
||||
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
|
||||
faceData = picturesPath + "face-data/";
|
||||
|
||||
let photoDB = null, faceDataExists = false;
|
||||
let photoDB = null;
|
||||
|
||||
console.log("Loading pictures out of: " + picturesPath);
|
||||
|
||||
require("./db/photos").then(function(db) {
|
||||
photoDB = db;
|
||||
faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
|
||||
.then(() => {
|
||||
console.log("ssdMobileNetv1 loaded.");
|
||||
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
|
||||
}).then(() => {
|
||||
console.log("DB connected.");
|
||||
console.log("landmark68 loaded.");
|
||||
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
|
||||
}).then(() => {
|
||||
console.log("Beginning face detection scanning.");
|
||||
return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
|
||||
"FROM photos " +
|
||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||
"WHERE faces=-1 ORDER BY albums.path,photos.filename", {
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}
|
||||
).then((results) => {
|
||||
console.log(`${results.length} photos have not had faces scanned.`);
|
||||
return Promise.map(results, (photo) => {
|
||||
const filePath = photo.path + photo.filename;
|
||||
console.log(`Processing ${filePath}...`);
|
||||
|
||||
return photoDB.sequelize.transaction(function(transaction) {
|
||||
/* Remove any existing face data for this photo */
|
||||
return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
|
||||
transaction: transaction,
|
||||
replacements: photo,
|
||||
}).then((faces) => {
|
||||
/* For each face-id, remove any face-data files, and then remove all the entries
|
||||
* from the DB */
|
||||
return Promise.map(faces, (id) => {
|
||||
return Promise.mapSeries(["-normalized.png", "-data.json" ], (fileSuffix) => {
|
||||
const filePath = faceData + "/" + (id % 100) + "/" + id + fileSuffix;
|
||||
console.log("faceRecognitionNet loaded.");
|
||||
return require("./db/photos").then(function(db) {
|
||||
photoDB = db;
|
||||
}).then(() => {
|
||||
console.log("DB connected.");
|
||||
}).then(() => {
|
||||
console.log("Beginning face detection scanning.");
|
||||
return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
|
||||
"FROM photos " +
|
||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||
"WHERE faces=-1 ORDER BY albums.path,photos.filename", {
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}
|
||||
).then((results) => {
|
||||
console.log(`${results.length} photos have not had faces scanned.`);
|
||||
return Promise.map(results, (photo) => {
|
||||
const filePath = photo.path + photo.filename;
|
||||
console.log(`Processing ${filePath}...`);
|
||||
/* Remove any existing face data for this photo */
|
||||
return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
|
||||
replacements: photo,
|
||||
}).then((faces) => {
|
||||
/* For each face-id, remove any face-data files, and then remove all the entries
|
||||
* from the DB */
|
||||
return Promise.map(faces, (face) => {
|
||||
const id = face.id,
|
||||
filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
|
||||
return exists(filePath).then((result) => {
|
||||
console.log(`...removing ${filePath}`);
|
||||
return unlink(filePath);
|
||||
});
|
||||
});
|
||||
}).then(() => {
|
||||
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
|
||||
transaction: transaction,
|
||||
replacements: photo,
|
||||
});
|
||||
}).then(() => {
|
||||
const image = fr.loadImage(filePath),
|
||||
detector = fr.FaceDetector();
|
||||
|
||||
console.log("...detecting faces.");
|
||||
const faceRectangles = detector.locateFaces(image)
|
||||
if (faceRectangles.length == 0) {
|
||||
console.log("...no faces found in image.");
|
||||
return;
|
||||
}
|
||||
|
||||
/* Create a face entry in photos for each face found. */
|
||||
const faceImages = detector.detectFaces(image, 200)
|
||||
|
||||
console.log(`...saving ${faceImages.length} faces.`);
|
||||
|
||||
return Promise.map(faceRectangles, (face, index) => {
|
||||
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
|
||||
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
|
||||
replacements: {
|
||||
id: photo.id,
|
||||
top: face.top / photo.height,
|
||||
left: face.left / photo.width,
|
||||
bottom: face.top / photo.height,
|
||||
right: face.right / photo.width,
|
||||
faceConfidence: face.confidence
|
||||
},
|
||||
transaction: transaction
|
||||
}).spread((results, metadata) => {
|
||||
return metadata.lastID;
|
||||
}).then((id) => {
|
||||
console.log(`...DB id ${id}. Writing data and images...`);
|
||||
const filePathPrefix = faceData + "/" + (id % 100) + "/" + id;
|
||||
/* https://medium.com/@ageitgey/machine-learning-is-fun-part-4-modern-face-recognition-with-deep-learning-c3cffc121d78 */
|
||||
const data = [];
|
||||
for (let i = 0; i < 128; i++) {
|
||||
data.push(Math.random() - 0.5);
|
||||
if (result) {
|
||||
console.log(`...removing ${filePath}`);
|
||||
return unlink(filePath);
|
||||
}
|
||||
fs.writeFileSync(filePathPrefix + "-data.json", JSON.stringify(data));
|
||||
fr.saveImage(filePathPrefix + "-normalized.png", faceImages[index]);
|
||||
});
|
||||
}).then(() => {
|
||||
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
|
||||
replacements: photo,
|
||||
});
|
||||
}).then(async () => {
|
||||
console.log("...loading image.");
|
||||
const image = await canvas.loadImage(picturesPath + filePath);
|
||||
|
||||
console.log("...detecting faces.");
|
||||
const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors();
|
||||
|
||||
console.log(`...${detections.length} faces identified.`);
|
||||
return Promise.map(detections, (face, index) => {
|
||||
const width = face.detection._box._width,
|
||||
height = face.detection._box._height,
|
||||
replacements = {
|
||||
id: photo.id,
|
||||
top: face.detection._box._y / photo.height,
|
||||
left: face.detection._box._x / photo.width,
|
||||
bottom: (face.detection._box._x + height) / photo.height,
|
||||
right: (face.detection._box._x + width) / photo.width,
|
||||
faceConfidence: face.detection._score
|
||||
};
|
||||
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
|
||||
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
|
||||
replacements: replacements
|
||||
}).catch(() => {
|
||||
console.error(filePath, index);
|
||||
console.error(JSON.stringify(face, null, 2));
|
||||
console.error(JSON.stringify(replacements, null, 2));
|
||||
process.exit(-1);
|
||||
}).spread((results, metadata) => {
|
||||
return metadata.lastID;
|
||||
}).then((id) => {
|
||||
console.log(`...DB id ${id}. Writing descriptor data...`);
|
||||
const path = faceData + "/" + (id % 100);
|
||||
return mkdir(path).then(() => {
|
||||
const filePath = path + "/" + id + "-data.json",
|
||||
data = [];
|
||||
for (let i = 0; i < 128; i++) {
|
||||
data.push(face.descriptor[i]);
|
||||
}
|
||||
fs.writeFileSync(filePath, JSON.stringify(data));
|
||||
});
|
||||
});
|
||||
}).then(() => {
|
||||
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
|
||||
replacements: {
|
||||
id: photo.id,
|
||||
faces: detections.length
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}, {
|
||||
concurrency: maxConcurrency
|
||||
});
|
||||
});
|
||||
}).then(() => {
|
||||
console.log("Face detection scanning completed.");
|
||||
}).catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(-1);
|
||||
});
|
||||
}).then(() => {
|
||||
console.log("Face detection scanning completed.");
|
||||
}).catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(-1);
|
||||
});
|
||||
|
||||
/* TODO:
|
||||
1. For each path / person, look up highest face confidence and tag
|
||||
2. Use highest face and identity confidence for input into
|
||||
https://github.com/justadudewhohacks/face-api.js#face-recognition-by-matching-descriptors
|
||||
|
||||
const labeledDescriptors = [
|
||||
new faceapi.LabeledFaceDescriptors(
|
||||
'obama',
|
||||
[descriptorObama1, descriptorObama2]
|
||||
),
|
||||
new faceapi.LabeledFaceDescriptors(
|
||||
'trump',
|
||||
[descriptorTrump]
|
||||
)
|
||||
]
|
||||
|
||||
const faceMatcher = new faceapi.FaceMatcher(labeledDescriptors)
|
||||
*/
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user