Add face-crop creation
Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>
This commit is contained in:
James Ketrenos
parent
ca956cc8af
commit
553a80fce1
@ -28,7 +28,7 @@ const config = require("config"),
|
|||||||
fs = require("fs"),
|
fs = require("fs"),
|
||||||
canvas = require("canvas");
|
canvas = require("canvas");
|
||||||
|
|
||||||
const { Canvas, Image, ImageData } = canvas;
|
const { createCanvas, Canvas, Image, ImageData } = canvas;
|
||||||
|
|
||||||
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
||||||
|
|
||||||
@ -43,211 +43,245 @@ let photoDB = null;
|
|||||||
|
|
||||||
console.log("Loading pictures out of: " + picturesPath);
|
console.log("Loading pictures out of: " + picturesPath);
|
||||||
|
|
||||||
faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
|
process.stdout.write("Loading DB.");
|
||||||
.then(() => {
|
require("./db/photos").then(function(db) {
|
||||||
console.log("ssdMobileNetv1 loaded.");
|
process.stdout.write("done\n");
|
||||||
|
photoDB = db;
|
||||||
|
}).then(() => {
|
||||||
|
console.log("DB connected.");
|
||||||
|
process.stdout.write("Loading models.");
|
||||||
|
return faceapi.nets.ssdMobilenetv1.loadFromDisk('./models');
|
||||||
|
}).then(() => {
|
||||||
|
process.stdout.write(".");
|
||||||
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
|
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
|
||||||
}).then(() => {
|
}).then(() => {
|
||||||
console.log("landmark68 loaded.");
|
process.stdout.write(".");
|
||||||
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
|
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
|
||||||
}).then(() => {
|
}).then(() => {
|
||||||
console.log("faceRecognitionNet loaded.");
|
console.log("Beginning face detection scanning.");
|
||||||
return require("./db/photos").then(function(db) {
|
return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
|
||||||
photoDB = db;
|
"FROM photos " +
|
||||||
}).then(() => {
|
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||||
console.log("DB connected.");
|
"WHERE faces=-1 AND photos.duplicate=0 AND photos.deleted=0 ORDER BY albums.path,photos.filename", {
|
||||||
}).then(() => {
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
||||||
console.log("Beginning face detection scanning.");
|
raw: true
|
||||||
return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
|
});
|
||||||
"FROM photos " +
|
}).then((needToScan) => {
|
||||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
const total = needToScan.length;
|
||||||
"WHERE faces=-1 AND photos.duplicate=0 AND photos.deleted=0 ORDER BY albums.path,photos.filename", {
|
let remaining = total,
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
processed = 0,
|
||||||
raw: true
|
lastStatus = Date.now();
|
||||||
}
|
|
||||||
).then((results) => {
|
|
||||||
const total = results.length;
|
|
||||||
let remaining = total,
|
|
||||||
processed = 0,
|
|
||||||
lastStatus = Date.now();
|
|
||||||
console.log(`${results.length} photos have not had faces scanned.`);
|
|
||||||
return Promise.map(results, (photo) => {
|
|
||||||
const photoPath = photo.path + photo.filename;
|
|
||||||
|
|
||||||
console.log(`Processing ${photoPath}...`);
|
console.log(`${needToScan.length} photos have not had faces scanned.`);
|
||||||
|
|
||||||
/* Remove any existing face data for this photo */
|
return Promise.map(needToScan, (photo) => {
|
||||||
return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
|
const photoPath = photo.path + photo.filename;
|
||||||
replacements: photo,
|
|
||||||
}).then((faces) => {
|
console.log(`Processing ${photoPath}...`);
|
||||||
/* For each face-id, remove any face-data files, and then remove all the entries
|
|
||||||
* from the DB */
|
/* Remove any existing face data for this photo */
|
||||||
return Promise.map(faces, (face) => {
|
return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
|
||||||
const id = face.id,
|
replacements: photo,
|
||||||
dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
|
}).then((faces) => {
|
||||||
return exists(dataPath).then((result) => {
|
/* For each face-id, remove any face-data files, and then remove all the entries
|
||||||
if (result) {
|
* from the DB */
|
||||||
console.log(`...removing ${dataPath}`);
|
return Promise.map(faces, (face) => {
|
||||||
return unlink(dataPath);
|
return Promise.map([ "-data.json", "-original.png" ], (suffix) => {
|
||||||
}
|
const id = face.id,
|
||||||
});
|
dataPath = faceData + "/" + (id % 100) + "/" + id + suffix;
|
||||||
}).then(() => {
|
return exists(dataPath).then((result) => {
|
||||||
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
|
if (result) {
|
||||||
replacements: photo,
|
console.log(`...removing ${dataPath}`);
|
||||||
});
|
return unlink(dataPath);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}).then(async () => {
|
|
||||||
const image = await canvas.loadImage(picturesPath + photoPath);
|
|
||||||
|
|
||||||
const detections = await faceapi.detectAllFaces(image,
|
|
||||||
new faceapi.SsdMobilenetv1Options({
|
|
||||||
minConfidence: 0.8
|
|
||||||
})
|
|
||||||
).withFaceLandmarks().withFaceDescriptors();
|
|
||||||
|
|
||||||
if (detections.length > 0) {
|
|
||||||
console.log(`...${detections.length} faces identified.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Promise.map(detections, (face) => {
|
|
||||||
const detection = face.detection;
|
|
||||||
const width = detection._box._width,
|
|
||||||
height = detection._box._height,
|
|
||||||
replacements = {
|
|
||||||
id: photo.id,
|
|
||||||
top: detection._box._y / detection._imageDims.height,
|
|
||||||
left: detection._box._x / detection._imageDims.width,
|
|
||||||
bottom: (detection._box._y + height) / detection._imageDims.height,
|
|
||||||
right: (detection._box._x + width) / detection._imageDims.width,
|
|
||||||
faceConfidence: detection._score
|
|
||||||
};
|
|
||||||
|
|
||||||
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
|
|
||||||
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
|
|
||||||
replacements: replacements
|
|
||||||
}).spread((results, metadata) => {
|
|
||||||
return metadata.lastID;
|
|
||||||
}).then((id) => {
|
|
||||||
console.log(`...DB id ${id}. Writing descriptor data...`);
|
|
||||||
const path = faceData + "/" + (id % 100);
|
|
||||||
return mkdir(path).then(() => {
|
|
||||||
const dataPath = path + "/" + id + "-data.json",
|
|
||||||
data = [];
|
|
||||||
for (let i = 0; i < 128; i++) {
|
|
||||||
data.push(face.descriptor[i]);
|
|
||||||
}
|
|
||||||
fs.writeFileSync(dataPath, JSON.stringify(data));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
|
|
||||||
replacements: {
|
|
||||||
id: photo.id,
|
|
||||||
faces: detections.length
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).catch((error) => {
|
|
||||||
console.warn("Skipping out on image " + photoPath + " and marking to 0 faces to prevent future scanning.");
|
|
||||||
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
|
|
||||||
replacements: {
|
|
||||||
id: photo.id,
|
|
||||||
faces: 0
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
processed++;
|
|
||||||
const now = Date.now();
|
|
||||||
if (now - lastStatus > 5000) {
|
|
||||||
const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
|
|
||||||
eta = Math.round((total - processed) / rate);
|
|
||||||
lastStatus = now;
|
|
||||||
remaining = total - processed;
|
|
||||||
console.log(`Processing ${rate} images per second. ${remaining} images to be processed. ETA: ${eta}s`);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}, {
|
}).then(() => {
|
||||||
concurrency: maxConcurrency
|
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
|
||||||
|
replacements: photo,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
});
|
}).then(async () => {
|
||||||
}).then(() => {
|
/* Process image for faces data */
|
||||||
console.log("Looking for face distances that need to be updated...");
|
const image = await canvas.loadImage(picturesPath + photoPath);
|
||||||
const descriptors = {};
|
const detections = await faceapi.detectAllFaces(image,
|
||||||
|
new faceapi.SsdMobilenetv1Options({
|
||||||
|
minConfidence: 0.8
|
||||||
|
})
|
||||||
|
).withFaceLandmarks().withFaceDescriptors();
|
||||||
|
|
||||||
return photoDB.sequelize.query("SELECT faces.id FROM faces ORDER BY faces.id DESC LIMIT 1", {
|
if (detections.length > 0) {
|
||||||
|
console.log(`...${detections.length} faces identified in ${photoPath}.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.map(detections, (face) => {
|
||||||
|
const detection = face.detection;
|
||||||
|
const width = detection._box._width,
|
||||||
|
height = detection._box._height,
|
||||||
|
replacements = {
|
||||||
|
id: photo.id,
|
||||||
|
top: detection._box._y / detection._imageDims.height,
|
||||||
|
left: detection._box._x / detection._imageDims.width,
|
||||||
|
bottom: (detection._box._y + height) / detection._imageDims.height,
|
||||||
|
right: (detection._box._x + width) / detection._imageDims.width,
|
||||||
|
faceConfidence: detection._score
|
||||||
|
};
|
||||||
|
|
||||||
|
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
|
||||||
|
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
|
||||||
|
replacements: replacements
|
||||||
|
}).spread((results, metadata) => {
|
||||||
|
return metadata.lastID;
|
||||||
|
}).then((id) => {
|
||||||
|
const path = faceData + (id % 100);
|
||||||
|
return mkdir(path).then(() => {
|
||||||
|
const dataPath = `${path}/${id}-data.json`, data = [];
|
||||||
|
console.log(`...writing descriptor data to ${dataPath}...`);
|
||||||
|
for (let i = 0; i < 128; i++) {
|
||||||
|
data.push(face.descriptor[i]);
|
||||||
|
}
|
||||||
|
fs.writeFileSync(dataPath, JSON.stringify(data));
|
||||||
|
}).then(() => {
|
||||||
|
const canvas = createCanvas(200, 200),
|
||||||
|
target = `${path}/${id}-original.png`,
|
||||||
|
ctx = canvas.getContext('2d'),
|
||||||
|
box = face.detection._box,
|
||||||
|
aspect = box._width / box._height,
|
||||||
|
dx = (aspect > 1.0) ? 200 : (200 * aspect),
|
||||||
|
dy = (aspect < 1.0) ? 200 : (200 / aspect);
|
||||||
|
ctx.fillStyle = "rgba(0, 0, 0, 0)";
|
||||||
|
ctx.fillRect(0, 0, 200, 200);
|
||||||
|
ctx.drawImage(image, box._x, box._y, box._width, box._height,
|
||||||
|
Math.floor((200 - dx) * 0.5),
|
||||||
|
Math.floor((200 - dy) * 0.5), dx, dy);
|
||||||
|
console.log(`...writing face crop to ${target}.`);
|
||||||
|
fs.writeFileSync(target, canvas.toBuffer("image/png", {
|
||||||
|
quality: 0.95,
|
||||||
|
chromaSubsampling: false
|
||||||
|
}));
|
||||||
|
}).catch((error) => {
|
||||||
|
console.error(error);
|
||||||
|
process.exit(-1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}).then(() => {
|
||||||
|
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
|
||||||
|
replacements: {
|
||||||
|
id: photo.id,
|
||||||
|
faces: detections.length
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}).catch((error) => {
|
||||||
|
console.warn("Skipping out on image " + photoPath + " and marking to 0 faces to prevent future scanning.");
|
||||||
|
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
|
||||||
|
replacements: {
|
||||||
|
id: photo.id,
|
||||||
|
faces: 0
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}).then(() => {
|
||||||
|
processed++;
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - lastStatus > 5000) {
|
||||||
|
const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
|
||||||
|
eta = Math.round((total - processed) / rate);
|
||||||
|
lastStatus = now;
|
||||||
|
remaining = total - processed;
|
||||||
|
console.log(`Processing ${rate} images per second. ${remaining} images to be processed. ETA: ${eta}s`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, {
|
||||||
|
concurrency: maxConcurrency
|
||||||
|
});
|
||||||
|
}).then(() => {
|
||||||
|
console.log("Looking for face distances that need to be updated...");
|
||||||
|
let maxId;
|
||||||
|
|
||||||
|
return photoDB.sequelize.query("SELECT faces.id FROM faces ORDER BY faces.id DESC LIMIT 1", {
|
||||||
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
||||||
|
raw: true
|
||||||
|
}).then((results) => {
|
||||||
|
if (!results.length) {
|
||||||
|
console.log("...no faces exist yet to generate distances.");
|
||||||
|
maxId = 0;
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
maxId = results[0].id;
|
||||||
|
return photoDB.sequelize.query(
|
||||||
|
"SELECT faces.id,faces.lastComparedId " +
|
||||||
|
"FROM faces INNER JOIN photos ON photos.duplicate=0 AND photos.deleted=0 AND photos.id=faces.photoId " +
|
||||||
|
"WHERE faces.lastComparedId<:maxId OR faces.lastComparedId IS NULL " +
|
||||||
|
"ORDER BY faces.id ASC", {
|
||||||
|
replacements: {
|
||||||
|
maxId: maxId
|
||||||
|
},
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
||||||
raw: true
|
raw: true
|
||||||
}).then((results) => {
|
});
|
||||||
if (!results.length) {
|
}).then((facesToUpdate) => {
|
||||||
console.log("...no faces exist yet to generate distances.");
|
console.log(`...${facesToUpdate.length} faces need distances updated.`);
|
||||||
return;
|
if (facesToUpdate.length == 0) {
|
||||||
}
|
return facesToUpdate;
|
||||||
const maxId = results[0].id;
|
}
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"SELECT faces.id,faces.lastComparedId " +
|
const descriptors = {};
|
||||||
"FROM faces INNER JOIN photos ON photos.duplicate=0 AND photos.deleted=0 AND photos.id=faces.photoId " +
|
|
||||||
"WHERE faces.lastComparedId<:maxId OR faces.lastComparedId IS NULL", {
|
return photoDB.sequelize.query(
|
||||||
replacements: {
|
"SELECT id FROM faces ORDER BY id ASC", {
|
||||||
maxId: maxId
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
||||||
},
|
raw: true
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
}).then((allFaces) => {
|
||||||
raw: true
|
console.log(`...reading ${allFaces.length} descriptors...`);
|
||||||
}).then((facesToUpdate) => {
|
return Promise.map(allFaces, (face) => {
|
||||||
console.log(`...${facesToUpdate.length} faces need to be updated.`);
|
const id = face.id,
|
||||||
if (facesToUpdate.length == 0) {
|
dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
|
||||||
return facesToUpdate;
|
|
||||||
|
if (id in descriptors) {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log("...removing old assets.");
|
return exists(dataPath).then((doesExist) => {
|
||||||
|
if (!doesExist) {
|
||||||
return photoDB.sequelize.query(
|
console.warn(`${dataPath} is missing!`);
|
||||||
"SELECT id FROM faces", {
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true
|
|
||||||
}).then((allFaces) => {
|
|
||||||
console.log(`...reading ${allFaces.length} descriptors...`);
|
|
||||||
return Promise.map(allFaces, (face) => {
|
|
||||||
const id = face.id,
|
|
||||||
dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
|
|
||||||
|
|
||||||
if (id in descriptors) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
return exists(dataPath).then((doesExist) => {
|
|
||||||
if (!doesExist) {
|
|
||||||
console.warn(`${dataPath} is missing!`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
descriptors[id] = JSON.parse(fs.readFileSync(dataPath));
|
|
||||||
});
|
|
||||||
}, {
|
|
||||||
concurrency: maxConcurrency
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
return facesToUpdate;
|
|
||||||
});
|
|
||||||
}).then((facesToUpdate) => {
|
|
||||||
const total = facesToUpdate.length;
|
|
||||||
let remaining = total,
|
|
||||||
processed = 0,
|
|
||||||
lastStatus = Date.now(),
|
|
||||||
targets = [];
|
|
||||||
|
|
||||||
for (let target in descriptors) {
|
|
||||||
targets.push({ id: target, descriptor: descriptors[target] });
|
|
||||||
}
|
|
||||||
|
|
||||||
return Promise.mapSeries(facesToUpdate, (face) => {
|
|
||||||
if (!(face.id in descriptors)) {
|
|
||||||
console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const faceDescriptor = descriptors[face.id];
|
descriptors[id] = JSON.parse(fs.readFileSync(dataPath));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}).then(() => {
|
||||||
|
const total = facesToUpdate.length;
|
||||||
|
let remaining = total,
|
||||||
|
processed = 0,
|
||||||
|
lastStatus = Date.now(),
|
||||||
|
targets = [];
|
||||||
|
|
||||||
return photoDB.sequelize.transaction((transaction) => {
|
for (let target in descriptors) {
|
||||||
|
targets.push({ id: target, descriptor: descriptors[target] });
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.mapSeries(facesToUpdate, (face) => {
|
||||||
|
if (!(face.id in descriptors)) {
|
||||||
|
console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const faceDescriptor = descriptors[face.id];
|
||||||
|
|
||||||
|
return photoDB.sequelize.transaction((transaction) => {
|
||||||
|
return photoDB.sequelize.query(
|
||||||
|
"SELECT distance,face1Id,face2Id " +
|
||||||
|
"FROM facedistances " +
|
||||||
|
"WHERE face1Id=:id OR face2Id=:id " +
|
||||||
|
"ORDER BY face1Id ASC", {
|
||||||
|
replacements: {
|
||||||
|
id: face.id
|
||||||
|
},
|
||||||
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
||||||
|
raw: true,
|
||||||
|
transaction: transaction
|
||||||
|
}).then((distances) => {
|
||||||
return Promise.map(targets, (target) => {
|
return Promise.map(targets, (target) => {
|
||||||
/* Skip comparing to self */
|
/* Skip comparing to self */
|
||||||
if (target.id == face.id) {
|
if (target.id == face.id) {
|
||||||
@ -259,87 +293,66 @@ faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const index = distances.findIndex((distance) => {
|
||||||
|
return distance.face1Id == target.id || distance.face2Id == target.id
|
||||||
|
});
|
||||||
|
|
||||||
|
if (index != -1) {
|
||||||
|
/* A distance has already been calculated between face and target */
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const distance = faceapi.euclideanDistance(faceDescriptor, target.descriptor);
|
||||||
|
|
||||||
|
/* If the distance > 0.6, we don't want to store this in the DB */
|
||||||
|
if (distance > 0.6) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (distance < 0.4) {
|
||||||
|
console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
|
||||||
|
}
|
||||||
|
|
||||||
return photoDB.sequelize.query(
|
return photoDB.sequelize.query(
|
||||||
"SELECT distance " +
|
"INSERT INTO facedistances (face1Id,face2Id,distance) " +
|
||||||
"FROM facedistances " +
|
"VALUES (:first,:second,:distance)", {
|
||||||
"WHERE face1Id=:first AND face2Id=:second", {
|
|
||||||
replacements: {
|
replacements: {
|
||||||
first: Math.min(face.id, target.id),
|
first: Math.min(face.id, target.id),
|
||||||
second: Math.max(face.id, target.id)
|
second: Math.max(face.id, target.id),
|
||||||
|
distance: distance
|
||||||
},
|
},
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true,
|
|
||||||
transaction: transaction
|
transaction: transaction
|
||||||
}).then((results) => {
|
|
||||||
if (results.length) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const distance = faceapi.euclideanDistance(faceDescriptor, target.descriptor);
|
|
||||||
|
|
||||||
if (distance < 0.4) {
|
|
||||||
console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"INSERT INTO facedistances (face1Id, face2Id, distance) " +
|
|
||||||
"VALUES (:first, :second, :distance)", {
|
|
||||||
replacements: {
|
|
||||||
first: Math.min(face.id, target.id),
|
|
||||||
second: Math.max(face.id, target.id),
|
|
||||||
distance: distance
|
|
||||||
},
|
|
||||||
transaction: transaction
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
}, {
|
}, {
|
||||||
concurrency: maxConcurrency
|
concurrency: maxConcurrency
|
||||||
}).then(() => {
|
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
|
|
||||||
replacements: {
|
|
||||||
lastId: maxId,
|
|
||||||
id: face.id
|
|
||||||
},
|
|
||||||
transaction: transaction
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
}).then(() => {
|
}).then(() => {
|
||||||
processed++;
|
return photoDB.sequelize.query(
|
||||||
const now = Date.now();
|
"UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
|
||||||
if (now - lastStatus > 5000) {
|
replacements: {
|
||||||
const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
|
lastId: maxId,
|
||||||
eta = Math.round((total - processed) / rate);
|
id: face.id
|
||||||
lastStatus = now;
|
},
|
||||||
remaining = total - processed;
|
transaction: transaction
|
||||||
console.log(`Processing ${rate} faces per second. ${remaining} faces to be processed. ETA: ${eta}s`);
|
});
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
}).then(() => {
|
||||||
|
processed++;
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - lastStatus > 5000) {
|
||||||
|
const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
|
||||||
|
eta = Math.round((total - processed) / rate);
|
||||||
|
lastStatus = now;
|
||||||
|
remaining = total - processed;
|
||||||
|
console.log(`Processing ${rate} faces per second. ${remaining} faces to be processed. ETA: ${eta}s`);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}).then(() => {
|
|
||||||
console.log("Face detection scanning completed.");
|
|
||||||
}).catch((error) => {
|
|
||||||
console.error(error);
|
|
||||||
process.exit(-1);
|
|
||||||
});
|
});
|
||||||
|
}).then(() => {
|
||||||
|
console.log("Face detection scanning completed.");
|
||||||
|
}).catch((error) => {
|
||||||
|
console.error(error);
|
||||||
|
process.exit(-1);
|
||||||
});
|
});
|
||||||
|
|
||||||
/* TODO:
|
|
||||||
1. For each path / person, look up highest face confidence and tag
|
|
||||||
2. Use highest face and identity confidence for input into
|
|
||||||
https://github.com/justadudewhohacks/face-api.js#face-recognition-by-matching-descriptors
|
|
||||||
|
|
||||||
const labeledDescriptors = [
|
|
||||||
new faceapi.LabeledFaceDescriptors(
|
|
||||||
'obama',
|
|
||||||
[descriptorObama1, descriptorObama2]
|
|
||||||
),
|
|
||||||
new faceapi.LabeledFaceDescriptors(
|
|
||||||
'trump',
|
|
||||||
[descriptorTrump]
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
const faceMatcher = new faceapi.FaceMatcher(labeledDescriptors)
|
|
||||||
*/
|
|
||||||
|
|||||||
202
server/face.js
202
server/face.js
@ -4,6 +4,8 @@ process.env.TZ = "Etc/GMT";
|
|||||||
|
|
||||||
require('@tensorflow/tfjs-node');
|
require('@tensorflow/tfjs-node');
|
||||||
|
|
||||||
|
let photoDB = null;
|
||||||
|
|
||||||
const config = require("config"),
|
const config = require("config"),
|
||||||
Promise = require("bluebird"),
|
Promise = require("bluebird"),
|
||||||
{ exists, mkdir, unlink } = require("./lib/util"),
|
{ exists, mkdir, unlink } = require("./lib/util"),
|
||||||
@ -11,7 +13,7 @@ const config = require("config"),
|
|||||||
fs = require("fs"),
|
fs = require("fs"),
|
||||||
canvas = require("canvas");
|
canvas = require("canvas");
|
||||||
|
|
||||||
const { Canvas, Image, ImageData } = canvas;
|
const { createCanvas, Canvas, Image, ImageData } = canvas;
|
||||||
|
|
||||||
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
||||||
|
|
||||||
@ -19,45 +21,179 @@ const maxConcurrency = require("os").cpus().length;
|
|||||||
|
|
||||||
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
|
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
|
||||||
|
|
||||||
faceapi.nets.ssdMobilenetv1.loadFromDisk('./models')
|
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
|
||||||
.then(() => {
|
faceData = picturesPath + "face-data/";
|
||||||
console.log("ssdMobileNetv1 loaded.");
|
|
||||||
|
process.stdout.write("Loading DB.");
|
||||||
|
require("./db/photos").then(function(db) {
|
||||||
|
process.stdout.write("done\n");
|
||||||
|
photoDB = db;
|
||||||
|
}).then(() => {
|
||||||
|
console.log("DB connected.");
|
||||||
|
process.stdout.write("Loading models.");
|
||||||
|
return faceapi.nets.ssdMobilenetv1.loadFromDisk('./models');
|
||||||
|
}).then(() => {
|
||||||
|
process.stdout.write(".");
|
||||||
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
|
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
|
||||||
}).then(() => {
|
}).then(() => {
|
||||||
console.log("landmark68 loaded.");
|
process.stdout.write(".");
|
||||||
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
|
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
|
||||||
}).then(async () => {
|
}).then(async () => {
|
||||||
console.log("faceRecognitionNet loaded.");
|
process.stdout.write(".done\n");
|
||||||
let faces = [];
|
|
||||||
for (let a = 2; a < process.argv.length; a++) {
|
|
||||||
const file = process.argv[a];
|
|
||||||
process.stdout.write(`Loading ${file}...`);
|
|
||||||
const image = await canvas.loadImage(file),
|
|
||||||
detectors = await faceapi.detectAllFaces(image,
|
|
||||||
new faceapi.SsdMobilenetv1Options({
|
|
||||||
minConfidence: 0.8
|
|
||||||
})
|
|
||||||
).withFaceLandmarks().withFaceDescriptors();
|
|
||||||
process.stdout.write(`${detectors.length} faces.\n`);
|
|
||||||
|
|
||||||
detectors.forEach((face, index) => {
|
if (process.argv[0].match(/node/)) {
|
||||||
faces.push({
|
process.argv.shift(); /* node */
|
||||||
file: file,
|
}
|
||||||
index: index,
|
process.argv.shift(); /* script name */
|
||||||
descriptor: face.descriptor
|
|
||||||
|
return Promise.resolve().then(() => {
|
||||||
|
console.log(process.argv.length);
|
||||||
|
|
||||||
|
if (process.argv.length != 0) {
|
||||||
|
return process.argv;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* If no parameters provided, scan all faces to create image crops */
|
||||||
|
return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id ASC", {
|
||||||
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
||||||
|
raw: true
|
||||||
|
}).then((results) => {
|
||||||
|
return results.map(result => result.id);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}).then((args) => {
|
||||||
|
const faces = [];
|
||||||
|
|
||||||
|
return Promise.map(args, (arg) => {
|
||||||
|
const file = arg,
|
||||||
|
id = parseInt(arg);
|
||||||
|
|
||||||
|
let loader;
|
||||||
|
|
||||||
|
if (id == file) {
|
||||||
|
/* This is a face id */
|
||||||
|
console.log(`Looking up face-id ${id}...`);
|
||||||
|
loader = photoDB.sequelize.query(
|
||||||
|
"SELECT albums.path,photos.filename,photos.width,photos.height,faces.* " +
|
||||||
|
"FROM faces,photos,albums " +
|
||||||
|
"WHERE photos.id=faces.photoId " +
|
||||||
|
"AND albums.id=photos.albumId " +
|
||||||
|
"AND faces.id=:id", {
|
||||||
|
replacements: {
|
||||||
|
id: id
|
||||||
|
},
|
||||||
|
type: photoDB.sequelize.QueryTypes.SELECT,
|
||||||
|
raw: true
|
||||||
|
}).then((results) => {
|
||||||
|
if (results.length != 1) {
|
||||||
|
console.error(`...error. No face-id found: ${id}.\n`);
|
||||||
|
process.exit(-1);
|
||||||
|
}
|
||||||
|
const photo = results[0];
|
||||||
|
console.log(`...loading ${photo.filename}`);
|
||||||
|
|
||||||
|
const file = photo.path + photo.filename;
|
||||||
|
return canvas.loadImage(picturesPath + file).then(async (image) => {
|
||||||
|
const detectors = [ {
|
||||||
|
detection: {
|
||||||
|
_box: {
|
||||||
|
_x: photo.left * photo.width,
|
||||||
|
_y: photo.top * photo.height,
|
||||||
|
_width: (photo.right - photo.left) * photo.width,
|
||||||
|
_height: (photo.bottom - photo.top) * photo.height,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
descriptor: JSON.parse(fs.readFileSync(faceData + (id % 100) + "/" + id + "-data.json"))
|
||||||
|
} ];
|
||||||
|
return [ file, image, detectors ];
|
||||||
|
});
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
/* This is a file */
|
||||||
|
console.log(`Loading ${file}...`);
|
||||||
|
id = undefined;
|
||||||
|
loader = canvas.loadImage(picturesPath + file).then(async (image) => {
|
||||||
|
const detectors = await faceapi.detectAllFaces(image,
|
||||||
|
new faceapi.SsdMobilenetv1Options({
|
||||||
|
minConfidence: 0.8
|
||||||
|
})
|
||||||
|
).withFaceLandmarks().withFaceDescriptors();
|
||||||
|
detectors.forEach((detector) => {
|
||||||
|
const data = [];
|
||||||
|
/* Confert from sparse object to dense array */
|
||||||
|
for (let i = 0; i < 128; i++) {
|
||||||
|
data.push(detector.descriptor[i]);
|
||||||
|
}
|
||||||
|
detector.descriptor = data;
|
||||||
|
});
|
||||||
|
return [ file, image, detectors ];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return loader.spread((filepath, image, detectors) => {
|
||||||
|
process.stdout.write(`${detectors.length} faces.\n`);
|
||||||
|
|
||||||
|
return Promise.map(detectors, (face, index) => {
|
||||||
|
faces.push({
|
||||||
|
filepath: filepath,
|
||||||
|
index: index,
|
||||||
|
descriptor: face.descriptor
|
||||||
|
})
|
||||||
|
|
||||||
|
/* If this is a face-id, output the -original.png
|
||||||
|
* meta-data file */
|
||||||
|
if (!id) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const path = "face-data/" + (id % 100),
|
||||||
|
target = `${path}/${id}-original.png`,
|
||||||
|
box = face.detection._box,
|
||||||
|
aspect = box._width / box._height,
|
||||||
|
dx = (aspect > 1.0) ? 200 : (200 * aspect),
|
||||||
|
dy = (aspect < 1.0) ? 200 : (200 / aspect);
|
||||||
|
|
||||||
|
return exists(target).then((doesExist) => {
|
||||||
|
if (doesExist) {
|
||||||
|
console.log(`...${target} already exists.`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const canvas = createCanvas(200, 200),
|
||||||
|
ctx = canvas.getContext('2d');
|
||||||
|
|
||||||
|
ctx.fillStyle = "rgba(0, 0, 0, 0)";
|
||||||
|
ctx.fillRect(0, 0, 200, 200);
|
||||||
|
ctx.drawImage(image, box._x, box._y, box._width, box._height,
|
||||||
|
Math.floor((200 - dx) * 0.5),
|
||||||
|
Math.floor((200 - dy) * 0.5), dx, dy);
|
||||||
|
|
||||||
|
console.log(`...writing to ${target}.`);
|
||||||
|
|
||||||
|
return mkdir(path).then(() => {
|
||||||
|
fs.writeFileSync(picturesPath + target, canvas.toBuffer("image/png", {
|
||||||
|
quality: 0.95,
|
||||||
|
chromaSubsampling: false
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}, {
|
||||||
|
concurrency: maxConcurrency
|
||||||
|
}).then(() => {
|
||||||
|
console.log("Face detection scanning completed.");
|
||||||
|
faces.forEach((a, i) => {
|
||||||
|
faces.forEach((b, j) => {
|
||||||
|
if (i == j) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const distance = faceapi.euclideanDistance(a.descriptor, b.descriptor);
|
||||||
|
if (distance < 0.4) {
|
||||||
|
console.log(`${a.filepath}.${a.index} is similar to ${b.filepath}.${b.index}: ${distance}`);
|
||||||
|
}
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
}
|
});
|
||||||
|
|
||||||
for (let i = 0; i < faces.length; i++) {
|
|
||||||
for (let j = 0; j < faces.length; j++) {
|
|
||||||
const a = faces[i], b = faces[j];
|
|
||||||
const distance = faceapi.euclideanDistance(a.descriptor, b.descriptor);
|
|
||||||
console.log(`${a.file}.${a.index} to ${b.file}.${b.index} = ${distance}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}).then(() => {
|
|
||||||
console.log("Face detection scanning completed.");
|
|
||||||
}).catch((error) => {
|
}).catch((error) => {
|
||||||
console.error(error);
|
console.error(error);
|
||||||
process.exit(-1);
|
process.exit(-1);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user