/* * Face recognition: * 1. For each photo, extract all faces. Store face rectangles. * face_id unique * photo_id foreign key * top left bottom right * identity_id * distance (0 == truth; manually assigned identity) * 2. For each face_id, create: * /${picturesPath}face-data/${face_id % 100}/ * ${face_id}-normalized * ${face_id}-original * ${face_id}-data */ "use strict"; process.env.TZ = "Etc/GMT"; console.log("Loading face-recognizer"); require('@tensorflow/tfjs-node'); const config = require("config"), Promise = require("bluebird"), { exists, mkdir, unlink } = require("./lib/util"), faceapi = require("face-api.js"), fs = require("fs"), canvas = require("canvas"); const { Canvas, Image, ImageData } = canvas; faceapi.env.monkeyPatch({ Canvas, Image, ImageData }); const maxConcurrency = require("os").cpus().length; require("./console-line.js"); /* Monkey-patch console.log with line numbers */ const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/", faceData = picturesPath + "face-data/"; let photoDB = null; console.log("Loading pictures out of: " + picturesPath); faceapi.nets.ssdMobilenetv1.loadFromDisk('./models') .then(() => { console.log("ssdMobileNetv1 loaded."); return faceapi.nets.faceLandmark68Net.loadFromDisk('./models'); }).then(() => { console.log("landmark68 loaded."); return faceapi.nets.faceRecognitionNet.loadFromDisk('./models'); }).then(() => { console.log("faceRecognitionNet loaded."); return require("./db/photos").then(function(db) { photoDB = db; }).then(() => { console.log("DB connected."); }).then(() => { console.log("Beginning face detection scanning."); return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " + "FROM photos " + "LEFT JOIN albums ON (albums.id=photos.albumId) " + "WHERE faces=-1 ORDER BY albums.path,photos.filename", { type: photoDB.sequelize.QueryTypes.SELECT } ).then((results) => { console.log(`${results.length} photos have not had faces scanned.`); return Promise.map(results, (photo) => { const filePath = photo.path + photo.filename; console.log(`Processing ${filePath}...`); /* Remove any existing face data for this photo */ return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", { replacements: photo, }).then((faces) => { /* For each face-id, remove any face-data files, and then remove all the entries * from the DB */ return Promise.map(faces, (face) => { const id = face.id, filePath = faceData + "/" + (id % 100) + "/" + id + "-data.json"; return exists(filePath).then((result) => { if (result) { console.log(`...removing ${filePath}`); return unlink(filePath); } }); }).then(() => { return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", { replacements: photo, }); }).then(async () => { console.log("...loading image."); const image = await canvas.loadImage(picturesPath + filePath); console.log("...detecting faces."); const detections = await faceapi.detectAllFaces(image).withFaceLandmarks().withFaceDescriptors(); console.log(`...${detections.length} faces identified.`); return Promise.map(detections, (face, index) => { const width = face.detection._box._width, height = face.detection._box._height, replacements = { id: photo.id, top: face.detection._box._y / photo.height, left: face.detection._box._x / photo.width, bottom: (face.detection._box._x + height) / photo.height, right: (face.detection._box._x + width) / photo.width, faceConfidence: face.detection._score }; return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " + "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", { replacements: replacements }).catch(() => { console.error(filePath, index); console.error(JSON.stringify(face, null, 2)); console.error(JSON.stringify(replacements, null, 2)); process.exit(-1); }).spread((results, metadata) => { return metadata.lastID; }).then((id) => { console.log(`...DB id ${id}. Writing descriptor data...`); const path = faceData + "/" + (id % 100); return mkdir(path).then(() => { const filePath = path + "/" + id + "-data.json", data = []; for (let i = 0; i < 128; i++) { data.push(face.descriptor[i]); } fs.writeFileSync(filePath, JSON.stringify(data)); }); }); }).then(() => { return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", { replacements: { id: photo.id, faces: detections.length }, }); }); }); }); }, { concurrency: maxConcurrency }); }); }).then(() => { console.log("Face detection scanning completed."); }).catch((error) => { console.error(error); process.exit(-1); }); }); /* TODO: 1. For each path / person, look up highest face confidence and tag 2. Use highest face and identity confidence for input into https://github.com/justadudewhohacks/face-api.js#face-recognition-by-matching-descriptors const labeledDescriptors = [ new faceapi.LabeledFaceDescriptors( 'obama', [descriptorObama1, descriptorObama2] ), new faceapi.LabeledFaceDescriptors( 'trump', [descriptorTrump] ) ] const faceMatcher = new faceapi.FaceMatcher(labeledDescriptors) */