"use strict"; const express = require("express"), Promise = require("bluebird"); let photoDB; require("../db/photos").then(function(db) { photoDB = db; }); const router = express.Router(); const upsertIdentity = async(id, { displayName, firstName, lastName, middleName }, res) => { if (displayName === undefined || firstName === undefined || lastName === undefined || middleName === undefined) { res.status(400).send({ message: `Missing fields` }); return undefined; } const identity = { displayName, firstName, lastName, middleName, id }; if (id === -1 || !id) { const [results, { lastId }] = await photoDB.sequelize.query( `INSERT INTO identities ` + '(displayName,firstName,lastName,middleName)' + 'VALUES(:displayName,:firstName,:lastName,:middleName)', { replacements: identity }); identity.id = lastId; } else { await photoDB.sequelize.query( `UPDATE identities ` + 'SET ' + 'displayName=:displayName, ' + 'firstName=:firstName, ' + 'lastName=:lastName, ' + 'middleName=:middleName ' + 'WHERE id=:id', { replacements: identity }); } return identity; }; const populateRelatedFaces = async (identity, count) => { let limit = ''; if (count) { limit = ` LIMIT ${count} `; } /* If this is a new identity, no faces are being requested -- * just return the empty 'unknown face'. * * Otherwise, query the DB for 'count' faces */ if (count === undefined) { identity.relatedFaces = await photoDB.sequelize.query( "SELECT id AS faceId,photoId,faceConfidence " + "FROM faces " + "WHERE identityId=:identityId " + limit, { replacements: { identityId: identity.id }, type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }); } else { identity.relatedFaces = []; } /* If there are no faces, return the 'unknown face' */ if (identity.relatedFaces.length === 0) { identity.relatedFaces.push({ faceId: -1, photoId: -1, faceConfidence: 0 }); } identity.relatedFaces.forEach(face => { face.identityId = identity.id; face.distance = face.faceConfidence; face.descriptors = []; delete face.faceConfidence; }); } router.post('/', async (req, res) => { console.log(`POST ${req.url}`) if (!req.user.maintainer) { console.warn(`${req.user.name} attempted to modify photos.`); return res.status(401).send({ message: "Unauthorized to modify photos." }); } const identity = await upsertIdentity(-1, req.body, res); if (!identity) { return; } populateRelatedFaces(identity, 1); return res.status(200).send(identity); }); router.put('/:id', async (req, res) => { console.log(`PUT ${req.url}`) if (!req.user.maintainer) { console.warn(`${req.user.name} attempted to modify photos.`); return res.status(401).send({ message: "Unauthorized to modify photos." }); } const { id } = req.params; if (!id || isNaN(+id)) { return res.status(400).send({ message: `Invalid identity id ${id}` }); } const identity = await upsertIdentity(id, req.body, res); if (!identity) { return; } populateRelatedFaces(identity); return res.status(200).send(identity); }); const addFaceToIdentityDescriptors = (identity, face) => { }; const removeFaceToIdentityDescriptors = (identity, face) => { }; const writeIdentityDescriptors = async (identity) => { await photoDB.sequelize.query( 'UPDATE identities ' + 'SET descriptors=:descriptors' + 'WHERE id=:identityId', { replacements: identity } ); }; router.put("/faces/remove/:id", async (req, res) => { console.log(`PUT ${req.url}`) if (!req.user.maintainer) { console.warn(`${req.user.name} attempted to modify photos.`); return res.status(401).send({ message: "Unauthorized to modify photos." }); } const id = parseInt(req.params.id); if (id != req.params.id) { return res.status(400).send({ message: "Invalid identity id." }); } if (!Array.isArray(req.body.faces) || req.body.faces.length == 0) { return res.status(400).send({ message: "No faces supplied." }); } try { await photoDB.sequelize.query( "UPDATE faces SET identityId=null " + "WHERE id IN (:faceIds)", { replacements: { identityId: id, faceIds: req.body.faces } }); const identity = { id: id, faces: req.body.faces }; identity.faces = identity.faces.map(id => +id); updateIdentityDescriptors(identity); return res.status(200).json(identity); } catch (error) { console.error(error); return res.status(500).send({message: "Error processing request." }); }; }); router.put("/faces/add/:id", async (req, res) => { if (!req.user.maintainer) { console.warn(`${req.user.name} attempted to modify photos.`); return res.status(401).send("Unauthorized to modify photos."); } const id = parseInt(req.params.id); if (id != req.params.id) { return res.status(400).send("Invalid identity id."); } if (!Array.isArray(req.body.faces) || req.body.faces.length == 0) { return res.status(400).send("No faces supplied."); } try { await photoDB.sequelize.query( "UPDATE faces SET identityId=:identityId,classifiedBy='human' " + "WHERE id IN (:faceIds)", { replacements: { identityId: id, faceIds: req.body.faces } }); const identity = { id: id, faces: req.body.faces }; identity.faces = identity.faces.map(id => +id); updateIdentityDescriptors(identity); return res.status(200).json(identity); } catch (error) { console.error(error); return res.status(500).send("Error processing request."); }; }); router.post("/", (req, res) => { if (!req.user.maintainer) { console.warn(`${req.user.name} attempted to modify photos.`); return res.status(401).send("Unauthorized to modify photos."); } const identity = { lastName: req.body.lastName || "", firstName: req.body.firstName || "", middleName: req.body.middleName || "" }; identity.name = req.body.name || (identity.firstName + " " + identity.lastName); let fields = []; for (let key in identity) { fields.push(key); } if (!Array.isArray(req.body.faces) || req.body.faces.length == 0) { return res.status(400).send("No faces supplied."); } return photoDB.sequelize.query("INSERT INTO identities " + "(" + fields.join(",") + ") " + "VALUES(:" + fields.join(",:") + ")", { replacements: identity, }).then(([ results, metadata ]) => { identity.id = metadata.lastID; return photoDB.sequelize.query( "UPDATE faces SET identityId=:identityId " + "WHERE id IN (:faceIds)", { replacements: { identityId: identity.id, faceIds: req.body.faces } }).then(() => { identity.faces = req.body.faces; return res.status(200).json([identity]); }); }).catch((error) => { console.error(error); return res.status(500).send("Error processing request."); }); }); function bufferToFloat32Array(buffer) { return new Float64Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float64Array.BYTES_PER_ELEMENT); } function euclideanDistance(a, b) { if (!a.buffer || !b.buffer) { return -1; } let A = bufferToFloat32Array(a); let B = bufferToFloat32Array(b); let sum = 0; for (let i = 0; i < A.length; i++) { let delta = A[i] - B[i]; sum += delta * delta; } return Math.sqrt(sum); } const getUnknownIdentity = async (faceCount) => { const unknownIdentity = { identityId: -1, lastName: '', firstName: '', middleName: '', displayName: 'Unknown', descriptors: new Float32Array(0), relatedFaces: [] }; const limit = faceCount ? ` LIMIT ${faceCount} ` : ' ORDER BY faceConfidence DESC '; unknownIdentity.relatedFaces = await photoDB.sequelize.query( "SELECT id AS faceId,photoId,faceConfidence " + "FROM faces WHERE identityId IS NULL AND classifiedBy != 'not-a-face' " + limit, { type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }); if (unknownIdentity.relatedFaces.length === 0) { unknownIdentity.relatedFaces.push({ faceId: -1, photoId: -1, faceConfidence: 0 }); } unknownIdentity.relatedFaces.forEach(face => { face.identityId = -1; face.distance = face.faceConfidence; face.descriptors = new Float32Array(0); delete face.faceConfidence; }); return unknownIdentity; } /* Compute the identity's centroid descriptor from all faces * and determine closest face to that centroid. If either of * those values have changed, update the identity. * * Also updates the 'distance' on each face to the identity * centroid. */ const updateIdentityFaces = async (identity) => { if (!identity.identityId) { identity.identityId = identity.id; } const faces = await photoDB.sequelize.query( "SELECT " + "faces.*,faceDescriptors.* " + "WHERE " + "faces.identityId=:identityId " + "AND faceDescriptors.id=faces.descriptorsId", { replacements: identity, type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }); let average = undefined, closestId = -1, closestDistance = -1, count = 0; faces.forEach((face) => { if (!descriptors[index]) { return; } if (!face.descriptors) { return; } face.distance = euclideanDistance( face.descriptors, identity.descriptors ); face.descriptors = bufferToFloat32Array(face.descriptors).map(x => x * x); if (closestId === -1) { closestId = face.id; closestDistance = face.distance; average = descriptors; count = 1; return; } descriptors.forEach((x, i) => { average[i] += x; }); count++; if (face.distance < closestDistance) { closestDistance = face.distance; closestId = face.id; } }); let same = true; if (average) { average = average.map(x => x / count); same = bufferToFloat32Array(identity.descriptors) .find((x, i) => average[i] === x) === undefined; await Promise(faces, async (face) => { const distance = euclideanDistanceArray(face.descriptors, average); if (distance !== face.distance) { await photoDB.sequelize.query( 'UPDATE faces SET distance=:distance WHERE id=:faceId', { replacements: face } ); } }); } let sql = ''; if (closestId !== undefined && closestId !== identity.faceId) { sql = `${sql} faceId=:faceId`; } if (!same) { if (sql !== '') { sql = `${sql}, `; } sql = `${sql} descriptors=:descriptors`; } if (sql !== '') { await photoDB.sequelize.select( `UPDATE identities SET ${sql} ` + `WHERE id=:identityId`, { replacements: { faceId: closestId, descriptors: average, identityId: identity.id } } ); } }; router.get("/:id?", async (req, res) => { console.log(`GET ${req.url}`); let id; if (req.params.id) { id = parseInt(req.params.id); if (id != req.params.id) { return res.status(400).send({ message: "Usage /[id]"}); } } /* If identityId requested is -1, this is the "Unknown" identity * where all unmapped faces live. */ if (id === -1) { const unknownIdentity = await getUnknownIdentity() return res.status(200).json([ unknownIdentity ]); } const filter = id ? "WHERE identities.id=:id " : ""; const identities = await photoDB.sequelize.query( "SELECT " + "identities.id AS identityId," + "identities.firstName," + "identities.lastName," + "identities.middleName," + "identities.displayName," + "identities.faceId " + "FROM identities " + filter, { replacements: { id }, type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }); await Promise.map(identities, async (identity) => { let where; /* If id was set, only return a single face */ if (id !== undefined) { if (identity.faceId !== -1) { where = 'faceId=:faceId'; } else { where = 'identityId=:identityId LIMIT 1'; } } else { where = 'identityId=:identityId' } identity.relatedFaces = await photoDB.sequelize.query( 'SELECT id as faceId,photoId,distance FROM faces ' + `WHERE ${where} ORDER BY distance ASC`, { replacements: identity, type: photoDB.Sequelize.QueryTypes.SELECT, raw: true } ); if (identity.relatedFaces.length !== 0 && (!identity.faceId || identity.faceId === -1)) { await updateIdentityFaces(identity); } /* If there were no faces, then add a 'Unknown' face */ if (identity.relatedFaces.length === 0) { identity.relatedFaces.push({ faceId: -1, photoId: -1, identityId: identity.identityId, distance: 0, faceConfidence: 0 }); } }); /* If no ID was provided then this call is returning * a list of all identities -- we create a fake identity for all * unlabeled faces */ if (!id) { const unknownIdentity = await getUnknownIdentity(1) identities.push(unknownIdentity); } return res.status(200).json(identities); }); module.exports = router;