703 lines
19 KiB
JavaScript
Executable File
703 lines
19 KiB
JavaScript
Executable File
"use strict";
|
|
|
|
const express = require("express"),
|
|
Promise = require("bluebird");
|
|
|
|
let photoDB;
|
|
|
|
require("../db/photos").then(function(db) {
|
|
photoDB = db;
|
|
});
|
|
|
|
const MIN_DISTANCE_COMMIT = 0.0000001
|
|
|
|
const router = express.Router();
|
|
|
|
const upsertIdentity = async(id, {
|
|
displayName,
|
|
firstName,
|
|
lastName,
|
|
middleName
|
|
}, res) => {
|
|
|
|
if (displayName === undefined
|
|
|| firstName === undefined
|
|
|| lastName === undefined
|
|
|| middleName === undefined) {
|
|
res.status(400).send({ message: `Missing fields` });
|
|
return undefined;
|
|
}
|
|
|
|
const identity = {
|
|
displayName,
|
|
firstName,
|
|
lastName,
|
|
middleName,
|
|
identityId: id
|
|
};
|
|
|
|
if (id === -1 || !id) {
|
|
const [results, { lastId }] = await photoDB.sequelize.query(
|
|
`INSERT INTO identities ` +
|
|
'(displayName,firstName,lastName,middleName)' +
|
|
'VALUES(:displayName,:firstName,:lastName,:middleName)', {
|
|
replacements: identity
|
|
});
|
|
identity.identityId = lastId;
|
|
console.log('Created identity: ', identity)
|
|
} else {
|
|
await photoDB.sequelize.query(
|
|
`UPDATE identities ` +
|
|
'SET ' +
|
|
'displayName=:displayName, ' +
|
|
'firstName=:firstName, ' +
|
|
'lastName=:lastName, ' +
|
|
'middleName=:middleName ' +
|
|
'WHERE id=:identityId', {
|
|
replacements: identity
|
|
});
|
|
console.log('Updated identity: ', identity)
|
|
}
|
|
|
|
return identity;
|
|
};
|
|
|
|
|
|
const populateRelatedFaces = async (identity, count) => {
|
|
let limit = '';
|
|
if (count) {
|
|
limit = ` LIMIT ${count} `;
|
|
}
|
|
/* If this is a new identity, no faces are being requested --
|
|
* just return the empty 'unknown face'.
|
|
*
|
|
* Otherwise, query the DB for 'count' faces */
|
|
if (count !== undefined) {
|
|
identity.relatedFaces = await photoDB.sequelize.query(
|
|
"SELECT id AS faceId,photoId,faceConfidence " +
|
|
"FROM faces " +
|
|
"WHERE identityId=:identityId " +
|
|
"ORDER BY distance ASC" +
|
|
limit, {
|
|
replacements: identity,
|
|
type: photoDB.Sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
});
|
|
} else {
|
|
identity.relatedFaces = [];
|
|
}
|
|
|
|
/* If there are no faces, return the 'unknown face' */
|
|
if (identity.relatedFaces.length === 0) {
|
|
identity.relatedFaces.push({
|
|
faceId: -1,
|
|
photoId: -1,
|
|
faceConfidence: 0
|
|
});
|
|
}
|
|
|
|
identity.relatedFaces.forEach(face => {
|
|
face.identityId = identity.identityId;
|
|
face.distance = face.faceConfidence;
|
|
face.descriptors = [];
|
|
delete face.faceConfidence;
|
|
});
|
|
}
|
|
|
|
/* Create new identity */
|
|
router.post('/', async (req, res) => {
|
|
console.log(`POST ${req.url}`)
|
|
if (!req.user.maintainer) {
|
|
console.warn(`${req.user.name} attempted to modify photos.`);
|
|
return res.status(401).send({ message: "Unauthorized to modify photos." });
|
|
}
|
|
|
|
const identity = await upsertIdentity(-1, req.body, res);
|
|
if (!identity) {
|
|
return;
|
|
}
|
|
updateIdentityFaces(identity);
|
|
await populateRelatedFaces(identity, 1);
|
|
return res.status(200).send(identity);
|
|
});
|
|
|
|
/* Update identity */
|
|
router.put('/:id', async (req, res) => {
|
|
console.log(`PUT ${req.url}`)
|
|
if (!req.user.maintainer) {
|
|
console.warn(`${req.user.name} attempted to modify photos.`);
|
|
return res.status(401).send({ message: "Unauthorized to modify photos." });
|
|
}
|
|
|
|
const { id } = req.params;
|
|
if (!id || isNaN(+id)) {
|
|
return res.status(400).send({ message: `Invalid identity id ${id}` });
|
|
}
|
|
|
|
const identity = await upsertIdentity(id, req.body, res);
|
|
if (!identity) {
|
|
return;
|
|
}
|
|
await updateIdentityFaces(identity);
|
|
await populateRelatedFaces(identity);
|
|
return res.status(200).send(identity);
|
|
});
|
|
|
|
|
|
router.delete('/:id', async (req, res) => {
|
|
console.log(`DELETE ${req.url}`)
|
|
if (!req.user.maintainer) {
|
|
console.warn(`${req.user.name} attempted to modify photos.`);
|
|
return res.status(401).send({ message: "Unauthorized to modify photos." });
|
|
}
|
|
|
|
const { id } = req.params;
|
|
if (!id || isNaN(+id)) {
|
|
return res.status(400).send({ message: `Invalid identity id ${id}` });
|
|
}
|
|
|
|
await photoDB.sequelize.query(
|
|
'UPDATE faces SET distance=0,identityId=NULL ' +
|
|
'WHERE identityId=:id', {
|
|
replacements: { id }
|
|
}
|
|
);
|
|
|
|
await photoDB.sequelize.query(
|
|
'DELETE FROM identities ' +
|
|
'WHERE id=:id', {
|
|
replacements: { id }
|
|
});
|
|
|
|
return res.status(200).send({});
|
|
});
|
|
|
|
/* Given a faceId, find the closest defined identity and return
|
|
* it as a guess -- does not modify the DB */
|
|
router.get("/faces/guess/:faceId", async (req, res) => {
|
|
const faceId = parseInt(req.params.faceId);
|
|
if (faceId != req.params.faceId) {
|
|
return res.status(400).send({ message: "Invalid identity id." });
|
|
}
|
|
|
|
try {
|
|
/* Look up the requested face... */
|
|
const faces = await photoDB.sequelize.query(
|
|
"SELECT faces.*,faceDescriptors.descriptors " +
|
|
"FROM faces,faceDescriptors " +
|
|
"WHERE faces.id=:faceId " +
|
|
"AND faceDescriptors.id=faces.descriptorId", {
|
|
replacements: { faceId },
|
|
type: photoDB.Sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
});
|
|
|
|
/* Look up all the identities... */
|
|
const identities = await photoDB.sequelize.query(
|
|
"SELECT * FROM identities", {
|
|
type: photoDB.Sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
});
|
|
identities.forEach(x => { x.identityId = x.id; delete x.id;});
|
|
|
|
faces.forEach((face) => {
|
|
const currentIdentityId = face.identityId;
|
|
face.faceId = face.id;
|
|
delete face.id;
|
|
face.identity = null;
|
|
|
|
|
|
identities.forEach((identity) => {
|
|
if (!identity.descriptors
|
|
|| currentIdentityId === identity.identityId) {
|
|
return;
|
|
}
|
|
|
|
const distance = euclideanDistance(
|
|
face.descriptors,
|
|
identity.descriptors
|
|
);
|
|
|
|
if (face.identity === null) {
|
|
face.identityId = identity.identityId;
|
|
face.identity = identity;
|
|
face.distance = distance;
|
|
return;
|
|
}
|
|
|
|
if (distance < face.distance) {
|
|
face.identityId = identity.identityId;
|
|
face.identity = identity;
|
|
face.distance = distance;
|
|
}
|
|
});
|
|
});
|
|
|
|
/* Delete the VGG-Face descriptors and add relatedFaces[0] */
|
|
await Promise.map(faces, async (face) => {
|
|
delete face.descriptors;
|
|
if (face.identity.descriptors) {
|
|
delete face.identity.descriptors;
|
|
}
|
|
if (!face.identity.relatedFaces) {
|
|
await populateRelatedFaces(face.identity, 1);
|
|
}
|
|
}, {
|
|
concurrency: 1
|
|
});
|
|
|
|
return res.status(200).json(faces);
|
|
} catch (error) {
|
|
console.error(error);
|
|
return res.status(500).send("Error processing request.");
|
|
};
|
|
});
|
|
|
|
router.put("/faces/remove/:id", async (req, res) => {
|
|
console.log(`PUT ${req.url}`)
|
|
if (!req.user.maintainer) {
|
|
console.warn(`${req.user.name} attempted to modify photos.`);
|
|
return res.status(401).send({ message: "Unauthorized to modify photos." });
|
|
}
|
|
|
|
const id = parseInt(req.params.id);
|
|
if (id != req.params.id) {
|
|
return res.status(400).send({ message: "Invalid identity id." });
|
|
}
|
|
|
|
if (!Array.isArray(req.body.faces) || req.body.faces.length == 0) {
|
|
return res.status(400).send({ message: "No faces supplied." });
|
|
}
|
|
|
|
try {
|
|
await photoDB.sequelize.query(
|
|
"UPDATE faces SET identityId=null " +
|
|
"WHERE id IN (:faceIds)", {
|
|
replacements: {
|
|
identityId: id,
|
|
faceIds: req.body.faces
|
|
}
|
|
});
|
|
const identity = {
|
|
identityId: id,
|
|
faces: req.body.faces
|
|
};
|
|
identity.faces = identity.faces.map(id => +id);
|
|
|
|
/* Do not block on this call finishing -- update can occur
|
|
* in the background */
|
|
updateIdentityFaces(identity);
|
|
|
|
return res.status(200).json(identity);
|
|
} catch (error) {
|
|
console.error(error);
|
|
return res.status(500).send({message: "Error processing request." });
|
|
};
|
|
});
|
|
|
|
router.put("/faces/add/:id", async (req, res) => {
|
|
if (!req.user.maintainer) {
|
|
console.warn(`${req.user.name} attempted to modify photos.`);
|
|
return res.status(401).send("Unauthorized to modify photos.");
|
|
}
|
|
|
|
const id = parseInt(req.params.id);
|
|
if (id != req.params.id) {
|
|
return res.status(400).send("Invalid identity id.");
|
|
}
|
|
|
|
if (!Array.isArray(req.body.faces) || req.body.faces.length == 0) {
|
|
return res.status(400).send("No faces supplied.");
|
|
}
|
|
|
|
try {
|
|
await photoDB.sequelize.query(
|
|
"UPDATE faces SET identityId=:identityId,classifiedBy='human' " +
|
|
"WHERE id IN (:faceIds)", {
|
|
replacements: {
|
|
identityId: id,
|
|
faceIds: req.body.faces
|
|
}
|
|
});
|
|
const identity = {
|
|
identityId: id,
|
|
faces: req.body.faces
|
|
};
|
|
identity.faces = identity.faces.map(id => +id);
|
|
|
|
/* Do not block on this call finishing -- update can occur
|
|
* in the background */
|
|
updateIdentityFaces(identity);
|
|
return res.status(200).json(identity);
|
|
} catch (error) {
|
|
console.error(error);
|
|
return res.status(500).send("Error processing request.");
|
|
};
|
|
});
|
|
|
|
|
|
function bufferToFloat32Array(buffer) {
|
|
return new Float64Array(buffer.buffer,
|
|
buffer.byteOffset,
|
|
buffer.byteLength / Float64Array.BYTES_PER_ELEMENT);
|
|
}
|
|
|
|
function euclideanDistanceArray(a, b) {
|
|
let sum = 0;
|
|
for (let i = 0; i < a.length; i++) {
|
|
let delta = a[i] - b[i];
|
|
sum += delta * delta;
|
|
}
|
|
return Math.sqrt(sum);
|
|
}
|
|
|
|
function euclideanDistance(a, b) {
|
|
if (!a.buffer || !b.buffer) {
|
|
return -1;
|
|
}
|
|
|
|
return euclideanDistanceArray(
|
|
bufferToFloat32Array(a),
|
|
bufferToFloat32Array(b)
|
|
);
|
|
}
|
|
|
|
const getUnknownIdentity = async (faceCount) => {
|
|
const unknownIdentity = {
|
|
identityId: -1,
|
|
lastName: '',
|
|
firstName: '',
|
|
middleName: '',
|
|
displayName: 'Unknown',
|
|
descriptors: new Float32Array(0),
|
|
relatedFaces: [],
|
|
facesCount: 0
|
|
};
|
|
const limit = faceCount
|
|
? ` ORDER BY RANDOM() LIMIT ${faceCount} `
|
|
: ' ORDER BY faceConfidence DESC ';
|
|
unknownIdentity.relatedFaces = await photoDB.sequelize.query(
|
|
"SELECT id AS faceId,photoId,faceConfidence " +
|
|
"FROM faces WHERE identityId IS NULL AND classifiedBy != 'not-a-face' " +
|
|
limit, {
|
|
type: photoDB.Sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
});
|
|
if (unknownIdentity.relatedFaces.length === 0) {
|
|
unknownIdentity.relatedFaces.push({
|
|
faceId: -1,
|
|
photoId: -1,
|
|
faceConfidence: 0
|
|
});
|
|
}
|
|
unknownIdentity.relatedFaces.forEach(face => {
|
|
face.identityId = -1;
|
|
face.distance = face.faceConfidence;
|
|
face.descriptors = new Float32Array(0);
|
|
delete face.faceConfidence;
|
|
});
|
|
return unknownIdentity;
|
|
}
|
|
|
|
const round = (x, precision) => {
|
|
if (precision === undefined) {
|
|
precision = 2;
|
|
}
|
|
return Number.parseFloat(x).toFixed(precision);
|
|
};
|
|
|
|
/* Compute the identity's centroid descriptor from all faces
|
|
* and determine closest face to that centroid. If either of
|
|
* those values have changed, update the identity.
|
|
*
|
|
* Also updates the 'distance' on each face to the identity
|
|
* centroid.
|
|
*/
|
|
const updateIdentityFaces = async (identity) => {
|
|
if (!identity.identityId) {
|
|
throw Error(`identityId is not set.`);
|
|
}
|
|
|
|
if (!identity.descriptors) {
|
|
const results = await photoDB.sequelize.query(
|
|
"SELECT " +
|
|
"descriptors,facesCount,faceId,displayName " +
|
|
"FROM identities " +
|
|
"WHERE id=:identityId", {
|
|
replacements: identity,
|
|
type: photoDB.Sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
});
|
|
Object.assign(identity, results[0]);
|
|
/* New identities do not have descriptors set */
|
|
}
|
|
|
|
if (identity.descriptors) {
|
|
identity.descriptors = bufferToFloat32Array(identity.descriptors);
|
|
}
|
|
|
|
const faces = await photoDB.sequelize.query(
|
|
"SELECT " +
|
|
"faces.*,faceDescriptors.* " +
|
|
"FROM faces,faceDescriptors " +
|
|
"WHERE " +
|
|
"faces.identityId=:identityId " +
|
|
"AND faceDescriptors.id=faces.descriptorId", {
|
|
replacements: identity,
|
|
type: photoDB.Sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
});
|
|
|
|
if (faces.length === 0) {
|
|
return;
|
|
}
|
|
|
|
let average = undefined,
|
|
closestId = -1,
|
|
closestDistance = -1;
|
|
|
|
/* First find the average centroid of all faces */
|
|
faces.forEach((face) => {
|
|
/* Convert the descriptors from buffer to array so they can be
|
|
* modified */
|
|
face.descriptors = bufferToFloat32Array(face.descriptors);
|
|
|
|
/* First face starts the average sum */
|
|
if (average === undefined) {
|
|
average = face.descriptors.slice();
|
|
return;
|
|
}
|
|
|
|
/* Add this face descriptor into the average descriptor */
|
|
for (let i = 0; i < face.descriptors.length; i++) {
|
|
average[i] = average[i] + face.descriptors[i];
|
|
};
|
|
});
|
|
|
|
/* Divide sum of descriptors to create average centroid */
|
|
for (let i = 0; i < average.length; i++) {
|
|
average[i] = average[i] / faces.length;
|
|
}
|
|
|
|
/* Now compute the distance from each face to the new centroid */
|
|
faces.forEach((face) => {
|
|
let distance;
|
|
|
|
distance = euclideanDistanceArray(
|
|
face.descriptors,
|
|
average
|
|
);
|
|
|
|
if (closestId === -1 || face.distance < closestDistance) {
|
|
closestDistance = distance;
|
|
closestId = face.id;
|
|
}
|
|
});
|
|
|
|
/* Determine if the centroid for this identity has moved
|
|
* and for each relatedFace, update its distance to the centroid */
|
|
if (!identity.descriptors) {
|
|
console.log(`Identity ${identity.identityId} has no descriptors`);
|
|
}
|
|
let moved = (identity.descriptors === null ? 1 : 0)
|
|
|| Number
|
|
.parseFloat(euclideanDistanceArray(identity.descriptors, average))
|
|
.toFixed(4);
|
|
|
|
const t = await photoDB.sequelize.transaction();
|
|
try {
|
|
/* If the average position has not changed, then face distances should
|
|
* not change either! */
|
|
await Promise.map(faces, async (face) => {
|
|
/* All the buffer are already arrays, so use the short-cut version */
|
|
const distance = Number
|
|
.parseFloat(euclideanDistanceArray(face.descriptors, average))
|
|
.toFixed(4);
|
|
|
|
if (Math.abs(distance - face.distance) > MIN_DISTANCE_COMMIT) {
|
|
console.log(
|
|
`Updating face ${face.id} to ${round(distance, 2)} ` +
|
|
`(${distance - face.distance}) ` +
|
|
`from identity ${identity.identityId} (${identity.displayName})`);
|
|
face.distance = distance;
|
|
await photoDB.sequelize.query(
|
|
'UPDATE faces SET distance=:distance WHERE id=:id', {
|
|
replacements: face,
|
|
transaction: t
|
|
}
|
|
);
|
|
}
|
|
}, {
|
|
concurrency: 5
|
|
});
|
|
|
|
let sql = '';
|
|
/* If there is a new closestId, then set the faceId field */
|
|
if (closestId !== -1 && closestId !== identity.faceId) {
|
|
console.log(
|
|
`Updating identity ${identity.identityId} closest face to ${closestId}`);
|
|
sql = `${sql} faceId=:faceId`;
|
|
identity.faceId = closestId;
|
|
}
|
|
|
|
/* If the centroid changed, update the identity descriptors to
|
|
* the new average */
|
|
if (Math.abs(moved) > MIN_DISTANCE_COMMIT) {
|
|
console.log(
|
|
`Updating identity ${identity.identityId} centroid ` +
|
|
`(moved ${Number.parseFloat(moved).toFixed(4)}).`);
|
|
if (sql !== '') {
|
|
sql = `${sql}, `;
|
|
}
|
|
sql = `${sql} descriptors=:descriptors`;
|
|
// this: identity.descriptors = average;
|
|
// gives: Invalid value Float64Array(2622)
|
|
//
|
|
// this: identity.descriptors = new Blob(average);
|
|
// gives: Invalid value Blob { size: 54008, type: '' }
|
|
//
|
|
// this: identity.descriptors = Buffer.from(average);
|
|
// gives: all zeroes
|
|
//
|
|
// this: identity.descriptors = Buffer.from(average.buffer);
|
|
// gives: IT WORKS!!!
|
|
identity.descriptors = Buffer.from(average.buffer);
|
|
}
|
|
|
|
/* If the number of faces changed, update the facesCount */
|
|
if (identity.facesCount !== faces.length) {
|
|
if (sql !== '') {
|
|
sql = `${sql}, `;
|
|
}
|
|
console.log(
|
|
`Updating identity ${identity.identityId} face count to ${faces.length}`);
|
|
identity.facesCount = faces.length;
|
|
sql = `${sql} facesCount=${faces.length}`;
|
|
}
|
|
|
|
/* If any of the above required changes, actually commit to the DB */
|
|
if (sql !== '') {
|
|
await photoDB.sequelize.query(
|
|
`UPDATE identities SET ${sql} ` +
|
|
`WHERE id=:identityId`, {
|
|
replacements: identity,
|
|
transaction: t
|
|
}
|
|
);
|
|
}
|
|
t.commit();
|
|
} catch (error) {
|
|
console.error(error);
|
|
t.rollback();
|
|
return;
|
|
}
|
|
};
|
|
|
|
router.get("/:id?", async (req, res) => {
|
|
console.log(`GET ${req.url}`);
|
|
|
|
let id;
|
|
|
|
if (req.params.id) {
|
|
id = parseInt(req.params.id);
|
|
if (id != req.params.id) {
|
|
return res.status(400).send({ message: "Usage /[id]"});
|
|
}
|
|
}
|
|
|
|
/* If identityId requested is -1, this is the "Unknown" identity
|
|
* where all unmapped faces live. */
|
|
if (id === -1) {
|
|
const unknownIdentity = await getUnknownIdentity()
|
|
return res.status(200).json([ unknownIdentity ]);
|
|
}
|
|
|
|
const filter = id ? "WHERE identities.id=:id " : "";
|
|
|
|
const identities = await photoDB.sequelize.query(
|
|
"SELECT " +
|
|
"identities.id AS identityId," +
|
|
"identities.firstName," +
|
|
"identities.lastName," +
|
|
"identities.middleName," +
|
|
"identities.displayName," +
|
|
"identities.faceId," +
|
|
"identities.facesCount " +
|
|
"FROM identities " +
|
|
filter, {
|
|
replacements: { id },
|
|
type: photoDB.Sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
});
|
|
|
|
await Promise.map(identities, async (identity) => {
|
|
for (let field in identity) {
|
|
if (field.match(/.*Name/) && identity[field] === null) {
|
|
identity[field] = '';
|
|
}
|
|
}
|
|
|
|
let where, limit = '';
|
|
/* If id was not set, only return a single face */
|
|
if (id === undefined) {
|
|
if (identity.faceId !== -1 && identity.faceId !== null) {
|
|
/* Return the identity faceId, and make sure the face
|
|
* is associated with that id -- they can get out of sync
|
|
* when faces are added/removed from an identity */
|
|
where = 'faceId=:faceId AND identityId=:identityId';
|
|
} else {
|
|
where = 'identityId=:identityId';
|
|
limit = 'LIMIT 1';
|
|
}
|
|
} else {
|
|
where = 'identityId=:identityId'
|
|
}
|
|
|
|
identity.relatedFaces = await photoDB.sequelize.query(
|
|
'SELECT id as faceId,identityId,photoId,distance ' +
|
|
'FROM faces ' +
|
|
`WHERE ${where} ` +
|
|
'ORDER BY distance ASC ' +
|
|
limit, {
|
|
replacements: identity,
|
|
type: photoDB.Sequelize.QueryTypes.SELECT,
|
|
raw: true
|
|
}
|
|
);
|
|
|
|
/* If this identity has at least one face associated with it,
|
|
* and it does not yet have the 'closest' face assigned, update
|
|
* the identity statistics.
|
|
*/
|
|
if (identity.relatedFaces.length !== 0
|
|
&& (!identity.faceId || identity.faceId === -1)) {
|
|
await updateIdentityFaces(identity);
|
|
}
|
|
|
|
/* If there were no faces, then add a 'Unknown' face */
|
|
if (identity.relatedFaces.length === 0) {
|
|
identity.relatedFaces.push({
|
|
faceId: -1,
|
|
photoId: -1,
|
|
identityId: identity.identityId,
|
|
distance: 0,
|
|
faceConfidence: 0
|
|
});
|
|
}
|
|
}, {
|
|
concurrency: 1
|
|
});
|
|
|
|
/* If no ID was provided then this call is returning
|
|
* a list of all identities -- we create a fake identity for all
|
|
* unlabeled faces */
|
|
if (!id) {
|
|
const unknownIdentity = await getUnknownIdentity(1)
|
|
identities.push(unknownIdentity);
|
|
}
|
|
|
|
return res.status(200).json(identities);
|
|
});
|
|
|
|
module.exports = router;
|