diff --git a/client/src/App.css b/client/src/App.css index 5506417..bb85f6c 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -85,6 +85,13 @@ div { background-position: 50% 50% !important; } +.UnknownFace { + display: flex; + align-items: center; + font-size: 4rem; + font-weight: bold; +} + .IdentityForm { display: grid; grid-template-columns: 1fr 1fr; @@ -116,6 +123,8 @@ div { box-sizing: border-box; width: 8rem; height: 8rem; + display: flex; + justify-content: center; } .Cluster { diff --git a/client/src/App.tsx b/client/src/App.tsx index 498758d..4de907a 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -123,14 +123,12 @@ const onFaceMouseEnter = (e: any, face: FaceData) => { const faceId = face.faceId; const els = [...document.querySelectorAll(`[data-face-id="${faceId}"]`)]; - if (face.identity) { - const identityId = face.identity.identityId; - els.splice(0, 0, - ...document.querySelectorAll( - `.Identities [data-identity-id="${identityId}"]`), - ...document.querySelectorAll( - `.Photo [data-identity-id="${identityId}"]`)); - } + const identityId = face.identityId; + els.splice(0, 0, + ...document.querySelectorAll( + `.Identities [data-identity-id="${identityId}"]`), + ...document.querySelectorAll( + `.Photo [data-identity-id="${identityId}"]`)); els.forEach(el => { el.classList.add('Active'); @@ -141,11 +139,9 @@ const onFaceMouseLeave = (e: any, face: FaceData) => { const faceId = face.faceId; const els = [...document.querySelectorAll(`[data-face-id="${faceId}"]`)]; - if (face.identity) { - const identityId = face.identity.identityId; - els.splice(0, 0, - ...document.querySelectorAll(`[data-identity-id="${identityId}"]`)); - } + const identityId = face.identityId; + els.splice(0, 0, + ...document.querySelectorAll(`[data-identity-id="${identityId}"]`)); els.forEach(el => { el.classList.remove('Active'); @@ -155,6 +151,14 @@ const onFaceMouseLeave = (e: any, face: FaceData) => { const Face = ({ face, onFaceClick, title, ...rest }: any) => { const faceId = face.faceId; const idPath = String(faceId % 100).padStart(2, '0'); + const img = faceId === -1 + ?
?
+ : ; return (
{ onMouseLeave={(e) => { onFaceMouseLeave(e, face) }} className='Face'>
- + { img }
{title}
@@ -185,8 +184,6 @@ type ClusterProps = { }; const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) => { - console.log(identity); - const relatedFacesJSX = useMemo(() => { const faceClicked = async (e: any, face: FaceData) => { if (!identity) { @@ -267,13 +264,37 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(filtered) }); - const data = await res.json(); + const updated = await res.json(); setIdentity({ ...identity }); } catch (error) { console.error(error); } }; + const createIdentity = async () => { + try { + const validFields = [ + 'id', 'displayName', 'firstName', 'lastName', 'middleName']; + const filtered: any = Object.assign({}, identity); + for (let key in filtered) { + if (validFields.indexOf(key) == -1) { + delete filtered[key] + } + } + const res = await window.fetch( + `${base}/api/v1/identities/`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(filtered) + }); + const created = await res.json(); + setIdentity(created); + } catch (error) { + console.error(error); + } + }; + + if (identity === undefined) { return (
Select identity to load. @@ -300,6 +321,7 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) value={identity.displayName} onChange={displayNameChanged} /> +
Faces: {identity.relatedFaces.length}
@@ -313,10 +335,10 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) type FaceData = { faceId: number, photoId: number, - lastName: string, + /* lastName: string, firstName: string, middleName: string, - displayName: string, + displayName: string,*/ identity: IdentityData, identityId: number, distance: number, @@ -500,11 +522,7 @@ const App = () => { }; const onFaceClick = (e: any, face: FaceData) => { - if (!face.identity) { - console.log(`Face ${face.faceId} does not have an Identity`); - return; - } - const identityId = face.identity.identityId; + const identityId = face.identityId; const faceId = face.faceId; console.log(`onFaceClick`, { faceId, identityId}); const faces = [ @@ -516,7 +534,7 @@ const App = () => { }; const identitiesOnFaceClick = (e: any, face: FaceData) => { - const identityId = face.identity.identityId; + const identityId = face.identityId; loadIdentity(identityId); } diff --git a/server/development.location b/server/development.location index 189aec2..b46bb92 100644 --- a/server/development.location +++ b/server/development.location @@ -1,6 +1,9 @@ # DEVELOPMENT -- use npm development server on port 3000 (entrypoint.sh) location /identities/api/v1/ { - rewrite ^/identities/api/v1/(.*)$ https://${host}/api/v1/$1 permanent; + rewrite ^/identities/api/v1/(.*)$ /api/v1/$1 break; + proxy_pass https://localhost/; + proxy_redirect off; + proxy_set_header Host $host; } location /identities { diff --git a/server/routes/identities.js b/server/routes/identities.js index 8a4193a..528bf29 100755 --- a/server/routes/identities.js +++ b/server/routes/identities.js @@ -11,6 +11,109 @@ require("../db/photos").then(function(db) { const router = express.Router(); +const addOrUpdateIdentity = async(id, { + displayName, + firstName, + lastName, + middleName + }, res) => { + + if (displayName === undefined + || firstName === undefined + || lastName === undefined + || middleName === undefined) { + res.status(400).send({ message: `Missing fields` }); + return undefined; + } + + const identity = { + displayName, + firstName, + lastName, + middleName, + id + }; + + if (id === -1 || !id) { + const [results, { lastId }] = await photoDB.sequelize.query( + `INSERT INTO identities ` + + '(displayName,firstName,lastName,middleName)' + + 'VALUES(:displayName,:firstName,:lastName,:middleName)', { + replacements: identity + }); + identity.id = lastId; + } else { + await photoDB.sequelize.query( + `UPDATE identities ` + + 'SET ' + + 'displayName=:displayName, ' + + 'firstName=:firstName, ' + + 'lastName=:lastName, ' + + 'middleName=:middleName ' + + 'WHERE id=:id', { + replacements: identity + }); + } + + return identity; +}; + + +const populateRelatedFaces = async (identity, count) => { + let limit = ''; + if (count) { + limit = ` LIMIT ${count} `; + } + /* If this is a new identity, no faces are being requested -- + * just return the empty 'unknown face'. + * + * Otherwise, query the DB for 'count' faces */ + if (count === undefined) { + identity.relatedFaces = await photoDB.sequelize.query( + "SELECT id AS faceId,photoId,faceConfidence " + + "FROM faces " + + "WHERE identityId=:identityId " + + limit, { + replacements: { identityId: identity.id }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }); + } else { + identity.relatedFaces = []; + } + + /* If there are no faces, return the 'unknown face' */ + if (identity.relatedFaces.length === 0) { + identity.relatedFaces.push({ + faceId: -1, + photoId: -1, + faceConfidence: 0 + }); + } + + identity.relatedFaces.forEach(face => { + face.identityId = identity.id; + face.distance = face.faceConfidence; + face.descriptors = []; + delete face.faceConfidence; + }); +} + +router.post('/', async (req, res) => { + console.log(`POST ${req.url}`) + if (!req.user.maintainer) { + console.warn(`${req.user.name} attempted to modify photos.`); + return res.status(401).send({ message: "Unauthorized to modify photos." }); + } + + const identity = await addOrUpdateIdentity(-1, req.body, res); + if (!identity) { + return; + } + populateRelatedFaces(identity, 1); + return res.status(200).send(identity); +}); + router.put('/:id', async (req, res) => { console.log(`PUT ${req.url}`) if (!req.user.maintainer) { @@ -23,45 +126,12 @@ router.put('/:id', async (req, res) => { return res.status(400).send({ message: `Invalid identity id ${id}` }); } - const { - displayName, - firstName, - lastName, - middleName - } = req.body; - - if (displayName === undefined - || firstName === undefined - || lastName === undefined - || middleName === undefined) { - return res.status(400).send({ message: `Missing fields` }); + const identity = await addOrUpdateIdentity(id, req.body, res); + if (!identity) { + return; } - - await photoDB.sequelize.query( - 'UPDATE identities ' + - 'SET ' + - 'displayName=:displayName, ' + - 'firstName=:firstName, ' + - 'lastName=:lastName, ' + - 'middleName=:middleName ' + - 'WHERE id=:id', { - replacements: { - displayName, - firstName, - lastName, - middleName, - id - } - } - ); - - return res.status(200).json({ - displayName, - firstName, - lastName, - middleName, - id - }); + populateRelatedFaces(identity); + return res.status(200).send(identity); }); router.put("/faces/remove/:id", (req, res) => { @@ -99,7 +169,7 @@ router.put("/faces/remove/:id", (req, res) => { }); }); -router.put("/faces/add/:id", (req, res) => { +router.put("/faces/add/:id", async (req, res) => { if (!req.user.maintainer) { console.warn(`${req.user.name} attempted to modify photos.`); return res.status(401).send("Unauthorized to modify photos."); @@ -114,23 +184,24 @@ router.put("/faces/add/:id", (req, res) => { return res.status(400).send("No faces supplied."); } - return photoDB.sequelize.query( - "UPDATE faces SET identityId=:identityId " + - "WHERE id IN (:faceIds)", { - replacements: { - identityId: id, - faceIds: req.body.faces - } - }).then(() => { + try { + await photoDB.sequelize.query( + "UPDATE faces SET identityId=:identityId,classifiedBy='human' " + + "WHERE id IN (:faceIds)", { + replacements: { + identityId: id, + faceIds: req.body.faces + } + }); const identity = { id: id, faces: req.body.faces }; return res.status(200).json([identity]); - }).catch((error) => { + } catch (error) { console.error(error); return res.status(500).send("Error processing request."); - }); + }; }); router.post("/", (req, res) => { @@ -192,6 +263,42 @@ function euclideanDistance(a, b) { return Math.sqrt(sum); } +const getUnknownIdentity = async (faceCount) => { + const unknownIdentity = { + identityId: -1, + lastName: '', + firstName: '', + middleName: '', + displayName: 'Unknown', + descriptors: [], + relatedFaces: [] + }; + const limit = faceCount + ? ` LIMIT ${faceCount} ` + : ' ORDER BY faceConfidence DESC '; + unknownIdentity.relatedFaces = await photoDB.sequelize.query( + "SELECT id AS faceId,photoId,faceConfidence " + + "FROM faces WHERE identityId IS NULL AND classifiedBy != 'not-a-face' " + + limit, { + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }); + if (unknownIdentity.relatedFaces.length === 0) { + unknownIdentity.relatedFaces.push({ + faceId: -1, + photoId: -1, + faceConfidence: 0 + }); + } + unknownIdentity.relatedFaces.forEach(face => { + face.identityId = -1; + face.distance = face.faceConfidence; + face.descriptors = []; + delete face.faceConfidence; + }); + return unknownIdentity; +} + router.get("/:id?", async (req, res) => { console.log(`GET ${req.url}`); @@ -204,6 +311,13 @@ router.get("/:id?", async (req, res) => { } } + /* If identityId requested is -1, this is the "Unknown" identity + * where all unmapped faces live. */ + if (id === -1) { + const unknownIdentity = await getUnknownIdentity() + return res.status(200).json([ unknownIdentity ]); + } + const filter = id ? "WHERE identities.id=:id " : ""; const identities = await photoDB.sequelize.query("SELECT " + @@ -212,14 +326,14 @@ router.get("/:id?", async (req, res) => { "GROUP_CONCAT(faces.descriptorId) AS relatedFaceDescriptorIds," + "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds " + "FROM identities " + - "INNER JOIN faces ON identities.id=faces.identityId " + + "LEFT JOIN faces ON identities.id=faces.identityId " + filter + "GROUP BY identities.id", { replacements: { id }, type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }); - + await Promise.map(identities, async (identity) => { [ 'firstName', 'middleName', 'lastName' ].forEach(key => { if (!identity[key]) { @@ -228,35 +342,49 @@ router.get("/:id?", async (req, res) => { }); identity.identityId = identity.id; - const relatedFaces = identity.relatedFaceIds.split(","), - relatedFacePhotos = identity.relatedFacePhotoIds.split(","); + if (!identity.relatedFaceIds) { + identity.relatedFaces = []; + } else { + const relatedFaces = identity.relatedFaceIds.split(","), + relatedFacePhotos = identity.relatedFacePhotoIds.split(","); - let descriptors = await photoDB.sequelize.query( - `SELECT descriptors FROM facedescriptors WHERE id in (:ids)`, { - replacements: { - ids: identity.relatedFaceDescriptorIds.split(',') - }, - type: photoDB.Sequelize.QueryTypes.SELECT, - raw: true - } - ); - - descriptors = descriptors.map(entry => entry.descriptors); - - identity.relatedFaces = relatedFaces.map((faceId, index) => { - const distance = euclideanDistance( - descriptors[index], - identity.descriptors + let descriptors = await photoDB.sequelize.query( + `SELECT descriptors FROM facedescriptors WHERE id in (:ids)`, { + replacements: { + ids: identity.relatedFaceDescriptorIds.split(',') + }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + } ); + + descriptors = descriptors.map(entry => entry.descriptors); - return { + identity.relatedFaces = relatedFaces.map((faceId, index) => { + const distance = euclideanDistance( + descriptors[index], + identity.descriptors + ); + + return { + identityId: identity.id, + faceId, + photoId: relatedFacePhotos[index], + distance + }; + }); + } + + if (identity.relatedFaces.length === 0) { + identity.relatedFaces.push({ + faceId: -1, + photoId: -1, identityId: identity.id, - faceId, - photoId: relatedFacePhotos[index], - distance - }; - }); - + distance: 0, + faceConfidence: 0 + }); + } + identity .relatedFaces .sort((A, B) => { @@ -277,6 +405,14 @@ router.get("/:id?", async (req, res) => { delete identity.relatedIdentityDescriptors; }); + /* If no ID was provided (so no 'filter') then this call is returning + * a list of all identities -- we create a fake identity for all + * unlabeled faces */ + if (!filter) { + const unknownIdentity = await getUnknownIdentity(1) + identities.push(unknownIdentity); + } + return res.status(200).json(identities); });