diff --git a/client/src/App.tsx b/client/src/App.tsx index d9410f7..04c78d9 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -1,6 +1,5 @@ import React, { useState, useMemo, useEffect, useRef, useCallback } from 'react'; -import { useApi } from './useApi'; import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels"; import { BrowserRouter as Router, @@ -224,20 +223,42 @@ const Cluster = ({ setSelected, setImage, }: ClusterProps) => { - + const [lastName, setLastName] = useState(identity.lastName); + const [firstName, setFirstName] = useState(identity.firstName); + const [middleName, setMiddleName] = useState(identity.middleName); + const [displayName, setDisplayName] = useState(identity.displayName); + const [updated, setUpdated] = useState(false); + const lastNameChanged = (e: any) => { - setIdentity({...identity, lastName: e.currentTarget.value }); + setLastName(e.currentTarget.value); }; const firstNameChanged = (e: any) => { - setIdentity({...identity, firstName: e.currentTarget.value }); + setFirstName(e.currentTarget.value); }; const middleNameChanged = (e: any) => { - setIdentity({...identity, middleName: e.currentTarget.value }); + setMiddleName(e.currentTarget.value); }; const displayNameChanged = (e: any) => { - setIdentity({...identity, displayName: e.currentTarget.value }); + setDisplayName(e.currentTarget.value); }; + /* If the user edits the identity, set the "updated" flag */ + useEffect(() => { + setUpdated(lastName !== identity.lastName + || firstName !== identity.firstName + || middleName !== identity.middleName + || displayName !== identity.displayName + ); + }, [setUpdated, identity, lastName, firstName, middleName, displayName]); + + /* If the identity changes, update all the fields */ + useEffect(() => { + setLastName(identity.lastName); + setFirstName(identity.firstName); + setMiddleName(identity.middleName); + setDisplayName(identity.displayName); + }, [identity]); + const faceClicked = useCallback((e: any, face: FaceData) => { const el = e.currentTarget; @@ -262,6 +283,9 @@ const Cluster = ({ }, [setSelected, setImage]); const deleteIdentity = async () => { + if (!identity || identity.identityId === -1) { + return; + } try { const res = await window.fetch( `${base}/api/v1/identities/${identity.identityId}`, { @@ -283,23 +307,39 @@ const Cluster = ({ }; const updateIdentity = async () => { + if (!identity || identity.identityId === -1) { + return; + } try { - const validFields = [ - 'id', 'displayName', 'firstName', 'lastName', 'middleName']; - const filtered: any = Object.assign({}, identity); - for (let key in filtered) { - if (validFields.indexOf(key) === -1) { - delete filtered[key] - } - } + const values = { + lastName, + firstName, + middleName, + displayName + }; + const res = await window.fetch( `${base}/api/v1/identities/${identity.identityId}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(filtered) + body: JSON.stringify(values) }); await res.json(); - setIdentity({ ...identity }); + setIdentity({ ...identity, ...values }); + setIdentities( + [...identities] + .sort((A: IdentityData, B: IdentityData) => { + /* Sort the Unknown (-1) identity to the end */ + if (A.identityId === -1) { + return +1; + } + if (B.identityId === -1) { + return -1; + } + /* Otherwise sort alphabetically by displayName */ + return A.displayName.localeCompare(B.displayName); + }) + ); } catch (error) { console.error(error); } @@ -307,23 +347,34 @@ const Cluster = ({ const createIdentity = async () => { try { - const validFields = [ - 'id', 'displayName', 'firstName', 'lastName', 'middleName']; - const filtered: any = Object.assign({}, identity); - for (let key in filtered) { - if (validFields.indexOf(key) === -1) { - delete filtered[key] - } - } + const values = { + lastName, + firstName, + middleName, + displayName + }; const res = await window.fetch( `${base}/api/v1/identities/`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(filtered) + body: JSON.stringify(values) }); const created = await res.json(); setIdentity(created); - setIdentities([identity, ...identities]); + setIdentities( + [created, ...identities] + .sort((A: IdentityData, B: IdentityData) => { + /* Sort the Unknown (-1) identity to the end */ + if (A.identityId === -1) { + return +1; + } + if (B.identityId === -1) { + return -1; + } + /* Otherwise sort alphabetically by displayName */ + return A.displayName.localeCompare(B.displayName); + }) + ); } catch (error) { console.error(error); } @@ -342,31 +393,32 @@ const Cluster = ({
Last name:
First name:
Middle name:
Display name:
{}} - title={`${identity.displayName} (${identity.facesCount})`} /> + title={`${displayName} (${identity.facesCount})`} />
{ identity.identityId !== -1 && <> - + { updated && } }
@@ -447,27 +499,33 @@ interface IdentitiesProps { }; const Identities = ({ identities, onFaceClick } : IdentitiesProps) => { - const identitiesJSX = identities.map((identity) => { - const face = identity.relatedFaces[0]; - return ( -
- -
- ); - }); + const [jsx, setJsx] = useState([]); + useEffect(() => { + setJsx(identities.map((identity) => { + const face = { + faceId: identity.faceId, + identityId: identity.identityId + }; + return ( +
+ +
+ ); + })); + }, [identities, onFaceClick]); return (
- { identitiesJSX } + { jsx }
); }; @@ -481,15 +539,13 @@ const Button = ({ onClick, children }: any) => { }; const App = () => { - const [identities, setIdentities] = useState([]); const { identityId, faceId } = useParams(); + const [identities, setIdentities] = useState([]); const [selectedIdentities, setSelectedIdentities] = useState([]); const [identity, setIdentity] = useState(EmptyIdentity); const [image, setImage] = useState(0); const [guess, setGuess] = useState(undefined); - const { loading, data } = useApi( /* TODO: Switch away from using useApi */ - `${base}/api/v1/identities` - ); + const [loaded, setLoaded] = useState(false); const [selected, setSelected] = useState([]); /* If 'selected' changes, clear any selected face which is not in the @@ -515,9 +571,10 @@ const App = () => { } }; - /* If the identity changes, update its entry in the identities list */ + /* If the identity changes, update its entry in the identities list + * NOTE: Blocks update to 'Unknown' (-1) fake identity */ useEffect(() => { - if (!identity || identities.length === 0) { + if (!identity || identities.length === 0 || identity.identityId === -1) { return; } for (let key in identities) { @@ -525,9 +582,11 @@ const App = () => { let same = true; [ 'displayName', 'firstName', 'lastName', 'middleName' ] .forEach((field: string) => { - same = same && (identities[key] as any)[field] === (identity as any)[field]; + same = same + && (identities[key] as any)[field] === (identity as any)[field]; }); if (!same) { + console.log(`Updating `, identity, identities[key]); identities[key] = { ...identity, relatedFaces: identities[key].relatedFaces @@ -543,6 +602,22 @@ const App = () => { } }, [identity, setIdentities, identities]); + /* If the identity changes, scroll it into view in the Identities list */ + useEffect(() => { + if (selectedIdentities.length !== 0) { + return; + } + const el = document.querySelector( + `.Identities [data-identity-id="${identity.identityId}"]`); + if (el) { + el.scrollIntoView({ + behavior: "smooth", + block: "nearest", + inline: "nearest" + }); + } + }, [identity, selectedIdentities]); + useEffect(() => { if (identityId !== undefined && !isNaN(+identityId)) { loadIdentity(+identityId); @@ -555,20 +630,38 @@ const App = () => { }, []); useEffect(() => { - if (data && data.length) { + if (identities.length !== 0 || loaded) { + return; + } + + const loadIdentities = async () => { + const res = await window.fetch(`${base}/api/v1/identities`, { + headers: { 'Content-Type': 'application/json' }, + }); + const data = await res.json(); data.forEach((identity: IdentityData) => { identity.relatedFaces.forEach(face => { face.identity = identity; }); }); data.sort((A: IdentityData, B: IdentityData) => { + /* Sort the Unknown (-1) identity to the end */ + if (A.identityId === -1) { + return +1; + } + if (B.identityId === -1) { + return -1; + } + /* Otherwise sort alphabetically by displayName */ return A.displayName.localeCompare(B.displayName); }); + setLoaded(true); setIdentities(data as IdentityData[]); } - }, [data]); + loadIdentities(); + }, [identities, setIdentities, setLoaded, loaded]); - const removeFacesFromIdentities = (faceIds: number[]) => { + const removeFacesFromIdentity = (faceIds: number[]) => { if (!identity) { return; } @@ -578,6 +671,7 @@ const App = () => { identity.relatedFaces = identity.relatedFaces.filter( (face: FaceData) => faceIds.indexOf(face.faceId) === -1); if (pre !== identity.relatedFaces.length) { + identity.facesCount = identity.relatedFaces.length; setIdentity({ ...identity }) } } @@ -587,7 +681,7 @@ const App = () => { window.alert('You need to select an identity first (CTRL+CLICK)'); return; } - if (!identity) { + if (!identity || identity.identityId === -1) { return; } try { @@ -598,18 +692,26 @@ const App = () => { body: JSON.stringify({ faces: identity.relatedFaces .map(face => face.faceId) }) }); - await res.json(); + const result = await res.json(); res = await window.fetch( `${base}/api/v1/identities/${identity.identityId}`, { method: 'DELETE', headers: { 'Content-Type': 'application/json' } }); await res.json(); - const index = identities + /* Delete the identity from the list of identities */ + const deleted = identities .findIndex((item: IdentityData) => item.identityId === identity.identityId); - if (index !== -1) { - identities.splice(index, 1); + if (deleted !== -1) { + identities.splice(deleted, 1); + } + /* Update the faces count on the target identity */ + const target = identities + .find((item: IdentityData) => + item.identityId === selectedIdentities[0]); + if (target) { + target.facesCount += result.added.length; } setIdentity(EmptyIdentity); setIdentities([...identities]); @@ -629,10 +731,14 @@ const App = () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ faces: selected }) }); - const data = await res.json(); + const results = await res.json(); - removeFacesFromIdentities(data.faces); + removeFacesFromIdentity(results.removed); deselectAll(); + if (identity.faceId !== results.faceId) { + setIdentity({...identity, ...{ faceId: results.faceId }}); + setIdentities([...identities]); + } } catch (error) { console.error(error); } @@ -650,10 +756,29 @@ const App = () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ faces: selected }) }); - const data = await res.json(); + const results = await res.json(); - removeFacesFromIdentities(data.faces); + removeFacesFromIdentity(results.added); + /* If the identity faceId was removed from the identity, select + * the next relatedFace */ + if (results.added.indexOf(identity.faceId) !== -1) { + if (identity.relatedFaces.length === 0) { + identity.faceId = -1; + } else { + identity.faceId = identity.relatedFaces[0].faceId; + } + setIdentity({...identity}); + } + /* Update the faces count on the target identity */ + const target = identities + .find((item: IdentityData) => + item.identityId === selectedIdentities[0]); + if (target) { + target.facesCount += results.added.length; + target.faceId = results.faceId; + } deselectAll(); + setIdentities([...identities]); } catch (error) { console.error(error); } @@ -685,7 +810,7 @@ const App = () => { }) }); await res.json(); - removeFacesFromIdentities(selected); + removeFacesFromIdentity(selected); deselectAll(); } catch (error) { console.error(error); @@ -707,7 +832,11 @@ const App = () => { ...document.querySelectorAll(`.Identities [data-identity-id="${identityId}"]`), ...document.querySelectorAll(`.Cluster [data-face-id="${faceId}"]`)]; faces.forEach((el: any) => { - el.scrollIntoView(); + el.scrollIntoView({ + behavior: "smooth", + block: "nearest", + inline: "nearest" + }); }); }; @@ -742,12 +871,14 @@ const App = () => { e.stopPropagation(); e.preventDefault(); - [...document.querySelectorAll('.Cluster .Faces img')] - .forEach((img: any) => { - img.src = 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7'; - }); - - loadIdentity(identityId); + if (identity.identityId !== identityId) { + [...document.querySelectorAll('.Cluster .Faces img')] + .forEach((img: any) => { + img.src = 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7'; + }); + + loadIdentity(identityId); + } } return ( @@ -756,19 +887,15 @@ const App = () => { - {loading &&
Loading...
} - - {!loading && identity === undefined &&
- Select identity to edit -
} +
{ selected.length === 1 && <> @@ -801,7 +928,10 @@ const App = () => { - { !loading && + Loading... +
} + { loaded && }
diff --git a/server/routes/identities.js b/server/routes/identities.js index e3fd017..a7fa0c6 100755 --- a/server/routes/identities.js +++ b/server/routes/identities.js @@ -28,22 +28,27 @@ const upsertIdentity = async(id, { return undefined; } - const identity = { - displayName, - firstName, - lastName, - middleName, - identityId: id + /* Create identity structure based on UnknownIdentity to set + * default values for new identities */ + const identity = { + ...UnknownIdentity, + ...{ + displayName, + firstName, + lastName, + middleName, + identityId: id + } }; if (id === -1 || !id) { - const [results, { lastId }] = await photoDB.sequelize.query( + const [, { lastID }] = await photoDB.sequelize.query( `INSERT INTO identities ` + '(displayName,firstName,lastName,middleName)' + 'VALUES(:displayName,:firstName,:lastName,:middleName)', { replacements: identity }); - identity.identityId = lastId; + identity.identityId = lastID; console.log('Created identity: ', identity) } else { await photoDB.sequelize.query( @@ -282,8 +287,39 @@ router.put("/faces/remove/:id", async (req, res) => { identityId: id, faces: req.body.faces }; - identity.faces = identity.faces.map(id => +id); + identity.removed = identity.faces.map(id => +id); + /* If the primary faceId was removed, update the identity's faceId + * to a new faceId */ + const faceIds = await photoDB.sequelize.query(` + SELECT faceId FROM identities WHERE id=:identityId`, { + replacements: identity, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }); + if (identity.removed.indexOf(faceIds[0]) !== -1) { + const newFaceId = await photoDB.sequelize.query(` + SELECT faceId FROM faces WHERE identityId=:identityId + ORDER BY distance ASC + LIMIT 1`, { + replacements: identity, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + } + ); + if (newFaceId.length === 0) { + identity.faceId = -1; + } else { + identity.faceId = newFaceId[0]; + } + await photoDB.sequelize.query(` + UPDATE identities + SET faceId=${identity.faceId} + WHERE id=${identity.identityId} + `); + console.log( + `New faceId for ${identity.identityId} set to ${identity.faceId}.`); + } /* Do not block on this call finishing -- update can occur * in the background */ updateIdentityFaces(identity); @@ -301,8 +337,8 @@ router.put("/faces/add/:id", async (req, res) => { return res.status(401).send("Unauthorized to modify photos."); } - const id = parseInt(req.params.id); - if (id != req.params.id) { + const identityId = parseInt(req.params.id); + if (identityId != req.params.id) { return res.status(400).send("Invalid identity id."); } @@ -310,24 +346,107 @@ router.put("/faces/add/:id", async (req, res) => { return res.status(400).send("No faces supplied."); } + /* Convert faces array to numbers and filter any non-numbers */ + let faceIds = req.body.faces + .map(faceId => +faceId) + .filter(faceId => !isNaN(faceId)); + + /* See which identities currently have these faces (if any) */ + let tuples = await photoDB.sequelize.query(` + SELECT + faces.identityId, + faces.id AS faceId + FROM + faces + WHERE faces.id IN (:faceIds)`, { + replacements: { + identityId, + faceIds + }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + } + ); + /* Filter out any faces which are already owned by this identity */ + tuples = tuples + .filter(tuple => tuple.identityId !== identityId); + if (tuples.length === 0) { + return res.status(400).json({ + message: 'No faceIds provided not owned by identity.' + }); + } + + /* Obtain faceId from all referenced identities (src and dsts) */ + const identityIds = [ + identityId, + ...tuples + .filter(tuple => + tuple.identityId !== null + && tuple.identityId != -1) + .map(tuple => tuple.identityId) + ]; + + let identities = await photoDB.sequelize.query(` + SELECT id AS identityId, faceId AS identityFaceId + FROM identities + WHERE id IN (:identityIds)`, { + replacements: { + identityIds + }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + } + ); + + /* Find the src identity */ + const identity = identities.find(tuple => tuple.identityId === identityId); + + /* Merge the identities and the tuples, first filtering out the src + * identity */ + tuples = tuples + .filter(tuple => tuple.identityId !== identityId) + .map(tuple => { + return { + faceId: tuple.faceId, + identityId: tuple.identityId, + identityFaceId: (tuple.identityId === null + || tuple.identityId === -1) + ? -1 + : identities + .find(identity => tuple.identityId === identity.identityId) + .identityFaceId + } + }); + + console.log({ dst: identity, src: tuples }); + console.log(`Need new faceId: `, + tuples.filter(tuple => tuple.faceId === tuple.identityFaceId)); + try { await photoDB.sequelize.query( "UPDATE faces SET identityId=:identityId,classifiedBy='human' " + "WHERE id IN (:faceIds)", { replacements: { - identityId: id, - faceIds: req.body.faces + identityId, + faceIds } }); - const identity = { - identityId: id, - faces: req.body.faces - }; - identity.faces = identity.faces.map(id => +id); + + identity.added = faceIds; + identity.faceId = identity.identityFaceId; + delete identity.identityFaceId; + + if (identity.faceId === -1 || identity.faceId === null) { + identity.faceId = faceIds[0]; + } /* Do not block on this call finishing -- update can occur * in the background */ - updateIdentityFaces(identity); + Promise.map([identity, ...tuples], identity => { + updateIdentityFaces(identity); + }, { + concurrency: 1 + }); return res.status(200).json(identity); } catch (error) { console.error(error); @@ -335,7 +454,6 @@ router.put("/faces/add/:id", async (req, res) => { }; }); - function bufferToFloat32Array(buffer) { return new Float64Array(buffer.buffer, buffer.byteOffset, @@ -362,39 +480,63 @@ function euclideanDistance(a, b) { ); } +const UnknownFace = { + faceId: -1, + identityId: -1, + photoId: -1, + distance: 0, + faceConfidence: 0 +}; + +const UnknownIdentity = { + identityId: -1, + lastName: '', + firstName: '', + middleName: '', + displayName: 'Unknown', + descriptors: new Float32Array(0), + relatedFaces: [ UnknownFace ], + facesCount: 0, + faceId: -1 +}; + const getUnknownIdentity = async (faceCount) => { - const unknownIdentity = { - identityId: -1, - lastName: '', - firstName: '', - middleName: '', - displayName: 'Unknown', - descriptors: new Float32Array(0), - relatedFaces: [], - facesCount: 0 - }; + const unknownIdentity = { ...UnknownIdentity }; const limit = faceCount ? ` ORDER BY RANDOM() LIMIT ${faceCount} ` : ' ORDER BY faceConfidence DESC '; - unknownIdentity.relatedFaces = await photoDB.sequelize.query( - "SELECT id AS faceId,photoId,faceConfidence " + - "FROM faces WHERE identityId IS NULL AND classifiedBy != 'not-a-face' " + - limit, { + const sql = ` + SELECT + faces.id AS faceId, + faces.photoId, + faces.faceConfidence, + total.facesCount + FROM + faces, + (SELECT COUNT(total.id) AS facesCount + FROM + faces AS total + WHERE + total.identityId IS NULL + AND total.classifiedBy != 'not-a-face') AS total + WHERE + faces.identityId IS NULL + AND faces.classifiedBy != 'not-a-face' + ${ limit }`; + + unknownIdentity.relatedFaces = await photoDB.sequelize.query(sql, { type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }); - if (unknownIdentity.relatedFaces.length === 0) { - unknownIdentity.relatedFaces.push({ - faceId: -1, - photoId: -1, - faceConfidence: 0 - }); + if (unknownIdentity.relatedFaces.length !== 0) { + unknownIdentity.facesCount = unknownIdentity.relatedFaces[0].facesCount; } unknownIdentity.relatedFaces.forEach(face => { face.identityId = -1; face.distance = face.faceConfidence; face.descriptors = new Float32Array(0); delete face.faceConfidence; + delete face.facesCount; }); return unknownIdentity; } @@ -452,9 +594,7 @@ const updateIdentityFaces = async (identity) => { return; } - let average = undefined, - closestId = -1, - closestDistance = -1; + let average = undefined; /* First find the average centroid of all faces */ faces.forEach((face) => { @@ -479,17 +619,18 @@ const updateIdentityFaces = async (identity) => { average[i] = average[i] / faces.length; } + let closestId = -1, + closestDistance = -1; + /* Now compute the distance from each face to the new centroid */ faces.forEach((face) => { - let distance; - - distance = euclideanDistanceArray( + face.updatedDistance = euclideanDistanceArray( face.descriptors, average ); - if (closestId === -1 || face.distance < closestDistance) { - closestDistance = distance; + if (closestId === -1 || face.updatedDistance < closestDistance) { + closestDistance = face.updatedDistance; closestId = face.id; } }); @@ -499,6 +640,7 @@ const updateIdentityFaces = async (identity) => { if (!identity.descriptors) { console.log(`Identity ${identity.identityId} has no descriptors`); } + let moved = (identity.descriptors === null ? 1 : 0) || Number .parseFloat(euclideanDistanceArray(identity.descriptors, average)) @@ -511,15 +653,17 @@ const updateIdentityFaces = async (identity) => { await Promise.map(faces, async (face) => { /* All the buffer are already arrays, so use the short-cut version */ const distance = Number - .parseFloat(euclideanDistanceArray(face.descriptors, average)) + .parseFloat(face.updatedDistance) .toFixed(4); - if (Math.abs(distance - face.distance) > MIN_DISTANCE_COMMIT) { + if (Math.abs(face.updatedDistance - face.distance) + > MIN_DISTANCE_COMMIT) { console.log( `Updating face ${face.id} to ${round(distance, 2)} ` + `(${distance - face.distance}) ` + `from identity ${identity.identityId} (${identity.displayName})`); - face.distance = distance; + face.distance = face.updatedDistance; + delete face.updatedDistance; await photoDB.sequelize.query( 'UPDATE faces SET distance=:distance WHERE id=:id', { replacements: face, @@ -677,11 +821,8 @@ router.get("/:id?", async (req, res) => { /* If there were no faces, then add a 'Unknown' face */ if (identity.relatedFaces.length === 0) { identity.relatedFaces.push({ - faceId: -1, - photoId: -1, - identityId: identity.identityId, - distance: 0, - faceConfidence: 0 + ...UnknownFace, + ...{ identityId: identity.identityId } }); } }, {